Index: projects/runtime-coverage/Makefile.inc1 =================================================================== --- projects/runtime-coverage/Makefile.inc1 (revision 321584) +++ projects/runtime-coverage/Makefile.inc1 (revision 321585) @@ -1,2862 +1,2864 @@ # # $FreeBSD$ # # Make command line options: # -DNO_CLEANDIR run ${MAKE} clean, instead of ${MAKE} cleandir # -DNO_CLEAN do not clean at all # -DDB_FROM_SRC use the user/group databases in src/etc instead of # the system database when installing. # -DNO_SHARE do not go into share subdir # -DKERNFAST define NO_KERNEL{CONFIG,CLEAN,OBJ} # -DNO_KERNELCONFIG do not run config in ${MAKE} buildkernel # -DNO_KERNELCLEAN do not run ${MAKE} clean in ${MAKE} buildkernel # -DNO_KERNELOBJ do not run ${MAKE} obj in ${MAKE} buildkernel # -DNO_PORTSUPDATE do not update ports in ${MAKE} update # -DNO_ROOT install without using root privilege # -DNO_DOCUPDATE do not update doc in ${MAKE} update # -DWITHOUT_CTF do not run the DTrace CTF conversion tools on built objects # LOCAL_DIRS="list of dirs" to add additional dirs to the SUBDIR list # LOCAL_ITOOLS="list of tools" to add additional tools to the ITOOLS list # LOCAL_LIB_DIRS="list of dirs" to add additional dirs to libraries target # LOCAL_MTREE="list of mtree files" to process to allow local directories # to be created before files are installed # LOCAL_TOOL_DIRS="list of dirs" to add additional dirs to the build-tools # list # LOCAL_XTOOL_DIRS="list of dirs" to add additional dirs to the # cross-tools target # METALOG="path to metadata log" to write permission and ownership # when NO_ROOT is set. (default: ${DESTDIR}/METALOG) # TARGET="machine" to crossbuild world for a different machine type # TARGET_ARCH= may be required when a TARGET supports multiple endians # BUILDENV_SHELL= shell to launch for the buildenv target (def:${SHELL}) # WORLD_FLAGS= additional flags to pass to make(1) during buildworld # KERNEL_FLAGS= additional flags to pass to make(1) during buildkernel # SUBDIR_OVERRIDE="list of dirs" to build rather than everything. # All libraries and includes, and some build tools will still build. # # The intended user-driven targets are: # buildworld - rebuild *everything*, including glue to help do upgrades # installworld- install everything built by "buildworld" # checkworld - run test suite on installed world # doxygen - build API documentation of the kernel # update - convenient way to update your source tree (eg: svn/svnup) # # Standard targets (not defined here) are documented in the makefiles in # /usr/share/mk. These include: # obj depend all install clean cleandepend cleanobj .if !defined(TARGET) || !defined(TARGET_ARCH) .error "Both TARGET and TARGET_ARCH must be defined." .endif SRCDIR?= ${.CURDIR} LOCALBASE?= /usr/local # Cross toolchain changes must be in effect before bsd.compiler.mk # so that gets the right CC, and pass CROSS_TOOLCHAIN to submakes. .if defined(CROSS_TOOLCHAIN) .include "${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk" CROSSENV+=CROSS_TOOLCHAIN="${CROSS_TOOLCHAIN}" .endif .if defined(CROSS_TOOLCHAIN_PREFIX) CROSS_COMPILER_PREFIX?=${CROSS_TOOLCHAIN_PREFIX} .endif XCOMPILERS= CC CXX CPP .for COMPILER in ${XCOMPILERS} .if defined(CROSS_COMPILER_PREFIX) X${COMPILER}?= ${CROSS_COMPILER_PREFIX}${${COMPILER}} .else X${COMPILER}?= ${${COMPILER}} .endif .endfor # If a full path to an external cross compiler is given, don't build # a cross compiler. .if ${XCC:N${CCACHE_BIN}:M/*} MK_CLANG_BOOTSTRAP= no MK_GCC_BOOTSTRAP= no .endif MAKEOBJDIRPREFIX?= /usr/obj .if ${MACHINE} == ${TARGET} && ${MACHINE_ARCH} == ${TARGET_ARCH} && !defined(CROSS_BUILD_TESTING) OBJTREE= ${MAKEOBJDIRPREFIX} .else OBJTREE= ${MAKEOBJDIRPREFIX}/${TARGET}.${TARGET_ARCH} .endif # Pull in compiler metadata from buildworld/toolchain if possible to avoid # running CC from bsd.compiler.mk. .if make(installworld) || make(install) || make(distributeworld) || \ make(stageworld) .-include "${OBJTREE}${.CURDIR}/compiler-metadata.mk" .endif # Pull in COMPILER_TYPE and COMPILER_FREEBSD_VERSION early. .include .include "share/mk/src.opts.mk" # Check if there is a local compiler that can satisfy as an external compiler. # Which compiler is expected to be used? .if ${MK_CLANG_BOOTSTRAP} == "yes" WANT_COMPILER_TYPE= clang .elif ${MK_GCC_BOOTSTRAP} == "yes" WANT_COMPILER_TYPE= gcc .else WANT_COMPILER_TYPE= .endif .if !defined(WANT_COMPILER_FREEBSD_VERSION) .if ${WANT_COMPILER_TYPE} == "clang" WANT_COMPILER_FREEBSD_VERSION_FILE= lib/clang/freebsd_cc_version.h WANT_COMPILER_FREEBSD_VERSION!= \ awk '$$2 == "FREEBSD_CC_VERSION" {printf("%d\n", $$3)}' \ ${SRCDIR}/${WANT_COMPILER_FREEBSD_VERSION_FILE} || echo unknown WANT_COMPILER_VERSION_FILE= lib/clang/include/clang/Basic/Version.inc WANT_COMPILER_VERSION!= \ awk '$$2 == "CLANG_VERSION" {split($$3, a, "."); print a[1] * 10000 + a[2] * 100 + a[3]}' \ ${SRCDIR}/${WANT_COMPILER_VERSION_FILE} || echo unknown .elif ${WANT_COMPILER_TYPE} == "gcc" WANT_COMPILER_FREEBSD_VERSION_FILE= gnu/usr.bin/cc/cc_tools/freebsd-native.h WANT_COMPILER_FREEBSD_VERSION!= \ awk '$$2 == "FBSD_CC_VER" {printf("%d\n", $$3)}' \ ${SRCDIR}/${WANT_COMPILER_FREEBSD_VERSION_FILE} || echo unknown WANT_COMPILER_VERSION_FILE= contrib/gcc/BASE-VER WANT_COMPILER_VERSION!= \ awk -F. '{print $$1 * 10000 + $$2 * 100 + $$3}' \ ${SRCDIR}/${WANT_COMPILER_VERSION_FILE} || echo unknown .endif .export WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_VERSION .endif # !defined(WANT_COMPILER_FREEBSD_VERSION) # It needs to be the same revision as we would build for the bootstrap. # If the expected vs CC is different then we can't skip. # GCC cannot be used for cross-arch yet. For clang we pass -target later if # TARGET_ARCH!=MACHINE_ARCH. .if ${MK_SYSTEM_COMPILER} == "yes" && \ (${MK_CLANG_BOOTSTRAP} == "yes" || ${MK_GCC_BOOTSTRAP} == "yes") && \ !make(showconfig) && !make(native-xtools) && !make(xdev*) && \ ${WANT_COMPILER_TYPE} == ${COMPILER_TYPE} && \ (${COMPILER_TYPE} == "clang" || ${TARGET_ARCH} == ${MACHINE_ARCH}) && \ ${COMPILER_VERSION} == ${WANT_COMPILER_VERSION} && \ ${COMPILER_FREEBSD_VERSION} == ${WANT_COMPILER_FREEBSD_VERSION} # Everything matches, disable the bootstrap compiler. MK_CLANG_BOOTSTRAP= no MK_GCC_BOOTSTRAP= no USING_SYSTEM_COMPILER= yes .endif # ${WANT_COMPILER_TYPE} == ${COMPILER_TYPE} USING_SYSTEM_COMPILER?= no TEST_SYSTEM_COMPILER_VARS= \ USING_SYSTEM_COMPILER MK_SYSTEM_COMPILER \ MK_CROSS_COMPILER MK_CLANG_BOOTSTRAP MK_GCC_BOOTSTRAP \ WANT_COMPILER_TYPE WANT_COMPILER_VERSION WANT_COMPILER_VERSION_FILE \ WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_FREEBSD_VERSION_FILE \ CC COMPILER_TYPE COMPILER_FEATURES COMPILER_VERSION \ COMPILER_FREEBSD_VERSION \ LINKER_TYPE LINKER_VERSION test-system-compiler: .PHONY .for v in ${TEST_SYSTEM_COMPILER_VARS} ${_+_}@printf "%-35s= %s\n" "${v}" "${${v}}" .endfor .if ${USING_SYSTEM_COMPILER} == "yes" && \ (make(buildworld) || make(buildkernel) || make(kernel-toolchain) || \ make(toolchain) || make(_cross-tools)) .info SYSTEM_COMPILER: Determined that CC=${CC} matches the source tree. Not bootstrapping a cross-compiler. .endif # For installworld need to ensure that the looked-up compiler metadata is # passed along rather than trying to run cc from the restricted # STRICTTMPPATH. .if ${MK_CLANG_BOOTSTRAP} == "no" && ${MK_GCC_BOOTSTRAP} == "no" .if !defined(X_COMPILER_TYPE) CROSSENV+= COMPILER_VERSION=${COMPILER_VERSION} \ COMPILER_TYPE=${COMPILER_TYPE} \ COMPILER_FEATURES=${COMPILER_FEATURES} \ COMPILER_FREEBSD_VERSION=${COMPILER_FREEBSD_VERSION} .else CROSSENV+= COMPILER_VERSION=${X_COMPILER_VERSION} \ COMPILER_FEATURES=${X_COMPILER_FEATURES} \ COMPILER_TYPE=${X_COMPILER_TYPE} \ COMPILER_FREEBSD_VERSION=${X_COMPILER_FREEBSD_VERSION} .endif .endif # Store some compiler metadata for use in installworld where we don't # want to invoke CC at all. _COMPILER_METADATA_VARS= COMPILER_VERSION \ COMPILER_TYPE \ COMPILER_FEATURES \ COMPILER_FREEBSD_VERSION \ LINKER_VERSION \ LINKER_TYPE compiler-metadata.mk: .PHONY .META @: > ${.TARGET} @echo ".info Using cached compiler metadata from build at $$(hostname) on $$(date)" \ > ${.TARGET} .for v in ${_COMPILER_METADATA_VARS} @echo "${v}=${${v}}" >> ${.TARGET} .endfor @echo ".export ${_COMPILER_METADATA_VARS}" >> ${.TARGET} # Handle external binutils. .if defined(CROSS_TOOLCHAIN_PREFIX) CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX} .endif # If we do not have a bootstrap binutils (because the in-tree one does not # support the target architecture), provide a default cross-binutils prefix. # This allows riscv64 builds, for example, to automatically use the # riscv64-binutils port or package. .if !make(showconfig) .if !empty(BROKEN_OPTIONS:MBINUTILS_BOOTSTRAP) && \ ${MK_LLD_BOOTSTRAP} == "no" && \ !defined(CROSS_BINUTILS_PREFIX) CROSS_BINUTILS_PREFIX=/usr/local/${TARGET_ARCH}-freebsd/bin/ .if !exists(${CROSS_BINUTILS_PREFIX}) .error In-tree binutils does not support the ${TARGET_ARCH} architecture. Install the ${TARGET_ARCH}-binutils port or package or set CROSS_BINUTILS_PREFIX. .endif .endif .endif XBINUTILS= AS AR LD NM OBJCOPY RANLIB SIZE STRINGS .for BINUTIL in ${XBINUTILS} .if defined(CROSS_BINUTILS_PREFIX) && \ exists(${CROSS_BINUTILS_PREFIX}${${BINUTIL}}) X${BINUTIL}?= ${CROSS_BINUTILS_PREFIX}${${BINUTIL}} .else X${BINUTIL}?= ${${BINUTIL}} .endif .endfor # We must do lib/ and libexec/ before bin/ in case of a mid-install error to # keep the users system reasonably usable. For static->dynamic root upgrades, # we don't want to install a dynamic binary without rtld and the needed # libraries. More commonly, for dynamic root, we don't want to install a # binary that requires a newer library version that hasn't been installed yet. # This ordering is not a guarantee though. The only guarantee of a working # system here would require fine-grained ordering of all components based # on their dependencies. .if !empty(SUBDIR_OVERRIDE) SUBDIR= ${SUBDIR_OVERRIDE} .else SUBDIR= lib libexec .if !defined(NO_ROOT) && (make(installworld) || make(install)) # Ensure libraries are installed before progressing. SUBDIR+=.WAIT .endif SUBDIR+=bin .if ${MK_CDDL} != "no" SUBDIR+=cddl .endif SUBDIR+=gnu include .if ${MK_KERBEROS} != "no" SUBDIR+=kerberos5 .endif .if ${MK_RESCUE} != "no" SUBDIR+=rescue .endif SUBDIR+=sbin .if ${MK_CRYPT} != "no" SUBDIR+=secure .endif .if !defined(NO_SHARE) SUBDIR+=share .endif SUBDIR+=sys usr.bin usr.sbin .if ${MK_TESTS} != "no" SUBDIR+= tests .endif .if ${MK_OFED} != "no" SUBDIR+=contrib/ofed .endif # Local directories are last, since it is nice to at least get the base # system rebuilt before you do them. .for _DIR in ${LOCAL_DIRS} .if exists(${.CURDIR}/${_DIR}/Makefile) SUBDIR+= ${_DIR} .endif .endfor # Add LOCAL_LIB_DIRS, but only if they will not be picked up as a SUBDIR # of a LOCAL_DIRS directory. This allows LOCAL_DIRS=foo and # LOCAL_LIB_DIRS=foo/lib to behave as expected. .for _DIR in ${LOCAL_DIRS:M*/} ${LOCAL_DIRS:N*/:S|$|/|} _REDUNDANT_LIB_DIRS+= ${LOCAL_LIB_DIRS:M${_DIR}*} .endfor .for _DIR in ${LOCAL_LIB_DIRS} .if empty(_REDUNDANT_LIB_DIRS:M${_DIR}) && exists(${.CURDIR}/${_DIR}/Makefile) SUBDIR+= ${_DIR} .endif .endfor # We must do etc/ last as it hooks into building the man whatis file # by calling 'makedb' in share/man. This is only relevant for # install/distribute so they build the whatis file after every manpage is # installed. .if make(installworld) || make(install) SUBDIR+=.WAIT .endif SUBDIR+=etc .endif # !empty(SUBDIR_OVERRIDE) .if defined(NOCLEAN) .warning NOCLEAN option is deprecated. Use NO_CLEAN instead. NO_CLEAN= ${NOCLEAN} .endif .if defined(NO_CLEANDIR) CLEANDIR= clean cleandepend .else CLEANDIR= cleandir .endif .if defined(WORLDFAST) NO_CLEAN= t NO_OBJ= t .endif .if ${MK_META_MODE} == "yes" # If filemon is used then we can rely on the build being incremental-safe. # The .meta files will also track the build command and rebuild should # it change. .if empty(.MAKE.MODE:Mnofilemon) NO_CLEAN= t .endif .endif .if defined(NO_OBJ) || ${MK_AUTO_OBJ} == "yes" NO_OBJ= t NO_KERNELOBJ= t .endif .if !defined(NO_OBJ) _obj= obj .endif LOCAL_TOOL_DIRS?= PACKAGEDIR?= ${DESTDIR}/${DISTDIR} .if empty(SHELL:M*csh*) BUILDENV_SHELL?=${SHELL} .else BUILDENV_SHELL?=/bin/sh .endif .if !defined(SVN) || empty(SVN) . for _P in /usr/bin /usr/local/bin . for _S in svn svnlite . if exists(${_P}/${_S}) SVN= ${_P}/${_S} . endif . endfor . endfor .endif SVNFLAGS?= -r HEAD .if !defined(OSRELDATE) .if exists(/usr/include/osreldate.h) OSRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ /usr/include/osreldate.h .else OSRELDATE= 0 .endif .export OSRELDATE .endif # Set VERSION for CTFMERGE to use via the default CTFFLAGS=-L VERSION. .if !defined(_REVISION) _REVISION!= MK_AUTO_OBJ=no ${MAKE} -C ${SRCDIR}/release -V REVISION .export _REVISION .endif .if !defined(_BRANCH) _BRANCH!= MK_AUTO_OBJ=no ${MAKE} -C ${SRCDIR}/release -V BRANCH .export _BRANCH .endif .if !defined(SRCRELDATE) SRCRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ ${SRCDIR}/sys/sys/param.h .export SRCRELDATE .endif .if !defined(VERSION) VERSION= FreeBSD ${_REVISION}-${_BRANCH:C/-p[0-9]+$//} ${TARGET_ARCH} ${SRCRELDATE} .export VERSION .endif .if !defined(PKG_VERSION) .if ${_BRANCH:MSTABLE*} || ${_BRANCH:MCURRENT*} || ${_BRANCH:MALPHA*} TIMENOW= %Y%m%d%H%M%S EXTRA_REVISION= .s${TIMENOW:gmtime} .endif .if ${_BRANCH:M*-p*} EXTRA_REVISION= _${_BRANCH:C/.*-p([0-9]+$)/\1/} .endif PKG_VERSION= ${_REVISION}${EXTRA_REVISION} .endif KNOWN_ARCHES?= aarch64/arm64 \ amd64 \ arm \ armeb/arm \ armv6/arm \ i386 \ mips \ mipsel/mips \ mips64el/mips \ mipsn32el/mips \ mips64/mips \ mipsn32/mips \ mipshf/mips \ mipselhf/mips \ mips64elhf/mips \ mips64hf/mips \ powerpc \ powerpc64/powerpc \ powerpcspe/powerpc \ riscv64/riscv \ riscv64sf/riscv \ sparc64 .if ${TARGET} == ${TARGET_ARCH} _t= ${TARGET} .else _t= ${TARGET_ARCH}/${TARGET} .endif .for _t in ${_t} .if empty(KNOWN_ARCHES:M${_t}) .error Unknown target ${TARGET_ARCH}:${TARGET}. .endif .endfor .if ${TARGET} == ${MACHINE} TARGET_CPUTYPE?=${CPUTYPE} .else TARGET_CPUTYPE?= .endif .if !empty(TARGET_CPUTYPE) _TARGET_CPUTYPE=${TARGET_CPUTYPE} .else _TARGET_CPUTYPE=dummy .endif _CPUTYPE!= MK_AUTO_OBJ=no MAKEFLAGS= CPUTYPE=${_TARGET_CPUTYPE} ${MAKE} \ -f /dev/null -m ${.CURDIR}/share/mk -V CPUTYPE .if ${_CPUTYPE} != ${_TARGET_CPUTYPE} .error CPUTYPE global should be set with ?=. .endif .if make(buildworld) BUILD_ARCH!= uname -p .if ${MACHINE_ARCH} != ${BUILD_ARCH} .error To cross-build, set TARGET_ARCH. .endif .endif WORLDTMP= ${OBJTREE}${.CURDIR}/tmp BPATH= ${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/bin XPATH= ${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin STRICTTMPPATH= ${BPATH}:${XPATH} TMPPATH= ${STRICTTMPPATH}:${PATH} # # Avoid running mktemp(1) unless actually needed. # It may not be functional, e.g., due to new ABI # when in the middle of installing over this system. # .if make(distributeworld) || make(installworld) || make(stageworld) INSTALLTMP!= /usr/bin/mktemp -d -u -t install .endif .if make(stagekernel) || make(distributekernel) TAGS+= kernel PACKAGE= kernel .endif # # Building a world goes through the following stages # # 1. legacy stage [BMAKE] # This stage is responsible for creating compatibility # shims that are needed by the bootstrap-tools, # build-tools and cross-tools stages. These are generally # APIs that tools from one of those three stages need to # build that aren't present on the host. # 1. bootstrap-tools stage [BMAKE] # This stage is responsible for creating programs that # are needed for backward compatibility reasons. They # are not built as cross-tools. # 2. build-tools stage [TMAKE] # This stage is responsible for creating the object # tree and building any tools that are needed during # the build process. Some programs are listed during # this phase because they build binaries to generate # files needed to build these programs. This stage also # builds the 'build-tools' target rather than 'all'. # 3. cross-tools stage [XMAKE] # This stage is responsible for creating any tools that # are needed for building the system. A cross-compiler is one # of them. This differs from build tools in two ways: # 1. the 'all' target is built rather than 'build-tools' # 2. these tools are installed into TMPPATH for stage 4. # 4. world stage [WMAKE] # This stage actually builds the world. # 5. install stage (optional) [IMAKE] # This stage installs a previously built world. # BOOTSTRAPPING?= 0 # Keep these in sync -- see below for special case exception MINIMUM_SUPPORTED_OSREL?= 900044 MINIMUM_SUPPORTED_REL?= 9.1 # Common environment for world related stages CROSSENV+= MAKEOBJDIRPREFIX=${OBJTREE} \ MACHINE_ARCH=${TARGET_ARCH} \ MACHINE=${TARGET} \ CPUTYPE=${TARGET_CPUTYPE} .if ${MK_META_MODE} != "no" # Don't rebuild build-tools targets during normal build. CROSSENV+= BUILD_TOOLS_META=.NOMETA .endif .if defined(TARGET_CFLAGS) CROSSENV+= ${TARGET_CFLAGS} .endif # bootstrap-tools stage BMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \ TOOLS_PREFIX=${WORLDTMP} \ PATH=${BPATH}:${PATH} \ WORLDTMP=${WORLDTMP} \ MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}" # need to keep this in sync with targets/pseudo/bootstrap-tools/Makefile BSARGS= DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ BWPHASE=${.TARGET:C,^_,,} \ SSP_CFLAGS= \ MK_COVERAGE=no MK_HTML=no NO_LINT=yes MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_TESTS=no \ MK_INCLUDES=yes BMAKE= MAKEOBJDIRPREFIX=${WORLDTMP} \ ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ ${BSARGS} # build-tools stage TMAKE= MAKEOBJDIRPREFIX=${OBJTREE} \ ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ BWPHASE=${.TARGET:C,^_,,} \ SSP_CFLAGS= \ -DNO_LINT \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_TESTS=no # cross-tools stage XMAKE= TOOLS_PREFIX=${WORLDTMP} ${BMAKE} \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ MK_GDB=no MK_TESTS=no # kernel-tools stage KTMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${BPATH}:${PATH} \ WORLDTMP=${WORLDTMP} KTMAKE= TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \ ${KTMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ SSP_CFLAGS= \ MK_COVERAGE=no MK_HTML=no -DNO_LINT MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no # world stage WMAKEENV= ${CROSSENV} \ INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${TMPPATH} \ SYSROOT=${WORLDTMP} # make hierarchy HMAKE= PATH=${TMPPATH} ${MAKE} LOCAL_MTREE=${LOCAL_MTREE:Q} .if defined(NO_ROOT) HMAKE+= PATH=${TMPPATH} METALOG=${METALOG} -DNO_ROOT .endif CROSSENV+= CC="${XCC} ${XCFLAGS}" CXX="${XCXX} ${XCXXFLAGS} ${XCFLAGS}" \ CPP="${XCPP} ${XCFLAGS}" \ AS="${XAS}" AR="${XAR}" LD="${XLD}" LLVM_LINK="${XLLVM_LINK}" \ NM=${XNM} OBJCOPY="${XOBJCOPY}" \ RANLIB=${XRANLIB} STRINGS=${XSTRINGS} \ SIZE="${XSIZE}" .if defined(CROSS_BINUTILS_PREFIX) && exists(${CROSS_BINUTILS_PREFIX}) # In the case of xdev-build tools, CROSS_BINUTILS_PREFIX won't be a # directory, but the compiler will look in the right place for its # tools so we don't need to tell it where to look. BFLAGS+= -B${CROSS_BINUTILS_PREFIX} .endif # The internal bootstrap compiler has a default sysroot set by TOOLS_PREFIX # and target set by TARGET/TARGET_ARCH. However, there are several needs to # always pass an explicit --sysroot and -target. # - External compiler needs sysroot and target flags. # - External ld needs sysroot. # - To be clear about the use of a sysroot when using the internal compiler. # - Easier debugging. # - Allowing WITH_SYSTEM_COMPILER+WITH_META_MODE to work together due to # the flip-flopping build command when sometimes using external and # sometimes using internal. # - Allow using lld which has no support for default paths. .if !defined(CROSS_BINUTILS_PREFIX) || !exists(${CROSS_BINUTILS_PREFIX}) BFLAGS+= -B${WORLDTMP}/usr/bin .endif .if ${TARGET} == "arm" .if ${TARGET_ARCH:Marmv6*} != "" && ${TARGET_CPUTYPE:M*soft*} == "" TARGET_ABI= gnueabihf .else TARGET_ABI= gnueabi .endif .endif .if ${WANT_COMPILER_TYPE} == gcc || \ (defined(X_COMPILER_TYPE) && ${X_COMPILER_TYPE} == gcc) # GCC requires -isystem and -L when using a cross-compiler. --sysroot # won't set header path and -L is used to ensure the base library path # is added before the port PREFIX library path. XCFLAGS+= -isystem ${WORLDTMP}/usr/include -L${WORLDTMP}/usr/lib # GCC requires -B to find /usr/lib/crti.o when using a cross-compiler # combined with --sysroot. XCFLAGS+= -B${WORLDTMP}/usr/lib # Force using libc++ for external GCC. # XXX: This should be checking MK_GNUCXX == no .if ${X_COMPILER_VERSION} >= 40800 XCXXFLAGS+= -isystem ${WORLDTMP}/usr/include/c++/v1 -std=c++11 \ -nostdinc++ .endif .elif ${WANT_COMPILER_TYPE} == clang || \ (defined(X_COMPILER_TYPE) && ${X_COMPILER_TYPE} == clang) TARGET_ABI?= unknown TARGET_TRIPLE?= ${TARGET_ARCH:C/amd64/x86_64/}-${TARGET_ABI}-freebsd12.0 XCFLAGS+= -target ${TARGET_TRIPLE} .endif XCFLAGS+= --sysroot=${WORLDTMP} .if !empty(BFLAGS) XCFLAGS+= ${BFLAGS} .endif .if ${MK_LIB32} != "no" && (${TARGET_ARCH} == "amd64" || \ ${TARGET_ARCH} == "powerpc64" || ${TARGET_ARCH:Mmips64*} != "") LIBCOMPAT= 32 .include "Makefile.libcompat" .elif ${MK_LIBSOFT} != "no" && ${TARGET_ARCH} == "armv6" LIBCOMPAT= SOFT .include "Makefile.libcompat" .endif # META_MODE normally ignores host file changes since every build updates # timestamps (see NO_META_IGNORE_HOST in sys.mk). There are known times # when the ABI breaks though that we want to force rebuilding WORLDTMP # to get updated host tools. .if ${MK_META_MODE} == "yes" && defined(NO_CLEAN) && \ !defined(NO_META_IGNORE_HOST) && !defined(NO_META_IGNORE_HOST_HEADERS) && \ !make(showconfig) # r318736 - ino64 major ABI breakage META_MODE_BAD_ABI_VERS+= 1200031 .if !defined(OBJDIR_HOST_OSRELDATE) .if exists(${OBJTREE}${.CURDIR}/host-osreldate.h) OBJDIR_HOST_OSRELDATE!= \ awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ ${OBJTREE}${.CURDIR}/host-osreldate.h .else OBJDIR_HOST_OSRELDATE= 0 .endif .export OBJDIR_HOST_OSRELDATE .endif # Note that this logic is the opposite of normal BOOTSTRAP handling. We want # to compare the WORLDTMP's OSRELDATE to the host's OSRELDATE. If the WORLDTMP # is older than the ABI-breakage OSRELDATE of the HOST then we rebuild. .for _ver in ${META_MODE_BAD_ABI_VERS} .if ${OSRELDATE} >= ${_ver} && ${OBJDIR_HOST_OSRELDATE} < ${_ver} _meta_mode_need_rebuild= ${_ver} .endif .endfor .if defined(_meta_mode_need_rebuild) .info META_MODE: Rebuilding host tools due to ABI breakage in __FreeBSD_version ${_meta_mode_need_rebuild}. NO_META_IGNORE_HOST_HEADERS= 1 .export NO_META_IGNORE_HOST_HEADERS .endif .endif # This is only used for META_MODE+filemon to track what the oldest # __FreeBSD_version is in WORLDTMP. This purposely does NOT have # a make dependency on /usr/include/osreldate.h as the file should # only be copied when it is missing or meta mode determines it has changed. # Since host files are normally ignored without NO_META_IGNORE_HOST # the file will never be updated unless that flag is specified. This # allows tracking the oldest osreldate to force rebuilds via # META_MODE_BADABI_REVS above. host-osreldate.h: # DO NOT ADD /usr/include/osreldate.h here @cp -f /usr/include/osreldate.h ${.TARGET} WMAKE= ${WMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ BWPHASE=${.TARGET:C,^_,,} \ DESTDIR=${WORLDTMP} IMAKEENV= ${CROSSENV} IMAKE= ${IMAKEENV} ${MAKE} -f Makefile.inc1 \ ${IMAKE_INSTALL} ${IMAKE_MTREE} .if empty(.MAKEFLAGS:M-n) IMAKEENV+= PATH=${STRICTTMPPATH}:${INSTALLTMP} \ LD_LIBRARY_PATH=${INSTALLTMP} \ PATH_LOCALE=${INSTALLTMP}/locale IMAKE+= __MAKE_SHELL=${INSTALLTMP}/sh .else IMAKEENV+= PATH=${TMPPATH}:${INSTALLTMP} .endif .if defined(DB_FROM_SRC) INSTALLFLAGS+= -N ${.CURDIR}/etc MTREEFLAGS+= -N ${.CURDIR}/etc .endif _INSTALL_DDIR= ${DESTDIR}/${DISTDIR} INSTALL_DDIR= ${_INSTALL_DDIR:S://:/:g:C:/$::} .if defined(NO_ROOT) METALOG?= ${DESTDIR}/${DISTDIR}/METALOG METALOG:= ${METALOG:C,//+,/,g} IMAKE+= -DNO_ROOT METALOG=${METALOG} INSTALLFLAGS+= -U -M ${METALOG} -D ${INSTALL_DDIR} MTREEFLAGS+= -W .endif .if defined(BUILD_PKGS) INSTALLFLAGS+= -h sha256 .endif .if defined(DB_FROM_SRC) || defined(NO_ROOT) IMAKE_INSTALL= INSTALL="install ${INSTALLFLAGS}" IMAKE_MTREE= MTREE_CMD="mtree ${MTREEFLAGS}" .endif # kernel stage KMAKEENV= ${WMAKEENV:NSYSROOT=*} KMAKE= ${KMAKEENV} ${MAKE} ${.MAKEFLAGS} ${KERNEL_FLAGS} KERNEL=${INSTKERNNAME} # # buildworld # # Attempt to rebuild the entire system, with reasonable chance of # success, regardless of how old your existing system is. # _worldtmp: .PHONY .if ${.CURDIR:C/[^,]//g} != "" # The m4 build of sendmail files doesn't like it if ',' is used # anywhere in the path of it's files. @echo @echo "*** Error: path to source tree contains a comma ','" @echo false .endif @echo @echo "--------------------------------------------------------------" @echo ">>> Rebuilding the temporary build tree" @echo "--------------------------------------------------------------" .if !defined(NO_CLEAN) rm -rf ${WORLDTMP} .if defined(LIBCOMPAT) rm -rf ${LIBCOMPATTMP} .endif .else + @echo ">>> Deleting stale files in build tree..." ${_+_}cd ${.CURDIR}; ${WMAKE} -DBATCH_DELETE_OLD_FILES \ - delete-old delete-old-libs + delete-old delete-old-libs >/dev/null .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${WMAKE} -DBATCH_DELETE_OLD_FILES \ DESTDIR=${LIBCOMPATTMP} \ - delete-old delete-old-libs + delete-old delete-old-libs >/dev/null .endif rm -rf ${WORLDTMP}/legacy/usr/include .if ${USING_SYSTEM_COMPILER} == "yes" .for cc in cc c++ if [ -x ${WORLDTMP}/usr/bin/${cc} ]; then \ inum=$$(stat -f %i ${WORLDTMP}/usr/bin/${cc}); \ find ${WORLDTMP}/usr/bin -inum $${inum} -delete; \ fi .endfor .endif # ${USING_SYSTEM_COMPILER} == "yes" # Our current approach to dependency tracking cannot cope with certain source # tree changes, particularly with respect to removing source files and # replacing generated files. Handle these cases here in an ad-hoc fashion. # # Syscall stubs rewritten in C # Date SVN Rev Syscalls # 20160829 r305012 ptrace # 20170624 r320278 fstat fstatat fstatfs getdirentries getfsstat statfs .for f in fstat fstatat fstatfs getdirentries getfsstat ptrace statfs .if exists(${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.o) @if egrep -qw '${f}\.[sS]' \ ${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.o; then \ echo Removing stale dependencies for ${f} syscall wrappers; \ rm -f ${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.* \ ${OBJTREE}${.CURDIR}/world32/${.CURDIR}/lib/libc/.depend.${f}.*; \ fi .endif .endfor # 20170607 remove stale dependencies for utimens* wrappers removed in r319663 .for f in futimens utimensat .if exists(${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.o) @if egrep -q '/${f}.c' \ ${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.o; then \ echo Removing stale dependencies for ${f} syscall wrappers; \ rm -f ${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.* \ ${OBJTREE}${.CURDIR}/world32/${.CURDIR}/lib/libc/.depend.${f}.*; \ fi .endif .endfor # 20170523 remove stale generated asm files for functions which are no longer # syscalls after r302092 (pipe) and r318736 (others) .for f in getdents lstat mknod pipe stat .if exists(${OBJTREE}${.CURDIR}/lib/libc/${f}.s) || \ exists(${OBJTREE}${.CURDIR}/lib/libc/${f}.S) @echo Removing stale generated ${f} syscall files @rm -f ${OBJTREE}${.CURDIR}/lib/libc/${f}.* \ ${OBJTREE}${.CURDIR}/lib/libc/.depend.${f}.* \ ${OBJTREE}${.CURDIR}/world32/${.CURDIR}/lib/libc/${f}.* \ ${OBJTREE}${.CURDIR}/world32/${.CURDIR}/lib/libc/.depend.${f}.* .endif .endfor .endif # !defined(NO_CLEAN) .for _dir in \ lib lib/casper usr legacy/bin legacy/usr mkdir -p ${WORLDTMP}/${_dir} .endfor mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${WORLDTMP}/legacy/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${WORLDTMP}/legacy/usr/include >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${WORLDTMP}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${WORLDTMP}/usr/include >/dev/null ln -sf ${.CURDIR}/sys ${WORLDTMP} .if ${MK_DEBUG_FILES} != "no" # We could instead disable debug files for these build stages mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${WORLDTMP}/legacy/usr/lib >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${WORLDTMP}/usr/lib >/dev/null .endif .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/usr >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/legacy/usr/lib/debug/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/usr/lib/debug/usr >/dev/null .endif .endif .if ${MK_TESTS} != "no" mkdir -p ${WORLDTMP}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${WORLDTMP}${TESTSBASE} >/dev/null .if ${MK_DEBUG_FILES} != "no" mkdir -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} >/dev/null .endif .endif .for _mtree in ${LOCAL_MTREE} mtree -deU -f ${.CURDIR}/${_mtree} -p ${WORLDTMP} > /dev/null .endfor _legacy: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1.1: legacy release compatibility shims" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${BMAKE} legacy _bootstrap-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1.2: bootstrap tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${BMAKE} bootstrap-tools _cleanobj: .if !defined(NO_CLEAN) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.1: cleaning up the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${WMAKE} ${CLEANDIR} .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${LIBCOMPATWMAKE} -f Makefile.inc1 ${CLEANDIR} .endif .endif _obj: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.2: rebuilding the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${WMAKE} obj _build-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.3: build tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${TMAKE} build-tools _cross-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 3: cross tools" @echo "--------------------------------------------------------------" @rm -f ${OBJTREE}${.CURDIR}/compiler-metadata.mk ${_+_}cd ${.CURDIR}; ${XMAKE} cross-tools ${_+_}cd ${.CURDIR}; ${XMAKE} kernel-tools _build-metadata: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 3.1: recording build metadata" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${WMAKE} compiler-metadata.mk ${_+_}cd ${.CURDIR}; ${WMAKE} host-osreldate.h _includes: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.1: building includes" @echo "--------------------------------------------------------------" # Special handling for SUBDIR_OVERRIDE in buildworld as they most likely need # headers from default SUBDIR. Do SUBDIR_OVERRIDE includes last. ${_+_}cd ${.CURDIR}; ${WMAKE} SUBDIR_OVERRIDE= SHARED=symlinks \ MK_INCLUDES=yes includes .if !empty(SUBDIR_OVERRIDE) && make(buildworld) ${_+_}cd ${.CURDIR}; ${WMAKE} MK_INCLUDES=yes SHARED=symlinks includes .endif _libraries: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.2: building libraries" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; \ ${WMAKE} -DNO_FSCHG MK_COVERAGE=no MK_HTML=no -DNO_LINT MK_MAN=no \ MK_PROFILE=no MK_TESTS=no MK_TESTS_SUPPORT=${MK_TESTS} libraries everything: .PHONY @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.3: building everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; _PARALLEL_SUBDIR_OK=1 ${WMAKE} all WMAKE_TGTS= .if !defined(WORLDFAST) WMAKE_TGTS+= _worldtmp _legacy .if empty(SUBDIR_OVERRIDE) WMAKE_TGTS+= _bootstrap-tools .endif WMAKE_TGTS+= _cleanobj .if !defined(NO_OBJ) WMAKE_TGTS+= _obj .endif WMAKE_TGTS+= _build-tools _cross-tools WMAKE_TGTS+= _build-metadata WMAKE_TGTS+= _includes .endif .if !defined(NO_LIBS) WMAKE_TGTS+= _libraries .endif WMAKE_TGTS+= everything .if defined(LIBCOMPAT) && empty(SUBDIR_OVERRIDE) WMAKE_TGTS+= build${libcompat} .endif buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .PHONY .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue buildworld_prologue: .PHONY @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" buildworld_epilogue: .PHONY @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a # modification of the current environment's PATH. In addition, we need # to quote multiword values. # buildenvvars: .PHONY @echo ${WMAKEENV:Q} ${.MAKE.EXPORTED:@v@$v=\"${$v}\"@} .if ${.TARGETS:Mbuildenv} .if ${.MAKEFLAGS:M-j} .error The buildenv target is incompatible with -j .endif .endif BUILDENV_DIR?= ${.CURDIR} buildenv: .PHONY @echo Entering world for ${TARGET_ARCH}:${TARGET} .if ${BUILDENV_SHELL:M*zsh*} @echo For ZSH you must run: export CPUTYPE=${TARGET_CPUTYPE} .endif @cd ${BUILDENV_DIR} && env ${WMAKEENV} BUILDENV=1 ${BUILDENV_SHELL} \ || true TOOLCHAIN_TGTS= ${WMAKE_TGTS:Neverything:Nbuild${libcompat}} toolchain: ${TOOLCHAIN_TGTS} .PHONY kernel-toolchain: ${TOOLCHAIN_TGTS:N_includes:N_libraries} .PHONY # # installcheck # # Checks to be sure system is ready for installworld/installkernel. # installcheck: _installcheck_world _installcheck_kernel .PHONY _installcheck_world: .PHONY _installcheck_kernel: .PHONY # # Require DESTDIR to be set if installing for a different architecture or # using the user/group database in the source tree. # .if ${TARGET_ARCH} != ${MACHINE_ARCH} || ${TARGET} != ${MACHINE} || \ defined(DB_FROM_SRC) .if !make(distributeworld) _installcheck_world: __installcheck_DESTDIR _installcheck_kernel: __installcheck_DESTDIR __installcheck_DESTDIR: .PHONY .if !defined(DESTDIR) || empty(DESTDIR) @echo "ERROR: Please set DESTDIR!"; \ false .endif .endif .endif .if !defined(DB_FROM_SRC) # # Check for missing UIDs/GIDs. # CHECK_UIDS= auditdistd CHECK_GIDS= audit .if ${MK_SENDMAIL} != "no" CHECK_UIDS+= smmsp CHECK_GIDS+= smmsp .endif .if ${MK_PF} != "no" CHECK_UIDS+= proxy CHECK_GIDS+= proxy authpf .endif .if ${MK_UNBOUND} != "no" CHECK_UIDS+= unbound CHECK_GIDS+= unbound .endif _installcheck_world: __installcheck_UGID __installcheck_UGID: .PHONY .for uid in ${CHECK_UIDS} @if ! `id -u ${uid} >/dev/null 2>&1`; then \ echo "ERROR: Required ${uid} user is missing, see /usr/src/UPDATING."; \ false; \ fi .endfor .for gid in ${CHECK_GIDS} @if ! `find / -prune -group ${gid} >/dev/null 2>&1`; then \ echo "ERROR: Required ${gid} group is missing, see /usr/src/UPDATING."; \ false; \ fi .endfor .endif # # If installing over the running system (DESTDIR is / or unset) and the install # includes rescue, try running rescue from the objdir as a sanity check. If # rescue is not functional (e.g., because it depends on a system call not # supported by the currently running kernel), abort the installation. # .if !make(distributeworld) && ${MK_RESCUE} != "no" && \ (empty(DESTDIR) || ${DESTDIR} == "/") && empty(BYPASS_INSTALLCHECK_SH) _installcheck_world: __installcheck_sh_check __installcheck_sh_check: .PHONY @if [ "`${OBJTREE}${.CURDIR}/rescue/rescue/rescue sh -c 'echo OK'`" != \ OK ]; then \ echo "rescue/sh check failed, installation aborted" >&2; \ false; \ fi .endif # # Required install tools to be saved in a scratch dir for safety. # .if ${MK_ZONEINFO} != "no" _zoneinfo= zic tzsetup .endif ITOOLS= [ awk cap_mkdb cat chflags chmod chown cmp cp \ date echo egrep find grep id install ${_install-info} \ ln make mkdir mtree mv pwd_mkdb \ rm sed services_mkdb sh strip sysctl test true uname wc ${_zoneinfo} \ ${LOCAL_ITOOLS} # Needed for share/man .if ${MK_MAN_UTILS} != "no" ITOOLS+=makewhatis .endif # # distributeworld # # Distributes everything compiled by a `buildworld'. # # installworld # # Installs everything compiled by a 'buildworld'. # # Non-base distributions produced by the base system EXTRA_DISTRIBUTIONS= doc .if defined(LIBCOMPAT) EXTRA_DISTRIBUTIONS+= lib${libcompat} .endif .if ${MK_TESTS} != "no" EXTRA_DISTRIBUTIONS+= tests .endif DEBUG_DISTRIBUTIONS= .if ${MK_DEBUG_FILES} != "no" DEBUG_DISTRIBUTIONS+= base ${EXTRA_DISTRIBUTIONS:S,doc,,:S,tests,,} .endif MTREE_MAGIC?= mtree 2.0 distributeworld installworld stageworld: _installcheck_world .PHONY mkdir -p ${INSTALLTMP} progs=$$(for prog in ${ITOOLS}; do \ if progpath=`which $$prog`; then \ echo $$progpath; \ else \ echo "Required tool $$prog not found in PATH." >&2; \ exit 1; \ fi; \ done); \ libs=$$(ldd -f "%o %p\n" -f "%o %p\n" $$progs 2>/dev/null | sort -u | \ while read line; do \ set -- $$line; \ if [ "$$2 $$3" != "not found" ]; then \ echo $$2; \ else \ echo "Required library $$1 not found." >&2; \ exit 1; \ fi; \ done); \ cp $$libs $$progs ${INSTALLTMP} cp -R $${PATH_LOCALE:-"/usr/share/locale"} ${INSTALLTMP}/locale .if defined(NO_ROOT) -mkdir -p ${METALOG:H} echo "#${MTREE_MAGIC}" > ${METALOG} .endif .if make(distributeworld) .for dist in ${EXTRA_DISTRIBUTIONS} -mkdir ${DESTDIR}/${DISTDIR}/${dist} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \ -p ${DESTDIR}/${DISTDIR}/${dist} >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/include >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib >/dev/null .endif .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/usr >/dev/null .endif .endif .if ${MK_TESTS} != "no" && ${dist} == "tests" -mkdir -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE} >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/${TESTSBASE} >/dev/null .endif .endif .if defined(NO_ROOT) ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.root.dist | \ sed -e 's#^\./#./${dist}/#' >> ${METALOG} ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.usr.dist | \ sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG} ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.include.dist | \ sed -e 's#^\./#./${dist}/usr/include/#' >> ${METALOG} .if defined(LIBCOMPAT) ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist | \ sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG} .endif .endif .endfor -mkdir ${DESTDIR}/${DISTDIR}/base ${_+_}cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \ METALOG=${METALOG} ${IMAKE_INSTALL} ${IMAKE_MTREE} \ DISTBASE=/base DESTDIR=${DESTDIR}/${DISTDIR}/base \ LOCAL_MTREE=${LOCAL_MTREE:Q} distrib-dirs .endif ${_+_}cd ${.CURDIR}; ${IMAKE} re${.TARGET:S/world$//}; \ ${IMAKEENV} rm -rf ${INSTALLTMP} .if make(distributeworld) .for dist in ${EXTRA_DISTRIBUTIONS} find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -type d -empty -delete .endfor .if defined(NO_ROOT) .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @# a line containing only the filename will sort immediately before @# the relevant mtree line. cd ${DESTDIR}/${DISTDIR}; \ find ./${dist} | sort -u ${METALOG} - | \ awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \ ${DESTDIR}/${DISTDIR}/${dist}.meta .endfor .for dist in ${DEBUG_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @# a line containing only the filename will sort immediately before @# the relevant mtree line. cd ${DESTDIR}/${DISTDIR}; \ find ./${dist}/usr/lib/debug | sort -u ${METALOG} - | \ awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \ ${DESTDIR}/${DISTDIR}/${dist}.debug.meta .endfor .endif .endif packageworld: .PHONY .for dist in base ${EXTRA_DISTRIBUTIONS} .if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - --exclude usr/lib/debug \ @${DESTDIR}/${DISTDIR}/${dist}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}.txz .else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - --exclude usr/lib/debug . | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}.txz .endif .endfor .for dist in ${DEBUG_DISTRIBUTIONS} . if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - @${DESTDIR}/${DISTDIR}/${dist}.debug.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}-dbg.txz . else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvLf - usr/lib/debug | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}-dbg.txz . endif .endfor # # reinstall # # If you have a build server, you can NFS mount the source and obj directories # and do a 'make reinstall' on the *client* to install new binaries from the # most recent server build. # restage reinstall: .MAKE .PHONY @echo "--------------------------------------------------------------" @echo ">>> Making hierarchy" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 \ LOCAL_MTREE=${LOCAL_MTREE:Q} hierarchy .if make(restage) @echo "--------------------------------------------------------------" @echo ">>> Making distribution" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 \ LOCAL_MTREE=${LOCAL_MTREE:Q} distribution .endif @echo @echo "--------------------------------------------------------------" @echo ">>> Installing everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install${libcompat} .endif redistribute: .MAKE .PHONY @echo "--------------------------------------------------------------" @echo ">>> Distributing everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute${libcompat} \ DISTRIBUTION=lib${libcompat} .endif distrib-dirs distribution: .MAKE .PHONY ${_+_}cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \ ${IMAKE_INSTALL} ${IMAKE_MTREE} METALOG=${METALOG} ${.TARGET} .if make(distribution) ${_+_}cd ${.CURDIR}; ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} -f Makefile.inc1 ${IMAKE_INSTALL} \ METALOG=${METALOG} MK_TESTS=no installconfig .endif # # buildkernel and installkernel # # Which kernels to build and/or install is specified by setting # KERNCONF. If not defined a GENERIC kernel is built/installed. # Only the existing (depending TARGET) config files are used # for building kernels and only the first of these is designated # as the one being installed. # # Note that we have to use TARGET instead of TARGET_ARCH when # we're in kernel-land. Since only TARGET_ARCH is (expected) to # be set to cross-build, we have to make sure TARGET is set # properly. .if defined(KERNFAST) NO_KERNELCLEAN= t NO_KERNELCONFIG= t NO_KERNELOBJ= t # Shortcut for KERNCONF=Blah -DKERNFAST is now KERNFAST=Blah .if !defined(KERNCONF) && ${KERNFAST} != "1" KERNCONF=${KERNFAST} .endif .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 .else KERNCONF?= GENERIC .endif INSTKERNNAME?= kernel KERNSRCDIR?= ${.CURDIR}/sys KRNLCONFDIR= ${KERNSRCDIR}/${TARGET}/conf KRNLOBJDIR= ${OBJTREE}${KERNSRCDIR} KERNCONFDIR?= ${KRNLCONFDIR} BUILDKERNELS= INSTALLKERNEL= .if defined(NO_INSTALLKERNEL) # All of the BUILDKERNELS loops start at index 1. BUILDKERNELS+= dummy .endif .for _kernel in ${KERNCONF} .if exists(${KERNCONFDIR}/${_kernel}) BUILDKERNELS+= ${_kernel} .if empty(INSTALLKERNEL) && !defined(NO_INSTALLKERNEL) INSTALLKERNEL= ${_kernel} .endif .endif .endfor ${WMAKE_TGTS:N_worldtmp:Nbuild${libcompat}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY # # buildkernel # # Builds all kernels defined by BUILDKERNELS. # buildkernel: .MAKE .PHONY .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false .endif @echo .for _kernel in ${BUILDKERNELS:Ndummy} @echo "--------------------------------------------------------------" @echo ">>> Kernel build for ${_kernel} started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" @echo "===> ${_kernel}" mkdir -p ${KRNLOBJDIR} .if !defined(NO_KERNELCONFIG) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1: configuring the kernel" @echo "--------------------------------------------------------------" cd ${KRNLCONFDIR}; \ PATH=${TMPPATH} \ config ${CONFIGARGS} -d ${KRNLOBJDIR}/${_kernel} \ -I '${KERNCONFDIR}' '${KERNCONFDIR}/${_kernel}' .endif .if !defined(NO_CLEAN) && !defined(NO_KERNELCLEAN) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.1: cleaning up the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} ${CLEANDIR} .endif .if !defined(NO_KERNELOBJ) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.2: rebuilding the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} obj .endif @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.3: build tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${KTMAKE} kernel-tools @echo @echo "--------------------------------------------------------------" @echo ">>> stage 3.1: building everything" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} all -DNO_MODULES_OBJ @echo "--------------------------------------------------------------" @echo ">>> Kernel build for ${_kernel} completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" .endfor NO_INSTALLEXTRAKERNELS?= yes # # installkernel, etc. # # Install the kernel defined by INSTALLKERNEL # installkernel installkernel.debug \ reinstallkernel reinstallkernel.debug: _installcheck_kernel .PHONY .if !defined(NO_INSTALLKERNEL) .if empty(INSTALLKERNEL) @echo "ERROR: No kernel \"${KERNCONF}\" to install."; \ false .endif @echo "--------------------------------------------------------------" @echo ">>> Installing kernel ${INSTALLKERNEL}" @echo "--------------------------------------------------------------" cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \ ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} ${IMAKE_INSTALL} KERNEL=${INSTKERNNAME} ${.TARGET:S/kernel//} .endif .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} @echo "--------------------------------------------------------------" @echo ">>> Installing kernel ${_kernel}" @echo "--------------------------------------------------------------" cd ${KRNLOBJDIR}/${_kernel}; \ ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} ${IMAKE_INSTALL} KERNEL=${INSTKERNNAME}.${_kernel} ${.TARGET:S/kernel//} .endfor .endif distributekernel distributekernel.debug: .PHONY .if !defined(NO_INSTALLKERNEL) .if empty(INSTALLKERNEL) @echo "ERROR: No kernel \"${KERNCONF}\" to install."; \ false .endif mkdir -p ${DESTDIR}/${DISTDIR} .if defined(NO_ROOT) @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.premeta .endif cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} KERNEL=${INSTKERNNAME} \ DESTDIR=${INSTALL_DDIR}/kernel \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) @sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta > \ ${DESTDIR}/${DISTDIR}/kernel.meta .endif .endif .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} .if defined(NO_ROOT) @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta .endif cd ${KRNLOBJDIR}/${_kernel}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.${_kernel}.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} \ KERNEL=${INSTKERNNAME}.${_kernel} \ DESTDIR=${INSTALL_DDIR}/kernel.${_kernel} \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) @sed -e "s|^./kernel.${_kernel}|.|" \ ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta > \ ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta .endif .endfor .endif packagekernel: .PHONY .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz .endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --include '*/*/*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --include '*/*/*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz .endfor .endif .else .if !defined(NO_INSTALLKERNEL) cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' . | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz .endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --include '*/*/*.debug' $$(eval find .) | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' . | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --include '*/*/*.debug' $$(eval find .) | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz .endfor .endif .endif stagekernel: .PHONY ${_+_}${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} distributekernel PORTSDIR?= /usr/ports WSTAGEDIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/${TARGET}.${TARGET_ARCH}/worldstage KSTAGEDIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/${TARGET}.${TARGET_ARCH}/kernelstage REPODIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/repo PKGSIGNKEY?= # empty .ORDER: stage-packages create-packages .ORDER: create-packages create-world-packages .ORDER: create-packages create-kernel-packages .ORDER: create-packages sign-packages _pkgbootstrap: .PHONY .if !exists(${LOCALBASE}/sbin/pkg) @env ASSUME_ALWAYS_YES=YES pkg bootstrap .endif packages: .PHONY ${_+_}${MAKE} -C ${.CURDIR} PKG_VERSION=${PKG_VERSION} real-packages package-pkg: .PHONY rm -rf /tmp/ports.${TARGET} || : env ${WMAKEENV:Q} SRCDIR=${.CURDIR} PORTSDIR=${PORTSDIR} REVISION=${_REVISION} \ PKG_CMD=${PKG_CMD} PKG_VERSION=${PKG_VERSION} REPODIR=${REPODIR} \ WSTAGEDIR=${WSTAGEDIR} \ sh ${.CURDIR}/release/scripts/make-pkg-package.sh real-packages: stage-packages create-packages sign-packages .PHONY stage-packages-world: .PHONY @mkdir -p ${WSTAGEDIR} ${_+_}@cd ${.CURDIR}; \ ${MAKE} DESTDIR=${WSTAGEDIR} -DNO_ROOT stageworld stage-packages-kernel: .PHONY @mkdir -p ${KSTAGEDIR} ${_+_}@cd ${.CURDIR}; \ ${MAKE} DESTDIR=${KSTAGEDIR} -DNO_ROOT stagekernel stage-packages: .PHONY stage-packages-world stage-packages-kernel _repodir: .PHONY @mkdir -p ${REPODIR} create-packages-world: _pkgbootstrap _repodir .PHONY ${_+_}@cd ${.CURDIR}; \ ${MAKE} -f Makefile.inc1 \ DESTDIR=${WSTAGEDIR} \ PKG_VERSION=${PKG_VERSION} create-world-packages create-packages-kernel: _pkgbootstrap _repodir .PHONY ${_+_}@cd ${.CURDIR}; \ ${MAKE} -f Makefile.inc1 \ DESTDIR=${KSTAGEDIR} \ PKG_VERSION=${PKG_VERSION} DISTDIR=kernel \ create-kernel-packages create-packages: .PHONY create-packages-world create-packages-kernel create-world-packages: _pkgbootstrap .PHONY @rm -f ${WSTAGEDIR}/*.plist 2>/dev/null || : @cd ${WSTAGEDIR} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ ${WSTAGEDIR}/METALOG @for plist in ${WSTAGEDIR}/*.plist; do \ plist=$${plist##*/} ; \ pkgname=$${plist%.plist} ; \ echo "_PKGS+= $${pkgname}" ; \ done > ${WSTAGEDIR}/packages.mk ${_+_}@cd ${.CURDIR}; \ ${MAKE} -f Makefile.inc1 create-world-packages-jobs \ .MAKE.JOB.PREFIX= .if make(create-world-packages-jobs) .include "${WSTAGEDIR}/packages.mk" .endif create-world-packages-jobs: .PHONY .for pkgname in ${_PKGS} create-world-packages-jobs: create-world-package-${pkgname} create-world-package-${pkgname}: .PHONY @sh ${SRCDIR}/release/packages/generate-ucl.sh -o ${pkgname} \ -s ${SRCDIR} -u ${WSTAGEDIR}/${pkgname}.ucl @awk -F\" ' \ /^name/ { printf("===> Creating %s-", $$2); next } \ /^version/ { print $$2; next } \ ' ${WSTAGEDIR}/${pkgname}.ucl ; \ ${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${WSTAGEDIR}/${pkgname}.ucl \ -p ${WSTAGEDIR}/${pkgname}.plist \ -r ${WSTAGEDIR} \ -o ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} .endfor create-kernel-packages: .PHONY _default_flavor= -default .if exists(${KSTAGEDIR}/kernel.meta) .for flavor in "" -debug create-kernel-packages: create-kernel-packages-flavor${flavor:C,^""$,${_default_flavor},} create-kernel-packages-flavor${flavor:C,^""$,${_default_flavor},}: _pkgbootstrap .PHONY @cd ${KSTAGEDIR}/${DISTDIR} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ -v kernel=yes -v _kernconf=${INSTALLKERNEL} \ ${KSTAGEDIR}/kernel.meta ; \ cap_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VCAP_MKDB_ENDIAN` ; \ pwd_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VPWD_MKDB_ENDIAN` ; \ sed -e "s/%VERSION%/${PKG_VERSION}/" \ -e "s/%PKGNAME%/kernel-${INSTALLKERNEL:tl}${flavor}/" \ -e "s/%COMMENT%/FreeBSD ${INSTALLKERNEL} kernel ${flavor}/" \ -e "s/%DESC%/FreeBSD ${INSTALLKERNEL} kernel ${flavor}/" \ -e "s/%CAP_MKDB_ENDIAN%/$${cap_arg}/g" \ -e "s/%PWD_MKDB_ENDIAN%/$${pwd_arg}/g" \ ${SRCDIR}/release/packages/kernel.ucl \ > ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \ awk -F\" ' \ /name/ { printf("===> Creating %s-", $$2); next } \ /version/ {print $$2; next } ' \ ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \ ${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl \ -p ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.plist \ -r ${KSTAGEDIR}/${DISTDIR} \ -o ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} .endfor .endif .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} .if exists(${KSTAGEDIR}/kernel.${_kernel}.meta) .for flavor in "" -debug create-kernel-packages: create-kernel-packages-extra-flavor${flavor:C,^""$,${_default_flavor},}-${_kernel} create-kernel-packages-extra-flavor${flavor:C,^""$,${_default_flavor},}-${_kernel}: _pkgbootstrap .PHONY @cd ${KSTAGEDIR}/kernel.${_kernel} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ -v kernel=yes -v _kernconf=${_kernel} \ ${KSTAGEDIR}/kernel.${_kernel}.meta ; \ cap_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VCAP_MKDB_ENDIAN` ; \ pwd_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VPWD_MKDB_ENDIAN` ; \ sed -e "s/%VERSION%/${PKG_VERSION}/" \ -e "s/%PKGNAME%/kernel-${_kernel:tl}${flavor}/" \ -e "s/%COMMENT%/FreeBSD ${_kernel} kernel ${flavor}/" \ -e "s/%DESC%/FreeBSD ${_kernel} kernel ${flavor}/" \ -e "s/%CAP_MKDB_ENDIAN%/$${cap_arg}/g" \ -e "s/%PWD_MKDB_ENDIAN%/$${pwd_arg}/g" \ ${SRCDIR}/release/packages/kernel.ucl \ > ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \ awk -F\" ' \ /name/ { printf("===> Creating %s-", $$2); next } \ /version/ {print $$2; next } ' \ ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \ ${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl \ -p ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.plist \ -r ${KSTAGEDIR}/kernel.${_kernel} \ -o ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} .endfor .endif .endfor .endif sign-packages: _pkgbootstrap .PHONY @[ -L "${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/latest" ] && \ unlink ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/latest ; \ ${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh repo \ -o ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} \ ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} \ ${PKGSIGNKEY} ; \ cd ${REPODIR}/$$(${PKG_CMD} -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI); \ ln -s ${PKG_VERSION} latest # # # checkworld # # Run test suite on installed world. # checkworld: .PHONY @if [ ! -x "${LOCALBASE}/bin/kyua" ]; then \ echo "You need kyua (devel/kyua) to run the test suite." | /usr/bin/fmt; \ exit 1; \ fi ${_+_}PATH="$$PATH:${LOCALBASE}/bin" kyua test -k ${TESTSBASE}/Kyuafile # # # doxygen # # Build the API documentation with doxygen # doxygen: .PHONY @if [ ! -x "${LOCALBASE}/bin/doxygen" ]; then \ echo "You need doxygen (devel/doxygen) to generate the API documentation of the kernel." | /usr/bin/fmt; \ exit 1; \ fi ${_+_}cd ${.CURDIR}/tools/kerneldoc/subsys; ${MAKE} obj all # # update # # Update the source tree(s), by running svn/svnup to update to the # latest copy. # update: .PHONY .if defined(SVN_UPDATE) @echo "--------------------------------------------------------------" @echo ">>> Updating ${.CURDIR} using Subversion" @echo "--------------------------------------------------------------" @(cd ${.CURDIR}; ${SVN} update ${SVNFLAGS}) .endif # # ------------------------------------------------------------------------ # # From here onwards are utility targets used by the 'make world' and # related targets. If your 'world' breaks, you may like to try to fix # the problem and manually run the following targets to attempt to # complete the build. Beware, this is *not* guaranteed to work, you # need to have a pretty good grip on the current state of the system # to attempt to manually finish it. If in doubt, 'make world' again. # # # legacy: Build compatibility shims for the next three targets. This is a # minimal set of tools and shims necessary to compensate for older systems # which don't have the APIs required by the targets built in bootstrap-tools, # build-tools or cross-tools. # # ELF Tool Chain libraries are needed for ELF tools and dtrace tools. # r296685 fix cross-endian objcopy .if ${BOOTSTRAPPING} < 1100102 _elftoolchain_libs= lib/libelf lib/libdwarf .endif legacy: .PHONY # Temporary special case for automatically detecting the clang compiler issue # Note: 9.x didn't have FreeBSD_version bumps often enough, so you may need to # set BOOTSTRAPPING to 0 if you're stable/9 tree post-dates r286035 but is before # the version bump in r296219 (from July 29, 2015 -> Feb 29, 2016). .if ${BOOTSTRAPPING} != 0 && \ ${WANT_COMPILER_TYPE} == "clang" && ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} < 30601 .if ${BOOTSTRAPPING} > 10000000 && ${BOOTSTRAPPING} < 1002501 @echo "ERROR: Source upgrades from stable/10 prior to r286033 are not supported."; false .elif ${BOOTSTRAPPING} > 9000000 && ${BOOTSTRAPPING} < 903509 @echo "ERROR: Source upgrades from stable/9 prior to r286035 are not supported."; false .endif .endif .if ${BOOTSTRAPPING} < ${MINIMUM_SUPPORTED_OSREL} && ${BOOTSTRAPPING} != 0 @echo "ERROR: Source upgrades from versions prior to ${MINIMUM_SUPPORTED_REL} are not supported."; \ false .endif .for _tool in tools/build ${_elftoolchain_libs} ${_+_}@${ECHODIR} "===> ${_tool} (obj,includes,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy includes; \ ${MAKE} DIRPRFX=${_tool}/ MK_INCLUDES=no all; \ ${MAKE} DIRPRFX=${_tool}/ MK_INCLUDES=no \ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install .endfor # # bootstrap-tools: Build tools needed for compatibility. These are binaries that # are built to build other binaries in the system. However, the focus of these # binaries is usually quite narrow. Bootstrap tools use the host's compiler and # libraries, augmented by -legacy. # _bt= _bootstrap-tools .if ${MK_GAMES} != "no" _strfile= usr.bin/fortune/strfile .endif .if ${MK_GCC} != "no" && ${MK_CXX} != "no" _gperf= gnu/usr.bin/gperf .endif .if ${MK_VT} != "no" _vtfontcvt= usr.bin/vtfontcvt .endif .if ${BOOTSTRAPPING} < 1000033 _libopenbsd= lib/libopenbsd _m4= usr.bin/m4 _lex= usr.bin/lex ${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd ${_bt}-usr.bin/lex: ${_bt}-usr.bin/m4 .endif # r245440 mtree -N support added # r313404 requires sha384.h for libnetbsd, added to libmd in r292782 .if ${BOOTSTRAPPING} < 1100093 _nmtree= lib/libmd \ lib/libnetbsd \ usr.sbin/nmtree ${_bt}-lib/libnetbsd: ${_bt}-lib/libmd ${_bt}-usr.sbin/nmtree: ${_bt}-lib/libnetbsd .endif # r246097: log addition login.conf.db, passwd, pwd.db, and spwd.db with cat -l .if ${BOOTSTRAPPING} < 1000027 _cat= bin/cat .endif # r277259 crunchide: Correct 64-bit section header offset # r281674 crunchide: always include both 32- and 64-bit ELF support .if ${BOOTSTRAPPING} < 1100078 _crunchide= usr.sbin/crunch/crunchide .endif # r285986 crunchen: use STRIPBIN rather than STRIP # 1100113: Support MK_AUTO_OBJ # 1200006: META_MODE fixes .if ${BOOTSTRAPPING} < 1100078 || \ (${MK_AUTO_OBJ} == "yes" && ${BOOTSTRAPPING} < 1100114) || \ (${MK_META_MODE} == "yes" && ${BOOTSTRAPPING} < 1200006) _crunchgen= usr.sbin/crunch/crunchgen .endif # r296926 -P keymap search path, MFC to stable/10 in r298297 .if ${BOOTSTRAPPING} < 1003501 || \ (${BOOTSTRAPPING} >= 1100000 && ${BOOTSTRAPPING} < 1100103) _kbdcontrol= usr.sbin/kbdcontrol .endif _yacc= lib/liby \ usr.bin/yacc ${_bt}-usr.bin/yacc: ${_bt}-lib/liby .if ${MK_BSNMP} != "no" _gensnmptree= usr.sbin/bsnmpd/gensnmptree .endif # We need to build tblgen when we're building clang or lld, either as # bootstrap tools, or as the part of the normal build. .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_CLANG} != "no" || \ ${MK_LLD_BOOTSTRAP} != "no" || ${MK_LLD} != "no" _clang_tblgen= \ lib/clang/libllvmminimal \ usr.bin/clang/llvm-tblgen \ usr.bin/clang/clang-tblgen ${_bt}-usr.bin/clang/clang-tblgen: ${_bt}-lib/clang/libllvmminimal ${_bt}-usr.bin/clang/llvm-tblgen: ${_bt}-lib/clang/libllvmminimal .endif # Default to building the GPL DTC, but build the BSDL one if users explicitly # request it. _dtc= usr.bin/dtc .if ${MK_GPL_DTC} != "no" _dtc= gnu/usr.bin/dtc .endif .if ${MK_KERBEROS} != "no" _kerberos5_bootstrap_tools= \ kerberos5/tools/make-roken \ kerberos5/lib/libroken \ kerberos5/lib/libvers \ kerberos5/tools/asn1_compile \ kerberos5/tools/slc \ usr.bin/compile_et .ORDER: ${_kerberos5_bootstrap_tools:C/^/${_bt}-/g} .endif # r283777 makewhatis(1) replaced with mandoc version which builds a database. _libopenbsd?= lib/libopenbsd _makewhatis= usr.bin/mandoc ${_bt}-usr.bin/mandoc: ${_bt}-lib/libopenbsd bootstrap-tools: .PHONY # Please document (add comment) why something is in 'bootstrap-tools'. # Try to bound the building of the bootstrap-tool to just the # FreeBSD versions that need the tool built at this stage of the build. .for _tool in \ ${_clang_tblgen} \ ${_kerberos5_bootstrap_tools} \ ${_strfile} \ ${_gperf} \ ${_dtc} \ ${_cat} \ ${_kbdcontrol} \ usr.bin/lorder \ ${_libopenbsd} \ ${_makewhatis} \ usr.bin/rpcgen \ ${_yacc} \ ${_m4} \ ${_lex} \ usr.bin/xinstall \ ${_gensnmptree} \ usr.sbin/config \ ${_crunchide} \ ${_crunchgen} \ ${_nmtree} \ ${_vtfontcvt} \ usr.bin/localedef ${_bt}-${_tool}: .PHONY .MAKE ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install bootstrap-tools: ${_bt}-${_tool} .endfor # # build-tools: Build special purpose build tools # .if !defined(NO_SHARE) _share= share/syscons/scrnmaps .endif .if ${MK_GCC} != "no" _gcc_tools= gnu/usr.bin/cc/cc_tools .endif .if ${MK_RESCUE} != "no" # rescue includes programs that have build-tools targets _rescue=rescue/rescue .endif .for _tool in \ bin/csh \ bin/sh \ ${LOCAL_TOOL_DIRS} \ lib/ncurses/ncurses \ lib/ncurses/ncursesw \ ${_rescue} \ ${_share} \ usr.bin/awk \ lib/libmagic \ usr.bin/mkesdb_static \ usr.bin/mkcsmapper_static \ usr.bin/vi/catalog build-tools_${_tool}: .PHONY ${_+_}@${ECHODIR} "===> ${_tool} (obj,build-tools)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ ${MAKE} DIRPRFX=${_tool}/ build-tools build-tools: build-tools_${_tool} .endfor .for _tool in \ ${_gcc_tools} build-tools_${_tool}: .PHONY ${_+_}@${ECHODIR} "===> ${_tool} (obj,all)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ ${MAKE} DIRPRFX=${_tool}/ all build-tools: build-tools_${_tool} .endfor # # kernel-tools: Build kernel-building tools # kernel-tools: .PHONY mkdir -p ${MAKEOBJDIRPREFIX}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${MAKEOBJDIRPREFIX}/usr >/dev/null # # cross-tools: All the tools needed to build the rest of the system after # we get done with the earlier stages. It is the last set of tools needed # to begin building the target binaries. # .if ${TARGET_ARCH} != ${MACHINE_ARCH} .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" _btxld= usr.sbin/btxld .endif .endif # Rebuild ctfconvert and ctfmerge to avoid difficult-to-diagnose failures # resulting from missing bug fixes or ELF Toolchain updates. .if ${MK_CDDL} != "no" _dtrace_tools= cddl/lib/libctf cddl/usr.bin/ctfconvert \ cddl/usr.bin/ctfmerge .endif # If we're given an XAS, don't build binutils. .if ${XAS:M/*} == "" .if ${MK_BINUTILS_BOOTSTRAP} != "no" _binutils= gnu/usr.bin/binutils .endif .if ${MK_ELFTOOLCHAIN_BOOTSTRAP} != "no" _elftctools= lib/libelftc \ lib/libpe \ usr.bin/elfcopy \ usr.bin/nm \ usr.bin/size \ usr.bin/strings # These are not required by the build, but can be useful for developers who # cross-build on a FreeBSD 10 host: _elftctools+= usr.bin/addr2line .endif .elif ${TARGET_ARCH} != ${MACHINE_ARCH} && ${MK_ELFTOOLCHAIN_BOOTSTRAP} != "no" # If cross-building with an external binutils we still need to build strip for # the target (for at least crunchide). _elftctools= lib/libelftc \ lib/libpe \ usr.bin/elfcopy .endif .if ${MK_CLANG_BOOTSTRAP} != "no" _clang= usr.bin/clang .endif .if ${MK_LLD_BOOTSTRAP} != "no" _lld= usr.bin/clang/lld .endif .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_LLD_BOOTSTRAP} != "no" _clang_libs= lib/clang .endif .if ${MK_GCC_BOOTSTRAP} != "no" _gcc= gnu/usr.bin/cc .endif .if ${MK_USB} != "no" _usb_tools= sys/boot/usb/tools .endif cross-tools: .MAKE .PHONY .for _tool in \ ${LOCAL_XTOOL_DIRS} \ ${_clang_libs} \ ${_clang} \ ${_lld} \ ${_binutils} \ ${_elftctools} \ ${_dtrace_tools} \ ${_gcc} \ ${_btxld} \ ${_usb_tools} ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX} install .endfor NXBDESTDIR= ${OBJTREE}/nxb-bin NXBENV= MAKEOBJDIRPREFIX=${OBJTREE}/nxb \ TOOLS_PREFIX= \ INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${PATH}:${OBJTREE}/gperf_for_gcc/usr/bin NXBMAKE= ${NXBENV} ${MAKE} \ LLVM_TBLGEN=${NXBDESTDIR}/usr/bin/llvm-tblgen \ CLANG_TBLGEN=${NXBDESTDIR}/usr/bin/clang-tblgen \ MACHINE=${TARGET} MACHINE_ARCH=${TARGET_ARCH} \ MK_GDB=no MK_TESTS=no \ SSP_CFLAGS= \ MK_HTML=no NO_LINT=yes MK_MAN=no MK_MAN_UTILS=yes \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_DEBUG_FILES=no # native-xtools is the current target for qemu-user cross builds of ports # via poudriere and the imgact_binmisc kernel module. # For non-clang enabled targets that are still using the in tree gcc # we must build a gperf binary for one instance of its Makefiles. On # clang-enabled systems, the gperf binary is obsolete. native-xtools: .PHONY .if ${MK_GCC_BOOTSTRAP} != "no" mkdir -p ${OBJTREE}/gperf_for_gcc/usr/bin ${_+_}@${ECHODIR} "===> ${_gperf} (obj,all,install)"; \ cd ${.CURDIR}/${_gperf}; \ if [ -z "${NO_OBJ}" ]; then ${NXBMAKE} DIRPRFX=${_gperf}/ obj; fi; \ ${NXBMAKE} DIRPRFX=${_gperf}/ all; \ ${NXBMAKE} DIRPRFX=${_gperf}/ DESTDIR=${OBJTREE}/gperf_for_gcc install .endif mkdir -p ${NXBDESTDIR}/bin ${NXBDESTDIR}/sbin ${NXBDESTDIR}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${NXBDESTDIR}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${NXBDESTDIR}/usr/include >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${NXBDESTDIR}/usr/lib >/dev/null .endif .for _tool in \ bin/cat \ bin/chmod \ bin/cp \ bin/csh \ bin/echo \ bin/expr \ bin/hostname \ bin/ln \ bin/ls \ bin/mkdir \ bin/mv \ bin/ps \ bin/realpath \ bin/rm \ bin/rmdir \ bin/sh \ bin/sleep \ ${_clang_tblgen} \ usr.bin/ar \ ${_binutils} \ ${_elftctools} \ ${_gcc} \ ${_gcc_tools} \ ${_clang_libs} \ ${_clang} \ ${_lld} \ sbin/md5 \ sbin/sysctl \ usr.bin/diff \ usr.bin/awk \ usr.bin/basename \ usr.bin/bmake \ usr.bin/bzip2 \ usr.bin/cmp \ usr.bin/dirname \ usr.bin/env \ usr.bin/fetch \ usr.bin/find \ usr.bin/grep \ usr.bin/gzip \ usr.bin/id \ usr.bin/lex \ usr.bin/limits \ usr.bin/lorder \ ${_libopenbsd} \ ${_makewhatis} \ usr.bin/mktemp \ usr.bin/mt \ usr.bin/patch \ usr.bin/readelf \ usr.bin/sed \ usr.bin/sort \ usr.bin/tar \ usr.bin/touch \ usr.bin/tr \ usr.bin/true \ usr.bin/uniq \ usr.bin/unzip \ usr.bin/xargs \ usr.bin/xinstall \ usr.bin/xz \ usr.bin/yacc \ usr.sbin/chown ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${NXBMAKE} DIRPRFX=${_tool}/ obj; fi; \ ${NXBMAKE} DIRPRFX=${_tool}/ all; \ ${NXBMAKE} DIRPRFX=${_tool}/ DESTDIR=${NXBDESTDIR} install .endfor # # hierarchy - ensure that all the needed directories are present # hierarchy hier: .MAKE .PHONY ${_+_}cd ${.CURDIR}/etc; ${HMAKE} distrib-dirs # # libraries - build all libraries, and install them under ${DESTDIR}. # # The list of libraries with dependents (${_prebuild_libs}) and their # interdependencies (__L) are built automatically by the # ${.CURDIR}/tools/make_libdeps.sh script. # libraries: .MAKE .PHONY ${_+_}cd ${.CURDIR}; \ ${MAKE} -f Makefile.inc1 _prereq_libs; \ ${MAKE} -f Makefile.inc1 _startup_libs; \ ${MAKE} -f Makefile.inc1 _prebuild_libs; \ ${MAKE} -f Makefile.inc1 _generic_libs # # static libgcc.a prerequisite for shared libc # _prereq_libs= lib/libcompiler_rt .if ${MK_SSP} != "no" _prereq_libs+= gnu/lib/libssp/libssp_nonshared .endif # These dependencies are not automatically generated: # # gnu/lib/csu, gnu/lib/libgcc, lib/csu and lib/libc must be built before # all shared libraries for ELF. # _startup_libs= gnu/lib/csu _startup_libs+= lib/csu _startup_libs+= lib/libcompiler_rt _startup_libs+= lib/libc _startup_libs+= lib/libc_nonshared .if ${MK_LIBCPLUSPLUS} != "no" _startup_libs+= lib/libcxxrt .endif .if ${MK_LLVM_LIBUNWIND} != "no" _prereq_libs+= lib/libgcc_eh lib/libgcc_s _startup_libs+= lib/libgcc_eh lib/libgcc_s lib/libgcc_s__L: lib/libc__L lib/libgcc_s__L: lib/libc_nonshared__L .if ${MK_LIBCPLUSPLUS} != "no" lib/libcxxrt__L: lib/libgcc_s__L .endif .else # MK_LLVM_LIBUNWIND == no _prereq_libs+= gnu/lib/libgcc _startup_libs+= gnu/lib/libgcc gnu/lib/libgcc__L: lib/libc__L gnu/lib/libgcc__L: lib/libc_nonshared__L .if ${MK_LIBCPLUSPLUS} != "no" lib/libcxxrt__L: gnu/lib/libgcc__L .endif .endif _prebuild_libs= ${_kerberos5_lib_libasn1} \ ${_kerberos5_lib_libhdb} \ ${_kerberos5_lib_libheimbase} \ ${_kerberos5_lib_libheimntlm} \ ${_libsqlite3} \ ${_kerberos5_lib_libheimipcc} \ ${_kerberos5_lib_libhx509} ${_kerberos5_lib_libkrb5} \ ${_kerberos5_lib_libroken} \ ${_kerberos5_lib_libwind} \ lib/libbz2 ${_libcom_err} lib/libcrypt \ lib/libelf lib/libexpat \ lib/libfigpar \ ${_lib_libgssapi} \ lib/libkiconv lib/libkvm lib/liblzma lib/libmd lib/libnv \ ${_lib_casper} \ lib/ncurses/ncurses lib/ncurses/ncursesw \ lib/libopie lib/libpam/libpam ${_lib_libthr} \ ${_lib_libradius} lib/libsbuf lib/libtacplus \ lib/libgeom \ ${_cddl_lib_libumem} ${_cddl_lib_libnvpair} \ ${_cddl_lib_libuutil} \ ${_cddl_lib_libavl} \ ${_cddl_lib_libzfs_core} \ ${_cddl_lib_libctf} \ lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \ ${_secure_lib_libcrypto} ${_lib_libldns} \ ${_secure_lib_libssh} ${_secure_lib_libssl} .if ${MK_GNUCXX} != "no" _prebuild_libs+= gnu/lib/libstdc++ gnu/lib/libsupc++ gnu/lib/libstdc++__L: lib/msun__L gnu/lib/libsupc++__L: gnu/lib/libstdc++__L .endif .if ${MK_DIALOG} != "no" _prebuild_libs+= gnu/lib/libdialog gnu/lib/libdialog__L: lib/msun__L lib/ncurses/ncursesw__L .endif .if ${MK_LIBCPLUSPLUS} != "no" _prebuild_libs+= lib/libc++ .endif lib/libgeom__L: lib/libexpat__L lib/libkvm__L: lib/libelf__L .if ${MK_LIBTHR} != "no" _lib_libthr= lib/libthr .endif .if ${MK_RADIUS_SUPPORT} != "no" _lib_libradius= lib/libradius .endif .if ${MK_OFED} != "no" _ofed_lib= contrib/ofed/usr.lib _prebuild_libs+= contrib/ofed/usr.lib/libosmcomp _prebuild_libs+= contrib/ofed/usr.lib/libopensm _prebuild_libs+= contrib/ofed/usr.lib/libibcommon _prebuild_libs+= contrib/ofed/usr.lib/libibverbs _prebuild_libs+= contrib/ofed/usr.lib/libibumad contrib/ofed/usr.lib/libopensm__L: lib/libthr__L contrib/ofed/usr.lib/libosmcomp__L: lib/libthr__L contrib/ofed/usr.lib/libibumad__L: contrib/ofed/usr.lib/libibcommon__L .endif .if ${MK_CASPER} != "no" _lib_casper= lib/libcasper .endif lib/libpjdlog__L: lib/libutil__L lib/libcasper__L: lib/libnv__L lib/liblzma__L: lib/libthr__L _generic_libs= ${_cddl_lib} gnu/lib ${_kerberos5_lib} lib ${_secure_lib} usr.bin/lex/lib ${_ofed_lib} .for _DIR in ${LOCAL_LIB_DIRS} .if exists(${.CURDIR}/${_DIR}/Makefile) && empty(_generic_libs:M${_DIR}) _generic_libs+= ${_DIR} .endif .endfor lib/libopie__L lib/libtacplus__L: lib/libmd__L .if ${MK_CDDL} != "no" _cddl_lib_libumem= cddl/lib/libumem _cddl_lib_libnvpair= cddl/lib/libnvpair _cddl_lib_libavl= cddl/lib/libavl _cddl_lib_libuutil= cddl/lib/libuutil +.if ${MK_ZFS} != "no" _cddl_lib_libzfs_core= cddl/lib/libzfs_core +cddl/lib/libzfs_core__L: cddl/lib/libnvpair__L +.endif _cddl_lib_libctf= cddl/lib/libctf _cddl_lib= cddl/lib -cddl/lib/libzfs_core__L: cddl/lib/libnvpair__L -cddl/lib/libzfs__L: lib/libgeom__L cddl/lib/libctf__L: lib/libz__L .endif # cddl/lib/libdtrace requires lib/libproc and lib/librtld_db; it's only built # on select architectures though (see cddl/lib/Makefile) .if ${MACHINE_CPUARCH} != "sparc64" _prebuild_libs+= lib/libprocstat lib/libproc lib/librtld_db lib/libprocstat__L: lib/libelf__L lib/libkvm__L lib/libutil__L lib/libproc__L: lib/libprocstat__L lib/librtld_db__L: lib/libprocstat__L .endif .if ${MK_CRYPT} != "no" .if ${MK_OPENSSL} != "no" _secure_lib_libcrypto= secure/lib/libcrypto _secure_lib_libssl= secure/lib/libssl lib/libradius__L secure/lib/libssl__L: secure/lib/libcrypto__L .if ${MK_LDNS} != "no" _lib_libldns= lib/libldns lib/libldns__L: secure/lib/libcrypto__L .endif .if ${MK_OPENSSH} != "no" _secure_lib_libssh= secure/lib/libssh secure/lib/libssh__L: lib/libz__L secure/lib/libcrypto__L lib/libcrypt__L .if ${MK_LDNS} != "no" secure/lib/libssh__L: lib/libldns__L .endif .if ${MK_GSSAPI} != "no" && ${MK_KERBEROS_SUPPORT} != "no" secure/lib/libssh__L: lib/libgssapi__L kerberos5/lib/libkrb5__L \ kerberos5/lib/libhx509__L kerberos5/lib/libasn1__L lib/libcom_err__L \ lib/libmd__L kerberos5/lib/libroken__L .endif .endif .endif _secure_lib= secure/lib .endif .if ${MK_KERBEROS} != "no" kerberos5/lib/libasn1__L: lib/libcom_err__L kerberos5/lib/libroken__L kerberos5/lib/libhdb__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ kerberos5/lib/libkrb5__L kerberos5/lib/libroken__L \ kerberos5/lib/libwind__L lib/libsqlite3__L kerberos5/lib/libheimntlm__L: secure/lib/libcrypto__L kerberos5/lib/libkrb5__L \ kerberos5/lib/libroken__L lib/libcom_err__L kerberos5/lib/libhx509__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ secure/lib/libcrypto__L kerberos5/lib/libroken__L kerberos5/lib/libwind__L kerberos5/lib/libkrb5__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ lib/libcrypt__L secure/lib/libcrypto__L kerberos5/lib/libhx509__L \ kerberos5/lib/libroken__L kerberos5/lib/libwind__L \ kerberos5/lib/libheimbase__L kerberos5/lib/libheimipcc__L kerberos5/lib/libroken__L: lib/libcrypt__L kerberos5/lib/libwind__L: kerberos5/lib/libroken__L lib/libcom_err__L kerberos5/lib/libheimbase__L: lib/libthr__L kerberos5/lib/libheimipcc__L: kerberos5/lib/libroken__L kerberos5/lib/libheimbase__L lib/libthr__L .endif lib/libsqlite3__L: lib/libthr__L .if ${MK_GSSAPI} != "no" _lib_libgssapi= lib/libgssapi .endif .if ${MK_KERBEROS} != "no" _kerberos5_lib= kerberos5/lib _kerberos5_lib_libasn1= kerberos5/lib/libasn1 _kerberos5_lib_libhdb= kerberos5/lib/libhdb _kerberos5_lib_libheimbase= kerberos5/lib/libheimbase _kerberos5_lib_libkrb5= kerberos5/lib/libkrb5 _kerberos5_lib_libhx509= kerberos5/lib/libhx509 _kerberos5_lib_libroken= kerberos5/lib/libroken _kerberos5_lib_libheimntlm= kerberos5/lib/libheimntlm _libsqlite3= lib/libsqlite3 _kerberos5_lib_libheimipcc= kerberos5/lib/libheimipcc _kerberos5_lib_libwind= kerberos5/lib/libwind _libcom_err= lib/libcom_err .endif .if ${MK_NIS} != "no" _lib_libypclnt= lib/libypclnt .endif .if ${MK_OPENSSL} == "no" lib/libradius__L: lib/libmd__L .endif lib/libproc__L: \ ${_cddl_lib_libctf:D${_cddl_lib_libctf}__L} lib/libelf__L lib/librtld_db__L lib/libutil__L .if ${MK_CXX} != "no" .if ${MK_LIBCPLUSPLUS} != "no" lib/libproc__L: lib/libcxxrt__L .else # This implies MK_GNUCXX != "no"; see lib/libproc lib/libproc__L: gnu/lib/libsupc++__L .endif .endif .for _lib in ${_prereq_libs} ${_lib}__PL: .PHONY .MAKE .if exists(${.CURDIR}/${_lib}) ${_+_}@${ECHODIR} "===> ${_lib} (obj,all,install)"; \ cd ${.CURDIR}/${_lib}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj; fi; \ ${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \ DIRPRFX=${_lib}/ all; \ ${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \ DIRPRFX=${_lib}/ install .endif .endfor .for _lib in ${_startup_libs} ${_prebuild_libs} ${_generic_libs} ${_lib}__L: .PHONY .MAKE .if exists(${.CURDIR}/${_lib}) ${_+_}@${ECHODIR} "===> ${_lib} (obj,all,install)"; \ cd ${.CURDIR}/${_lib}; \ if [ -z "${NO_OBJ}" ]; then ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj; fi; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ all; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ install .endif .endfor _prereq_libs: ${_prereq_libs:S/$/__PL/} _startup_libs: ${_startup_libs:S/$/__L/} _prebuild_libs: ${_prebuild_libs:S/$/__L/} _generic_libs: ${_generic_libs:S/$/__L/} # Enable SUBDIR_PARALLEL when not calling 'make all', unless called from # 'everything' with _PARALLEL_SUBDIR_OK set. This is because it is unlikely # that running 'make all' from the top-level, especially with a SUBDIR_OVERRIDE # or LOCAL_DIRS set, will have a reliable build if SUBDIRs are built in # parallel. This is safe for the world stage of buildworld though since it has # already built libraries in a proper order and installed includes into # WORLDTMP. Special handling is done for SUBDIR ordering for 'install*' to # avoid trashing a system if it crashes mid-install. .if !make(all) || defined(_PARALLEL_SUBDIR_OK) SUBDIR_PARALLEL= .endif .include .if make(check-old) || make(check-old-dirs) || \ make(check-old-files) || make(check-old-libs) || \ make(delete-old) || make(delete-old-dirs) || \ make(delete-old-files) || make(delete-old-libs) # # check for / delete old files section # .include "ObsoleteFiles.inc" OLD_LIBS_MESSAGE="Please be sure no application still uses those libraries, \ else you can not start such an application. Consult UPDATING for more \ information regarding how to cope with the removal/revision bump of a \ specific library." .if !defined(BATCH_DELETE_OLD_FILES) RM_I=-i .else RM_I=-v .endif delete-old-files: .PHONY @echo ">>> Removing old files (only deletes safe to delete libs)" # Ask for every old file if the user really wants to remove it. # It's annoying, but better safe than sorry. # NB: We cannot pass the list of OLD_FILES as a parameter because the # argument list will get too long. Using .for/.endfor make "loops" will make # the Makefile parser segfault. @exec 3<&0; \ cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \ rm ${RM_I} "${DESTDIR}/$${file}" <&3; \ fi; \ for ext in debug symbols; do \ if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \ "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \ <&3; \ fi; \ done; \ done # Remove catpages without corresponding manpages. @exec 3<&0; \ find ${DESTDIR}/usr/share/man/cat* ! -type d | \ sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \ while read catpage; do \ read manpage; \ if [ ! -e "$${manpage}" ]; then \ rm ${RM_I} $${catpage} <&3; \ fi; \ done @echo ">>> Old files removed" check-old-files: .PHONY @echo ">>> Checking for old files" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ echo "${DESTDIR}/$${file}"; \ fi; \ for ext in debug symbols; do \ if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \ fi; \ done; \ done # Check for catpages without corresponding manpages. @find ${DESTDIR}/usr/share/man/cat* ! -type d | \ sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \ while read catpage; do \ read manpage; \ if [ ! -e "$${manpage}" ]; then \ echo $${catpage}; \ fi; \ done delete-old-libs: .PHONY @echo ">>> Removing old libraries" @echo "${OLD_LIBS_MESSAGE}" | fmt @exec 3<&0; \ cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_LIBS | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \ rm ${RM_I} "${DESTDIR}/$${file}" <&3; \ fi; \ for ext in debug symbols; do \ if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \ "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \ <&3; \ fi; \ done; \ done @echo ">>> Old libraries removed" check-old-libs: .PHONY @echo ">>> Checking for old libraries" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_LIBS | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ echo "${DESTDIR}/$${file}"; \ fi; \ for ext in debug symbols; do \ if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \ fi; \ done; \ done delete-old-dirs: .PHONY @echo ">>> Removing old directories" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_DIRS | xargs -n1 | sort -r | \ while read dir; do \ if [ -d "${DESTDIR}/$${dir}" ]; then \ rmdir -v "${DESTDIR}/$${dir}" || true; \ elif [ -L "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ if [ -d "${DESTDIR}${DEBUGDIR}/$${dir}" ]; then \ rmdir -v "${DESTDIR}${DEBUGDIR}/$${dir}" || true; \ elif [ -L "${DESTDIR}${DEBUGDIR}/$${dir}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ done @echo ">>> Old directories removed" check-old-dirs: .PHONY @echo ">>> Checking for old directories" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_DIRS | xargs -n1 | \ while read dir; do \ if [ -d "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir}"; \ elif [ -L "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ if [ -d "${DESTDIR}${DEBUGDIR}/$${dir}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${dir}"; \ elif [ -L "${DESTDIR}${DEBUGDIR}/$${dir}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ done delete-old: delete-old-files delete-old-dirs .PHONY @echo "To remove old libraries run '${MAKE_CMD} delete-old-libs'." check-old: check-old-files check-old-libs check-old-dirs .PHONY @echo "To remove old files and directories run '${MAKE_CMD} delete-old'." @echo "To remove old libraries run '${MAKE_CMD} delete-old-libs'." .endif # # showconfig - show build configuration. # showconfig: .PHONY @(${MAKE} -n -f ${.CURDIR}/sys/conf/kern.opts.mk -V dummy -dg1 UPDATE_DEPENDFILE=no NO_OBJ=yes; \ ${MAKE} -n -f ${.CURDIR}/share/mk/src.opts.mk -V dummy -dg1 UPDATE_DEPENDFILE=no NO_OBJ=yes) 2>&1 | grep ^MK_ | sort -u .if !empty(KRNLOBJDIR) && !empty(KERNCONF) DTBOUTPUTPATH= ${KRNLOBJDIR}/${KERNCONF}/ .if !defined(FDT_DTS_FILE) || empty(FDT_DTS_FILE) .if exists(${KERNCONFDIR}/${KERNCONF}) FDT_DTS_FILE!= awk 'BEGIN {FS="="} /^makeoptions[[:space:]]+FDT_DTS_FILE/ {print $$2}' \ '${KERNCONFDIR}/${KERNCONF}' ; echo .endif .endif .endif .if !defined(DTBOUTPUTPATH) || !exists(${DTBOUTPUTPATH}) DTBOUTPUTPATH= ${.CURDIR} .endif # # Build 'standalone' Device Tree Blob # builddtb: .PHONY @PATH=${TMPPATH} MACHINE=${TARGET} \ ${.CURDIR}/sys/tools/fdt/make_dtb.sh ${.CURDIR}/sys \ "${FDT_DTS_FILE}" ${DTBOUTPUTPATH} ############### # cleanworld # In the following, the first 'rm' in a series will usually remove all # files and directories. If it does not, then there are probably some # files with file flags set, so this unsets them and tries the 'rm' a # second time. There are situations where this target will be cleaning # some directories via more than one method, but that duplication is # needed to correctly handle all the possible situations. Removing all # files without file flags set in the first 'rm' instance saves time, # because 'chflags' will need to operate on fewer files afterwards. # # It is expected that BW_CANONICALOBJDIR == the CANONICALOBJDIR as would be # created by bsd.obj.mk, except that we don't want to .include that file # in this makefile. # BW_CANONICALOBJDIR:=${OBJTREE}${.CURDIR} cleanworld: .PHONY .if exists(${BW_CANONICALOBJDIR}/) -rm -rf ${BW_CANONICALOBJDIR}/* -chflags -R 0 ${BW_CANONICALOBJDIR} rm -rf ${BW_CANONICALOBJDIR}/* .endif .if ${.CURDIR} == ${.OBJDIR} || ${.CURDIR}/obj == ${.OBJDIR} # To be safe in this case, fall back to a 'make cleandir' ${_+_}@cd ${.CURDIR}; ${MAKE} cleandir .endif .if defined(TARGET) && defined(TARGET_ARCH) .if ${TARGET} == ${MACHINE} && ${TARGET_ARCH} == ${MACHINE_ARCH} XDEV_CPUTYPE?=${CPUTYPE} .else XDEV_CPUTYPE?=${TARGET_CPUTYPE} .endif NOFUN=-DNO_FSCHG MK_HTML=no -DNO_LINT \ MK_MAN=no MK_NLS=no MK_PROFILE=no \ MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ CPUTYPE=${XDEV_CPUTYPE} XDDIR=${TARGET_ARCH}-freebsd XDTP?=/usr/${XDDIR} .if ${XDTP:N/*} .error XDTP variable should be an absolute path .endif CDBENV=MAKEOBJDIRPREFIX=${MAKEOBJDIRPREFIX}/${XDDIR} \ INSTALL="sh ${.CURDIR}/tools/install.sh" CDENV= ${CDBENV} \ TOOLS_PREFIX=${XDTP} .if ${WANT_COMPILER_TYPE} == gcc || \ (defined(X_COMPILER_TYPE) && ${X_COMPILER_TYPE} == gcc) # GCC requires -isystem and -L when using a cross-compiler. --sysroot # won't set header path and -L is used to ensure the base library path # is added before the port PREFIX library path. CD2CFLAGS+= -isystem ${XDDESTDIR}/usr/include -L${XDDESTDIR}/usr/lib # GCC requires -B to find /usr/lib/crti.o when using a cross-compiler # combined with --sysroot. CD2CFLAGS+= -B${XDDESTDIR}/usr/lib # Force using libc++ for external GCC. # XXX: This should be checking MK_GNUCXX == no .if ${X_COMPILER_VERSION} >= 40800 CD2CXXFLAGS+= -isystem ${XDDESTDIR}/usr/include/c++/v1 -std=c++11 \ -nostdinc++ .endif .endif CD2CFLAGS+= --sysroot=${XDDESTDIR}/ CD2ENV=${CDENV} CC="${CC} ${CD2CFLAGS}" CXX="${CXX} ${CD2CXXFLAGS} ${CD2CFLAGS}" \ CPP="${CPP} ${CD2CFLAGS}" \ MACHINE=${TARGET} MACHINE_ARCH=${TARGET_ARCH} CDTMP= ${MAKEOBJDIRPREFIX}/${XDDIR}/${.CURDIR}/tmp CDMAKE=${CDENV} PATH=${CDTMP}/usr/bin:${PATH} ${MAKE} ${NOFUN} CD2MAKE=${CD2ENV} PATH=${CDTMP}/usr/bin:${XDDESTDIR}/usr/bin:${PATH} ${MAKE} ${NOFUN} .if ${MK_META_MODE} != "no" # Don't rebuild build-tools targets during normal build. CD2MAKE+= BUILD_TOOLS_META=.NOMETA .endif XDDESTDIR=${DESTDIR}/${XDTP} .if !defined(OSREL) OSREL!= uname -r | sed -e 's/[-(].*//' .endif .ORDER: xdev-build xdev-install xdev-links xdev: xdev-build xdev-install .PHONY .ORDER: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools xdev-build: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools .PHONY _xb-worldtmp: .PHONY mkdir -p ${CDTMP}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${CDTMP}/usr >/dev/null _xb-bootstrap-tools: .PHONY .for _tool in \ ${_clang_tblgen} \ ${_gperf} \ ${_yacc} ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${CDMAKE} DIRPRFX=${_tool}/ obj; fi; \ ${CDMAKE} DIRPRFX=${_tool}/ all; \ ${CDMAKE} DIRPRFX=${_tool}/ DESTDIR=${CDTMP} install .endfor _xb-build-tools: .PHONY ${_+_}@cd ${.CURDIR}; \ ${CDBENV} ${MAKE} -f Makefile.inc1 ${NOFUN} build-tools _xb-cross-tools: .PHONY .for _tool in \ ${_binutils} \ ${_elftctools} \ usr.bin/ar \ ${_clang_libs} \ ${_clang} \ ${_gcc} ${_+_}@${ECHODIR} "===> xdev ${_tool} (obj,all)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJ}" ]; then ${CDMAKE} DIRPRFX=${_tool}/ obj; fi; \ ${CDMAKE} DIRPRFX=${_tool}/ all .endfor _xi-mtree: .PHONY ${_+_}@${ECHODIR} "mtree populating ${XDDESTDIR}" mkdir -p ${XDDESTDIR} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \ -p ${XDDESTDIR} >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${XDDESTDIR}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${XDDESTDIR}/usr/include >/dev/null .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${XDDESTDIR}/usr >/dev/null .endif .if ${MK_TESTS} != "no" mkdir -p ${XDDESTDIR}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${XDDESTDIR}${TESTSBASE} >/dev/null .endif .ORDER: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries xdev-install: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries .PHONY _xi-cross-tools: .PHONY @echo "_xi-cross-tools" .for _tool in \ ${_binutils} \ ${_elftctools} \ usr.bin/ar \ ${_clang_libs} \ ${_clang} \ ${_gcc} ${_+_}@${ECHODIR} "===> xdev ${_tool} (install)"; \ cd ${.CURDIR}/${_tool}; \ ${CDMAKE} DIRPRFX=${_tool}/ install DESTDIR=${XDDESTDIR} .endfor _xi-includes: .PHONY ${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 includes \ DESTDIR=${XDDESTDIR} _xi-libraries: .PHONY ${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 libraries \ DESTDIR=${XDDESTDIR} xdev-links: .PHONY ${_+_}cd ${XDDESTDIR}/usr/bin; \ mkdir -p ../../../../usr/bin; \ for i in *; do \ ln -sf ../../${XDTP}/usr/bin/$$i \ ../../../../usr/bin/${XDDIR}-$$i; \ ln -sf ../../${XDTP}/usr/bin/$$i \ ../../../../usr/bin/${XDDIR}${OSREL}-$$i; \ done .else xdev xdev-build xdev-install xdev-links: .PHONY @echo "*** Error: Both TARGET and TARGET_ARCH must be defined for \"${.TARGET}\" target" .endif Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.DestructWithFunction.d.ksh =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.DestructWithFunction.d.ksh (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.DestructWithFunction.d.ksh (revision 321585) @@ -1,56 +1,69 @@ #!/bin/ksh -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # #ident "%Z%%M% %I% %E% SMI" ## # # ASSERTION: # The -w option can be used to permit destructive actions in D programs. # # SECTION: dtrace Utility/-w Option; # dtrace Utility/-f Option # ## +reader() +{ + while true + do + sleep 0.1 + cat /etc/motd > /dev/null + done +} if [ $# != 1 ]; then echo expected one argument: '<'dtrace-path'>' exit 2 fi dtrace=$1 +reader & +child=$! + $dtrace -qwf read'{chill(15); printf("Done chilling"); exit(0);}' status=$? if [ "$status" -ne 0 ]; then echo $tst: dtrace failed fi + +kill $child exit $status Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.ZeroFunctionProbes.d.ksh =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.ZeroFunctionProbes.d.ksh (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/dtraceUtil/tst.ZeroFunctionProbes.d.ksh (revision 321585) @@ -1,58 +1,72 @@ #!/bin/ksh -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # #ident "%Z%%M% %I% %E% SMI" ## # # ASSERTION: # The -Z option can be used to permit descriptions that match # zero probes. # # SECTION: dtrace Utility/-Z Option; # dtrace Utility/-f Option # ## +reader() +{ + while true + do + sleep 0.1 + cat /etc/motd > /dev/null + done +} + if [ $# != 1 ]; then echo expected one argument: '<'dtrace-path'>' exit 2 fi dtrace=$1 +reader & +child=$! + $dtrace -qZf wassup'{printf("Iamkool");}' \ -qf read'{printf("I am done"); exit(0);}' status=$? if [ "$status" -ne 0 ]; then echo $tst: dtrace failed fi + +kill $child exit $status Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d (revision 321585) @@ -1,87 +1,106 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #pragma D option quiet +#pragma D option dynvarsize=2m /* * This test verifies that the basename() and dirname() functions are working * properly. Note that the output of this is a ksh script. When run, * it will give no output if the output is correct. */ BEGIN { dir[i++] = "/foo/bar/baz"; dir[i++] = "/foo/bar///baz/"; dir[i++] = "/foo/bar/baz/"; dir[i++] = "/foo/bar/baz//"; dir[i++] = "/foo/bar/baz/."; dir[i++] = "/foo/bar/baz/./"; dir[i++] = "/foo/bar/baz/.//"; dir[i++] = "foo/bar/baz/"; dir[i++] = "/"; dir[i++] = "./"; dir[i++] = "//"; dir[i++] = "/."; dir[i++] = "/./"; dir[i++] = "/./."; dir[i++] = "/.//"; dir[i++] = "."; dir[i++] = "f"; dir[i++] = "f/"; dir[i++] = "/////"; + /* + * basename(3) and basename(1) return different results for the empty + * string on FreeBSD, so we need special handling. dir[i++] = ""; + */ end = i; i = 0; - printf("#!/usr/bin/ksh\n\n"); + printf("#!/usr/bin/env ksh\n\n"); } tick-1ms /i < end/ { printf("if [ `basename \"%s\"` != \"%s\" ]; then\n", dir[i], basename(dir[i])); printf(" echo \"basename(\\\"%s\\\") is \\\"%s\\\"; ", dir[i], basename(dir[i])); printf("expected \\\"`basename \"%s\"`\"\\\"\n", dir[i]); printf("fi\n\n"); printf("if [ `dirname \"%s\"` != \"%s\" ]; then\n", dir[i], dirname(dir[i])); printf(" echo \"dirname(\\\"%s\\\") is \\\"%s\\\"; ", dir[i], dirname(dir[i])); printf("expected \\\"`dirname \"%s\"`\"\\\"\n", dir[i]); printf("fi\n\n"); i++; } tick-1ms /i == end/ { + dir[i] = ""; + printf("if [ \"`basename \"%s\"`\" != \"%s\" -a \".\" != \"%s\" ]; then\n", + dir[i], basename(dir[i]), basename(dir[i])); + printf(" echo \"basename(\\\"%s\\\") is \\\"%s\\\"; ", + dir[i], basename(dir[i])); + printf("expected \\\"`basename \"%s\"`\\\" or \\\".\\\"\"\n", dir[i]); + printf("fi\n\n"); + printf("if [ `dirname \"%s\"` != \"%s\" ]; then\n", + dir[i], dirname(dir[i])); + printf(" echo \"dirname(\\\"%s\\\") is \\\"%s\\\"; ", + dir[i], dirname(dir[i])); + printf("expected \\\"`dirname \"%s\"`\"\\\"\n", dir[i]); + printf("fi\n\n"); + exit(0); } Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d.out =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d.out (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.basename.d.out (revision 321585) @@ -1,163 +1,163 @@ -#!/usr/bin/ksh +#!/usr/bin/env ksh if [ `basename "/foo/bar/baz"` != "baz" ]; then echo "basename(\"/foo/bar/baz\") is \"baz\"; expected \"`basename "/foo/bar/baz"`"\" fi if [ `dirname "/foo/bar/baz"` != "/foo/bar" ]; then echo "dirname(\"/foo/bar/baz\") is \"/foo/bar\"; expected \"`dirname "/foo/bar/baz"`"\" fi if [ `basename "/foo/bar///baz/"` != "baz" ]; then echo "basename(\"/foo/bar///baz/\") is \"baz\"; expected \"`basename "/foo/bar///baz/"`"\" fi if [ `dirname "/foo/bar///baz/"` != "/foo/bar" ]; then echo "dirname(\"/foo/bar///baz/\") is \"/foo/bar\"; expected \"`dirname "/foo/bar///baz/"`"\" fi if [ `basename "/foo/bar/baz/"` != "baz" ]; then echo "basename(\"/foo/bar/baz/\") is \"baz\"; expected \"`basename "/foo/bar/baz/"`"\" fi if [ `dirname "/foo/bar/baz/"` != "/foo/bar" ]; then echo "dirname(\"/foo/bar/baz/\") is \"/foo/bar\"; expected \"`dirname "/foo/bar/baz/"`"\" fi if [ `basename "/foo/bar/baz//"` != "baz" ]; then echo "basename(\"/foo/bar/baz//\") is \"baz\"; expected \"`basename "/foo/bar/baz//"`"\" fi if [ `dirname "/foo/bar/baz//"` != "/foo/bar" ]; then echo "dirname(\"/foo/bar/baz//\") is \"/foo/bar\"; expected \"`dirname "/foo/bar/baz//"`"\" fi if [ `basename "/foo/bar/baz/."` != "." ]; then echo "basename(\"/foo/bar/baz/.\") is \".\"; expected \"`basename "/foo/bar/baz/."`"\" fi if [ `dirname "/foo/bar/baz/."` != "/foo/bar/baz" ]; then echo "dirname(\"/foo/bar/baz/.\") is \"/foo/bar/baz\"; expected \"`dirname "/foo/bar/baz/."`"\" fi if [ `basename "/foo/bar/baz/./"` != "." ]; then echo "basename(\"/foo/bar/baz/./\") is \".\"; expected \"`basename "/foo/bar/baz/./"`"\" fi if [ `dirname "/foo/bar/baz/./"` != "/foo/bar/baz" ]; then echo "dirname(\"/foo/bar/baz/./\") is \"/foo/bar/baz\"; expected \"`dirname "/foo/bar/baz/./"`"\" fi if [ `basename "/foo/bar/baz/.//"` != "." ]; then echo "basename(\"/foo/bar/baz/.//\") is \".\"; expected \"`basename "/foo/bar/baz/.//"`"\" fi if [ `dirname "/foo/bar/baz/.//"` != "/foo/bar/baz" ]; then echo "dirname(\"/foo/bar/baz/.//\") is \"/foo/bar/baz\"; expected \"`dirname "/foo/bar/baz/.//"`"\" fi if [ `basename "foo/bar/baz/"` != "baz" ]; then echo "basename(\"foo/bar/baz/\") is \"baz\"; expected \"`basename "foo/bar/baz/"`"\" fi if [ `dirname "foo/bar/baz/"` != "foo/bar" ]; then echo "dirname(\"foo/bar/baz/\") is \"foo/bar\"; expected \"`dirname "foo/bar/baz/"`"\" fi if [ `basename "/"` != "/" ]; then echo "basename(\"/\") is \"/\"; expected \"`basename "/"`"\" fi if [ `dirname "/"` != "/" ]; then echo "dirname(\"/\") is \"/\"; expected \"`dirname "/"`"\" fi if [ `basename "./"` != "." ]; then echo "basename(\"./\") is \".\"; expected \"`basename "./"`"\" fi if [ `dirname "./"` != "." ]; then echo "dirname(\"./\") is \".\"; expected \"`dirname "./"`"\" fi if [ `basename "//"` != "/" ]; then echo "basename(\"//\") is \"/\"; expected \"`basename "//"`"\" fi if [ `dirname "//"` != "/" ]; then echo "dirname(\"//\") is \"/\"; expected \"`dirname "//"`"\" fi if [ `basename "/."` != "." ]; then echo "basename(\"/.\") is \".\"; expected \"`basename "/."`"\" fi if [ `dirname "/."` != "/" ]; then echo "dirname(\"/.\") is \"/\"; expected \"`dirname "/."`"\" fi if [ `basename "/./"` != "." ]; then echo "basename(\"/./\") is \".\"; expected \"`basename "/./"`"\" fi if [ `dirname "/./"` != "/" ]; then echo "dirname(\"/./\") is \"/\"; expected \"`dirname "/./"`"\" fi if [ `basename "/./."` != "." ]; then echo "basename(\"/./.\") is \".\"; expected \"`basename "/./."`"\" fi if [ `dirname "/./."` != "/." ]; then echo "dirname(\"/./.\") is \"/.\"; expected \"`dirname "/./."`"\" fi if [ `basename "/.//"` != "." ]; then echo "basename(\"/.//\") is \".\"; expected \"`basename "/.//"`"\" fi if [ `dirname "/.//"` != "/" ]; then echo "dirname(\"/.//\") is \"/\"; expected \"`dirname "/.//"`"\" fi if [ `basename "."` != "." ]; then echo "basename(\".\") is \".\"; expected \"`basename "."`"\" fi if [ `dirname "."` != "." ]; then echo "dirname(\".\") is \".\"; expected \"`dirname "."`"\" fi if [ `basename "f"` != "f" ]; then echo "basename(\"f\") is \"f\"; expected \"`basename "f"`"\" fi if [ `dirname "f"` != "." ]; then echo "dirname(\"f\") is \".\"; expected \"`dirname "f"`"\" fi if [ `basename "f/"` != "f" ]; then echo "basename(\"f/\") is \"f\"; expected \"`basename "f/"`"\" fi if [ `dirname "f/"` != "." ]; then echo "dirname(\"f/\") is \".\"; expected \"`dirname "f/"`"\" fi if [ `basename "/////"` != "/" ]; then echo "basename(\"/////\") is \"/\"; expected \"`basename "/////"`"\" fi if [ `dirname "/////"` != "/" ]; then echo "dirname(\"/////\") is \"/\"; expected \"`dirname "/////"`"\" fi -if [ `basename ""` != "." ]; then - echo "basename(\"\") is \".\"; expected \"`basename ""`"\" +if [ "`basename ""`" != "." -a "." != "." ]; then + echo "basename(\"\") is \".\"; expected \"`basename ""`\" or \".\"" fi if [ `dirname ""` != "." ]; then echo "dirname(\"\") is \".\"; expected \"`dirname ""`"\" fi Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.cleanpath.d =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.cleanpath.d (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/funcs/tst.cleanpath.d (revision 321585) @@ -1,70 +1,71 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #pragma D option quiet +#pragma D option dynvarsize=2m BEGIN { path[i++] = "/foo/bar/baz"; path[i++] = "/foo/bar///baz/"; path[i++] = "/foo/bar/baz/"; path[i++] = "/foo/bar/baz//"; path[i++] = "/foo/bar/baz/."; path[i++] = "/foo/bar/baz/./"; path[i++] = "/foo/bar/../../baz/.//"; path[i++] = "foo/bar/./././././baz/"; path[i++] = "/foo/bar/baz/../../../../../../"; path[i++] = "/../../../../../../"; path[i++] = "/./"; path[i++] = "/foo/bar/baz/../../bop/bang/../../bar/baz/"; path[i++] = "./"; path[i++] = "//"; path[i++] = "/."; path[i++] = "/./"; path[i++] = "/./."; path[i++] = "/.//"; path[i++] = "."; path[i++] = "/////"; path[i++] = ""; end = i; i = 0; } tick-1ms /i < end/ { printf("cleanpath(\"%s\") = \"%s\"\n", path[i], cleanpath(path[i])); i++; } tick-1ms /i == end/ { exit(0); } Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh (revision 321585) @@ -1,41 +1,42 @@ # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # if [ $# != 1 ]; then echo expected one argument: '<'dtrace-path'>' exit 2 fi dtrace=$1 -$dtrace -ln 'syscall::*$1:entry' read | awk '{print $(NF-1),$NF}' | sort -$dtrace -ln 'syscall::$1*:entry' read | awk '{print $(NF-1),$NF}' | sort -$dtrace -ln 'syscall::re$1*:entry' ad | awk '{print $(NF-1),$NF}' | sort -$dtrace -ln 'syscall::$1l*:entry' read | awk '{print $(NF-1),$NF}' | sort -$dtrace -ln 'syscall::p$1[0-9][0-9]:entry' read | awk '{print $(NF-1),$NF}' | \ - sort +$dtrace -ln 'syscall:freebsd:*$1:entry' read | \ + awk '{print $(NF-1),$NF}' | grep -v -E 'compat.\.' | sort +$dtrace -ln 'syscall:freebsd:$1*:entry' read | awk '{print $(NF-1),$NF}' | sort +$dtrace -ln 'syscall:freebsd:re$1*:entry' ad | awk '{print $(NF-1),$NF}' | sort +$dtrace -ln 'syscall:freebsd:$1l*:entry' read | awk '{print $(NF-1),$NF}' | sort +$dtrace -ln 'syscall:freebsd:w$1[0-9]:entry' ait | \ + awk '{print $(NF-1),$NF}' | sort exit $status Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh.out =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh.out (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/misc/tst.macroglob.ksh.out (revision 321585) @@ -1,15 +1,22 @@ FUNCTION NAME +aio_read entry +obs_vread entry pread entry read entry +rtprio_thread entry FUNCTION NAME read entry readlink entry +readlinkat entry readv entry FUNCTION NAME read entry readlink entry +readlinkat entry readv entry FUNCTION NAME readlink entry +readlinkat entry FUNCTION NAME -pread64 entry +wait4 entry +wait6 entry Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.c =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.c (nonexistent) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.c (revision 321585) @@ -0,0 +1,47 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2017 Li-Wen Hsu + */ + +#include +#include +#include + +int +main(void) +{ + + sigset_t set; + siginfo_t info; + struct timespec timeout; + + (void)sigemptyset(&set); + (void)sigaddset(&set, SIGHUP); + timeout.tv_sec = 1; + timeout.tv_nsec = 0; + + for (;;) + (void)sigtimedwait(&set, &info, &timeout); + + return (0); +} Property changes on: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.d =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.d (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/probes/tst.probestar.d (revision 321585) @@ -1,50 +1,51 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" /* * ASSERTION: * * Call matching probe clauses with given name using "*" * * SECTION: Program Structure/Probe Clauses and Declarations * */ #pragma D option quiet int i; BEGIN { i = 0; } syscall::*wait*:entry +/pid == $1/ { exit(0); } Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.d =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.d (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.d (revision 321585) @@ -1,64 +1,65 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" /* * ASSERTION: * The number of speculative buffers defaults to one. If no speculative buffer * is available when speculation is called, an ID of zero is returned. * * SECTION: Speculative Tracing/Creating a Speculation * */ #pragma D option quiet BEGIN { i = 0; + notused = $1; } syscall::open:entry /i < 2/ { self->spec = speculation(); printf("Speculative buffer ID: %d\n", self->spec); i++; } syscall::: /(2 == i) && (0 == self->spec)/ { printf("i: %d\tself->spec: %d", i, self->spec); exit(0); } syscall::: /(2 == i) && (0 != self->spec)/ { printf("i: %d\tself->spec: %d", i, self->spec); exit(1); } Index: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.exe =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.exe (nonexistent) +++ projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.exe (revision 321585) @@ -0,0 +1,30 @@ +#!/usr/bin/env ksh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2017 Li-Wen Hsu + +while true +do + sleep 0.1 + cat /etc/motd > /dev/null +done Property changes on: projects/runtime-coverage/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/speculation/tst.NoSpecBuffer.exe ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/runtime-coverage/cddl/contrib/opensolaris =================================================================== --- projects/runtime-coverage/cddl/contrib/opensolaris (revision 321584) +++ projects/runtime-coverage/cddl/contrib/opensolaris (revision 321585) Property changes on: projects/runtime-coverage/cddl/contrib/opensolaris ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/cddl/contrib/opensolaris:r321426-321584 Index: projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/probes/Makefile =================================================================== --- projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/probes/Makefile (revision 321584) +++ projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/probes/Makefile (revision 321585) @@ -1,38 +1,39 @@ # $FreeBSD$ # # This Makefile was generated by $srcdir/cddl/usr.sbin/dtrace/tests/tools/genmakefiles.sh. # PACKAGE= tests ${PACKAGE}FILES= \ err.D_PDESC_ZERO.probeqtn.d \ err.D_PDESC_ZERO.probestar.d \ err.D_PDESC_ZERO.tickstar.d \ err.D_SYNTAX.assign.d \ err.D_SYNTAX.declare.d \ err.D_SYNTAX.declarein.d \ err.D_SYNTAX.lbraces.d \ err.D_SYNTAX.probespec.d \ err.D_SYNTAX.rbraces.d \ err.D_SYNTAX.recdec.d \ tst.basic1.d \ tst.check.d \ tst.declare.d \ tst.declareafter.d \ tst.emptyprobe.d \ tst.pragma.d \ tst.pragmaaftertab.d \ tst.pragmainside.d \ tst.pragmaoutside.d \ tst.probestar.d \ TESTEXES= \ CFILES= \ + tst.probestar.c \ .include "../../dtrace.test.mk" Index: projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/speculation/Makefile =================================================================== --- projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/speculation/Makefile (revision 321584) +++ projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/common/speculation/Makefile (revision 321585) @@ -1,77 +1,78 @@ # $FreeBSD$ # # This Makefile was generated by $srcdir/cddl/usr.sbin/dtrace/tests/tools/genmakefiles.sh. # PACKAGE= tests ${PACKAGE}FILES= \ bug.1001148.SpecSizeVariations.d \ err.BufSizeVariations1.d \ err.BufSizeVariations2.d \ err.D_ACT_SPEC.SpeculateWithBreakPoint.d \ err.D_ACT_SPEC.SpeculateWithChill.d \ err.D_ACT_SPEC.SpeculateWithCopyOut.d \ err.D_ACT_SPEC.SpeculateWithCopyOutStr.d \ err.D_ACT_SPEC.SpeculateWithPanic.d \ err.D_ACT_SPEC.SpeculateWithRaise.d \ err.D_ACT_SPEC.SpeculateWithStop.d \ err.D_AGG_COMM.AggAftCommit.d \ err.D_AGG_SPEC.SpeculateWithAvg.d \ err.D_AGG_SPEC.SpeculateWithCount.d \ err.D_AGG_SPEC.SpeculateWithLquant.d \ err.D_AGG_SPEC.SpeculateWithMax.d \ err.D_AGG_SPEC.SpeculateWithMin.d \ err.D_AGG_SPEC.SpeculateWithQuant.d \ err.D_AGG_SPEC.SpeculateWithStddev.d \ err.D_AGG_SPEC.SpeculateWithSum.d \ err.D_COMM_COMM.CommitAftCommit.d \ err.D_COMM_COMM.DisjointCommit.d \ err.D_COMM_DREC.CommitAftDataRec.d \ err.D_DREC_COMM.DataRecAftCommit.d \ err.D_DREC_COMM.ExitAfterCommit.d \ err.D_EXIT_SPEC.ExitAftSpec.d \ err.D_PRAGMA_MALFORM.NspecExpr.d \ err.D_PRAGMA_OPTSET.HugeNspecValue.d \ err.D_PRAGMA_OPTSET.InvalidSpecSize.d \ err.D_PRAGMA_OPTSET.NegSpecSize.d \ err.D_PROTO_LEN.SpecNoId.d \ err.D_SPEC_COMM.SpecAftCommit.d \ err.D_SPEC_DREC.SpecAftDataRec.d \ err.D_SPEC_SPEC.SpecAftSpec.d \ err.NegativeBufSize.d \ err.NegativeNspec.d \ err.NegativeSpecSize.d \ err.SpecSizeVariations1.d \ err.SpecSizeVariations2.d \ tst.CommitAfterDiscard.d \ tst.CommitWithZero.d \ tst.DataRecAftDiscard.d \ tst.DiscardAftCommit.d \ tst.DiscardAftDataRec.d \ tst.DiscardAftDiscard.d \ tst.DiscardWithZero.d \ tst.ExitAftDiscard.d \ tst.NoSpecBuffer.d \ tst.SpecSizeVariations1.d \ tst.SpecSizeVariations2.d \ tst.SpecSizeVariations3.d \ tst.SpeculateWithRandom.d \ tst.SpeculationCommit.d \ tst.SpeculationDiscard.d \ tst.SpeculationID.d \ tst.SpeculationWithZero.d \ tst.TwoSpecBuffers.d \ tst.negcommit.d \ tst.negspec.d \ tst.zerosize.d \ TESTEXES= \ + tst.NoSpecBuffer.exe \ CFILES= \ .include "../../dtrace.test.mk" Index: projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/tools/dtest.sh =================================================================== --- projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/tools/dtest.sh (revision 321584) +++ projects/runtime-coverage/cddl/usr.sbin/dtrace/tests/tools/dtest.sh (revision 321585) @@ -1,129 +1,137 @@ +#!/bin/sh + # $FreeBSD$ usage() { cat >&2 <<__EOF__ A harness for test cases in the DTrace test suite. usage: $(basename $0) __EOF__ exit 1 } gettag() { local tag tag=$(basename $1) tag=${tag#*.} tag=${tag%%[a-z.]*} echo $tag } runtest() { local dflags exe exstatus pid retval status exstatus=0 retval=0 case $TFILE in drp.DTRACEDROP_*.d|err.*.d|tst.*.d) case $TFILE in drp.DTRACEDROP_*.d) dflags="-x droptags" tag=$(gettag "$TFILE") ;; err.D_*.d) exstatus=1 dflags="-x errtags" tag=$(gettag "$TFILE") ;; err.*.d) exstatus=1 ;; esac exe=${TFILE%.*}.exe if [ -f "$exe" -a -x "$exe" ]; then ./$exe & pid=$! dflags="$dflags ${pid}" fi dtrace -C -s "${TFILE}" $dflags >$STDOUT 2>$STDERR status=$? if [ $status -ne $exstatus ]; then ERRMSG="dtrace exited with status ${status}, expected ${exstatus}" retval=1 elif [ -n "${tag}" ] && ! grep -Fq " [${tag}] " ${STDERR}; then ERRMSG="dtrace's error output did not contain expected tag ${tag}" retval=1 fi if [ -n "$pid" ]; then kill -0 $pid >/dev/null 2>&1 && kill -9 $pid >/dev/null 2>&1 wait fi ;; err.*.ksh|tst.*.ksh) expr "$TFILE" : 'err.*' >/dev/null && exstatus=1 tst=$TFILE ksh "$TFILE" /usr/sbin/dtrace >$STDOUT 2>$STDERR status=$? if [ $status -ne $exstatus ]; then ERRMSG="script exited with status ${status}, expected ${exstatus}" retval=1 fi ;; *) ERRMSG="unexpected test file name $TFILE" retval=1 ;; esac + + if [ $retval -eq 0 ] && \ + head -n 1 $STDOUT | grep -q -E '^#!/.*ksh$'; then + ksh $STDOUT + retval=$? + fi return $retval } [ $# -eq 1 ] || usage readonly STDERR=$(mktemp) readonly STDOUT=$(mktemp) readonly TFILE=$(basename $1) readonly EXOUT=${TFILE}.out kldstat -q -m dtrace_test || kldload dtrace_test cd $(dirname $1) runtest RESULT=$? if [ $RESULT -eq 0 -a -f $EXOUT -a -r $EXOUT ] && \ ! cmp $STDOUT $EXOUT >/dev/null 2>&1; then ERRMSG="test output mismatch" RESULT=1 fi if [ $RESULT -ne 0 ]; then echo "test $TFILE failed: $ERRMSG" >&2 if [ $(stat -f '%z' $STDOUT) -gt 0 ]; then cat >&2 <<__EOF__ test stdout: -- $(cat $STDOUT) -- __EOF__ fi if [ $(stat -f '%z' $STDERR) -gt 0 ]; then cat >&2 <<__EOF__ test stderr: -- $(cat $STDERR) -- __EOF__ fi fi rm -f $STDERR $STDOUT exit $RESULT Index: projects/runtime-coverage/cddl =================================================================== --- projects/runtime-coverage/cddl (revision 321584) +++ projects/runtime-coverage/cddl (revision 321585) Property changes on: projects/runtime-coverage/cddl ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/cddl:r321426-321584 Index: projects/runtime-coverage/kerberos5/lib/libkafs5/Makefile =================================================================== --- projects/runtime-coverage/kerberos5/lib/libkafs5/Makefile (revision 321584) +++ projects/runtime-coverage/kerberos5/lib/libkafs5/Makefile (revision 321585) @@ -1,39 +1,39 @@ # $FreeBSD$ LIB= kafs5 LDFLAGS= -Wl,--no-undefined LIBADD= asn1 krb5 roken INCS= kafs.h MAN= kafs5.3 VERSION_MAP= ${.CURDIR}/version.map MLINKS= kafs5.3 k_afs_cell_of_file.3 \ kafs5.3 k_hasafs.3 \ kafs5.3 k_pioctl.3 \ kafs5.3 k_setpag.3 \ kafs5.3 k_unlog.3 \ kafs5.3 kafs.3 \ kafs5.3 kafs_set_verbose.3 \ kafs5.3 kafs_settoken.3 \ kafs5.3 kafs_settoken5.3 \ kafs5.3 kafs_settoken_rxkad.3 \ kafs5.3 krb5_afslog.3 \ kafs5.3 krb5_afslog_uid.3 \ kafs5.3 krb_afslog.3 \ kafs5.3 krb_afslog_uid.3 SRCS= afssys.c afskrb5.c common.c krb5_err.h CFLAGS+= -I${KRB5DIR}/lib/kafs \ -I${KRB5DIR}/lib/krb5 \ - -I${.OBJDIR:H}/libkrb5/ \ + -I${.OBJDIR:H}/libkrb5 \ -I${KRB5DIR}/lib/roken CLEANFILES= kafs5.3 kafs5.3: kafs.3 sed -e 's/libkafs, -lkafs/libkafs5, -lkafs5/g' ${.ALLSRC} >${.TARGET} .include .PATH: ${KRB5DIR}/lib/kafs Index: projects/runtime-coverage/lib/libc/x86/sys/__vdso_gettc.c =================================================================== --- projects/runtime-coverage/lib/libc/x86/sys/__vdso_gettc.c (revision 321584) +++ projects/runtime-coverage/lib/libc/x86/sys/__vdso_gettc.c (revision 321585) @@ -1,283 +1,283 @@ /*- * Copyright (c) 2012 Konstantin Belousov * Copyright (c) 2016, 2017 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include "namespace.h" #include #include #include #include #include #include #include #include #include #include "un-namespace.h" #include #include #include #include #ifdef WANT_HYPERV #include #endif #include "libc_private.h" static void lfence_mb(void) { #if defined(__i386__) static int lfence_works = -1; u_int cpuid_supported, p[4]; if (lfence_works == -1) { __asm __volatile( " pushfl\n" " popl %%eax\n" " movl %%eax,%%ecx\n" " xorl $0x200000,%%eax\n" " pushl %%eax\n" " popfl\n" " pushfl\n" " popl %%eax\n" " xorl %%eax,%%ecx\n" " je 1f\n" " movl $1,%0\n" " jmp 2f\n" "1: movl $0,%0\n" "2:\n" : "=r" (cpuid_supported) : : "eax", "ecx", "cc"); if (cpuid_supported) { __asm __volatile( " pushl %%ebx\n" " cpuid\n" " movl %%ebx,%1\n" " popl %%ebx\n" : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3]) : "0" (0x1)); lfence_works = (p[3] & CPUID_SSE2) != 0; } else lfence_works = 0; } if (lfence_works == 1) lfence(); #elif defined(__amd64__) lfence(); #else #error "arch" #endif } static u_int __vdso_gettc_rdtsc_low(const struct vdso_timehands *th) { u_int rv; lfence_mb(); __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); return (rv); } static u_int __vdso_rdtsc32(void) { lfence_mb(); return (rdtsc32()); } #define HPET_DEV_MAP_MAX 10 static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX]; static void __vdso_init_hpet(uint32_t u) { static const char devprefix[] = "/dev/hpet"; char devname[64], *c, *c1, t; volatile char *new_map, *old_map; unsigned int mode; uint32_t u1; int fd; c1 = c = stpcpy(devname, devprefix); u1 = u; do { *c++ = u1 % 10 + '0'; u1 /= 10; } while (u1 != 0); *c = '\0'; for (c--; c1 != c; c1++, c--) { t = *c1; *c1 = *c; *c = t; } old_map = hpet_dev_map[u]; if (old_map != NULL) return; if (cap_getmode(&mode) == 0 && mode != 0) goto fail; fd = _open(devname, O_RDONLY); if (fd == -1) goto fail; new_map = mmap(NULL, PAGE_SIZE, PROT_READ, MAP_SHARED, fd, 0); _close(fd); if (atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u], (uintptr_t)old_map, (uintptr_t)new_map) == 0 && new_map != MAP_FAILED) - munmap((void *)new_map, PAGE_SIZE); + munmap((void *)new_map, PAGE_SIZE); return; fail: /* Prevent the caller from re-entering. */ atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u], (uintptr_t)old_map, (uintptr_t)MAP_FAILED); } #ifdef WANT_HYPERV #define HYPERV_REFTSC_DEVPATH "/dev/" HYPERV_REFTSC_DEVNAME /* * NOTE: * We use 'NULL' for this variable to indicate that initialization * is required. And if this variable is 'MAP_FAILED', then Hyper-V * reference TSC can not be used, e.g. in misconfigured jail. */ static struct hyperv_reftsc *hyperv_ref_tsc; static void __vdso_init_hyperv_tsc(void) { int fd; unsigned int mode; if (cap_getmode(&mode) == 0 && mode != 0) goto fail; fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY); if (fd < 0) goto fail; hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ, MAP_SHARED, fd, 0); _close(fd); return; fail: /* Prevent the caller from re-entering. */ hyperv_ref_tsc = MAP_FAILED; } static int __vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc) { uint64_t disc, ret, tsc, scale; uint32_t seq; int64_t ofs; while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { scale = tsc_ref->tsc_scale; ofs = tsc_ref->tsc_ofs; lfence_mb(); tsc = rdtsc(); /* ret = ((tsc * scale) >> 64) + ofs */ __asm__ __volatile__ ("mulq %3" : "=d" (ret), "=a" (disc) : "a" (tsc), "r" (scale)); ret += ofs; atomic_thread_fence_acq(); if (tsc_ref->tsc_seq == seq) { *tc = ret; return (0); } /* Sequence changed; re-sync. */ } return (ENOSYS); } #endif /* WANT_HYPERV */ #pragma weak __vdso_gettc int __vdso_gettc(const struct vdso_timehands *th, u_int *tc) { volatile char *map; uint32_t idx; switch (th->th_algo) { case VDSO_TH_ALGO_X86_TSC: *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) : __vdso_rdtsc32(); return (0); case VDSO_TH_ALGO_X86_HPET: idx = th->th_x86_hpet_idx; if (idx >= HPET_DEV_MAP_MAX) return (ENOSYS); map = (volatile char *)atomic_load_acq_ptr( (volatile uintptr_t *)&hpet_dev_map[idx]); if (map == NULL) { __vdso_init_hpet(idx); map = (volatile char *)atomic_load_acq_ptr( (volatile uintptr_t *)&hpet_dev_map[idx]); } if (map == MAP_FAILED) return (ENOSYS); *tc = *(volatile uint32_t *)(map + HPET_MAIN_COUNTER); return (0); #ifdef WANT_HYPERV case VDSO_TH_ALGO_X86_HVTSC: if (hyperv_ref_tsc == NULL) __vdso_init_hyperv_tsc(); if (hyperv_ref_tsc == MAP_FAILED) return (ENOSYS); return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc)); #endif default: return (ENOSYS); } } #pragma weak __vdso_gettimekeep int __vdso_gettimekeep(struct vdso_timekeep **tk) { return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); } Index: projects/runtime-coverage/lib/msun/tests/Makefile =================================================================== --- projects/runtime-coverage/lib/msun/tests/Makefile (revision 321584) +++ projects/runtime-coverage/lib/msun/tests/Makefile (revision 321585) @@ -1,104 +1,108 @@ # $FreeBSD$ .include TESTSRC= ${SRCTOP}/contrib/netbsd-tests/lib/libm # All architectures on FreeBSD have fenv.h CFLAGS+= -DHAVE_FENV_H # For isqemu.h CFLAGS+= -I${TESTSRC:H}/libc/gen # Not sure why this isn't defined for all architectures, since most # have long double. .if ${MACHINE_CPUARCH} == "aarch64" || \ ${MACHINE_CPUARCH} == "amd64" || \ ${MACHINE_CPUARCH} == "i386" CFLAGS+= -D__HAVE_LONG_DOUBLE .endif NETBSD_ATF_TESTS_C= acos_test NETBSD_ATF_TESTS_C+= asin_test NETBSD_ATF_TESTS_C+= atan_test NETBSD_ATF_TESTS_C+= cbrt_test NETBSD_ATF_TESTS_C+= ceil_test NETBSD_ATF_TESTS_C+= casinh_test NETBSD_ATF_TESTS_C+= cos_test NETBSD_ATF_TESTS_C+= cosh_test NETBSD_ATF_TESTS_C+= erf_test NETBSD_ATF_TESTS_C+= exp_test NETBSD_ATF_TESTS_C+= fmod_test NETBSD_ATF_TESTS_C+= fe_round_test NETBSD_ATF_TESTS_C+= infinity_test NETBSD_ATF_TESTS_C+= ilogb_test NETBSD_ATF_TESTS_C+= ldexp_test NETBSD_ATF_TESTS_C+= log_test NETBSD_ATF_TESTS_C+= pow_test NETBSD_ATF_TESTS_C+= precision_test NETBSD_ATF_TESTS_C+= round_test NETBSD_ATF_TESTS_C+= scalbn_test NETBSD_ATF_TESTS_C+= sin_test NETBSD_ATF_TESTS_C+= sinh_test NETBSD_ATF_TESTS_C+= sqrt_test NETBSD_ATF_TESTS_C+= tan_test NETBSD_ATF_TESTS_C+= tanh_test TAP_TESTS_C+= cexp_test TAP_TESTS_C+= conj_test .if ${MACHINE_CPUARCH} != "aarch64" # Hits an assert in llvm when building for arm64: # https://llvm.org/bugs/show_bug.cgi?id=26081 TAP_TESTS_C+= csqrt_test .endif ATF_TESTS_C+= ctrig_test TAP_TESTS_C+= exponential_test TAP_TESTS_C+= fenv_test TAP_TESTS_C+= fma_test TAP_TESTS_C+= fmaxmin_test TAP_TESTS_C+= ilogb2_test TAP_TESTS_C+= invtrig_test TAP_TESTS_C+= invctrig_test TAP_TESTS_C+= logarithm_test TAP_TESTS_C+= lrint_test # XXX: the testcase crashes on all platforms, but only on head # (bug 205451) #TAP_TESTS_C+= lround_test TAP_TESTS_C+= nan_test TAP_TESTS_C+= nearbyint_test TAP_TESTS_C+= next_test TAP_TESTS_C+= rem_test ATF_TESTS_C+= trig_test .if !empty(PROG) && !empty(TAP_TESTS_C:M${PROG}) CFLAGS+= -O0 .endif CSTD= c99 #COPTS+= -Wfloat-equal IGNORE_PRAGMA= SRCS.ilogb2_test= ilogb_test.c LIBADD+= m +.if ${MACHINE_CPUARCH} == "i386" +# XXX: __fldcw macro mismatch between fenv.h and ieeefp.h . +CWARNFLAGS.clang+= -Wno-error=macro-redefined +.endif WARNS?= 1 # Copied from lib/msun/Makefile .if ${MACHINE_CPUARCH} == "i386" ARCH_SUBDIR= i387 .else ARCH_SUBDIR= ${MACHINE_CPUARCH} .endif .include "../${ARCH_SUBDIR}/Makefile.inc" # XXX: for some odd reason float.h doesn't tell the full story about what the # precision is. CFLAGS+= -DLDBL_PREC=${LDBL_PREC} .include .include Index: projects/runtime-coverage/lib/msun/tests/trig_test.c =================================================================== --- projects/runtime-coverage/lib/msun/tests/trig_test.c (revision 321584) +++ projects/runtime-coverage/lib/msun/tests/trig_test.c (revision 321585) @@ -1,289 +1,285 @@ /*- * Copyright (c) 2008 David Schultz * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Tests for corner cases in trigonometric functions. Some accuracy tests * are included as well, but these are very basic sanity checks, not * intended to be comprehensive. * * The program for generating representable numbers near multiples of pi is * available at http://www.cs.berkeley.edu/~wkahan/testpi/ . */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "test-utils.h" #pragma STDC FENV_ACCESS ON /* * Test that a function returns the correct value and sets the * exception flags correctly. The exceptmask specifies which * exceptions we should check. We need to be lenient for several * reasons, but mainly because on some architectures it's impossible * to raise FE_OVERFLOW without raising FE_INEXACT. * * These are macros instead of functions so that assert provides more * meaningful error messages. * * XXX The volatile here is to avoid gcc's bogus constant folding and work * around the lack of support for the FENV_ACCESS pragma. */ #define test(func, x, result, exceptmask, excepts) do { \ volatile long double _d = x; \ ATF_CHECK(feclearexcept(FE_ALL_EXCEPT) == 0); \ ATF_CHECK(fpequal((func)(_d), (result))); \ ATF_CHECK(((void)(func), fetestexcept(exceptmask) == (excepts))); \ } while (0) #define testall(prefix, x, result, exceptmask, excepts) do { \ test(prefix, x, (double)result, exceptmask, excepts); \ test(prefix##f, x, (float)result, exceptmask, excepts); \ test(prefix##l, x, result, exceptmask, excepts); \ } while (0) #define testdf(prefix, x, result, exceptmask, excepts) do { \ test(prefix, x, (double)result, exceptmask, excepts); \ test(prefix##f, x, (float)result, exceptmask, excepts); \ } while (0) ATF_TC(special); ATF_TC_HEAD(special, tc) { atf_tc_set_md_var(tc, "descr", "test special cases in sin(), cos(), and tan()"); } ATF_TC_BODY(special, tc) { /* Values at 0 should be exact. */ testall(tan, 0.0, 0.0, ALL_STD_EXCEPT, 0); testall(tan, -0.0, -0.0, ALL_STD_EXCEPT, 0); testall(cos, 0.0, 1.0, ALL_STD_EXCEPT, 0); testall(cos, -0.0, 1.0, ALL_STD_EXCEPT, 0); testall(sin, 0.0, 0.0, ALL_STD_EXCEPT, 0); testall(sin, -0.0, -0.0, ALL_STD_EXCEPT, 0); /* func(+-Inf) == NaN */ testall(tan, INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); testall(sin, INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); testall(cos, INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); testall(tan, -INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); testall(sin, -INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); testall(cos, -INFINITY, NAN, ALL_STD_EXCEPT, FE_INVALID); /* func(NaN) == NaN */ testall(tan, NAN, NAN, ALL_STD_EXCEPT, 0); testall(sin, NAN, NAN, ALL_STD_EXCEPT, 0); testall(cos, NAN, NAN, ALL_STD_EXCEPT, 0); } #ifndef __i386__ ATF_TC(reduction); ATF_TC_HEAD(reduction, tc) { atf_tc_set_md_var(tc, "descr", "tests to ensure argument reduction for large arguments is accurate"); } ATF_TC_BODY(reduction, tc) { /* floats very close to odd multiples of pi */ static const float f_pi_odd[] = { 85563208.0f, 43998769152.0f, 9.2763667655669323e+25f, 1.5458357838905804e+29f, }; /* doubles very close to odd multiples of pi */ static const double d_pi_odd[] = { 3.1415926535897931, 91.106186954104004, 642615.9188844458, 3397346.5699258847, 6134899525417045.0, 3.0213551960457761e+43, 1.2646209897993783e+295, 6.2083625380677099e+307, }; /* long doubles very close to odd multiples of pi */ #if LDBL_MANT_DIG == 64 static const long double ld_pi_odd[] = { 1.1891886960373841596e+101L, 1.07999475322710967206e+2087L, 6.522151627890431836e+2147L, 8.9368974898260328229e+2484L, 9.2961044110572205863e+2555L, 4.90208421886578286e+3189L, 1.5275546401232615884e+3317L, 1.7227465626338900093e+3565L, 2.4160090594000745334e+3808L, 9.8477555741888350649e+4314L, 1.6061597222105160737e+4326L, }; -#elif LDBL_MANT_DIG == 113 - static const long double ld_pi_odd[] = { - /* XXX */ - }; #endif #if defined(__clang__) && \ ((__clang_major__ >= 5)) atf_tc_expect_fail("test fails with clang 5.0+ - bug 220989"); #endif unsigned i; for (i = 0; i < nitems(f_pi_odd); i++) { ATF_CHECK(fabs(sinf(f_pi_odd[i])) < FLT_EPSILON); ATF_CHECK(cosf(f_pi_odd[i]) == -1.0); ATF_CHECK(fabs(tan(f_pi_odd[i])) < FLT_EPSILON); ATF_CHECK(fabs(sinf(-f_pi_odd[i])) < FLT_EPSILON); ATF_CHECK(cosf(-f_pi_odd[i]) == -1.0); ATF_CHECK(fabs(tanf(-f_pi_odd[i])) < FLT_EPSILON); ATF_CHECK(fabs(sinf(f_pi_odd[i] * 2)) < FLT_EPSILON); ATF_CHECK(cosf(f_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabs(tanf(f_pi_odd[i] * 2)) < FLT_EPSILON); ATF_CHECK(fabs(sinf(-f_pi_odd[i] * 2)) < FLT_EPSILON); ATF_CHECK(cosf(-f_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabs(tanf(-f_pi_odd[i] * 2)) < FLT_EPSILON); } for (i = 0; i < nitems(d_pi_odd); i++) { ATF_CHECK(fabs(sin(d_pi_odd[i])) < 2 * DBL_EPSILON); ATF_CHECK(cos(d_pi_odd[i]) == -1.0); ATF_CHECK(fabs(tan(d_pi_odd[i])) < 2 * DBL_EPSILON); ATF_CHECK(fabs(sin(-d_pi_odd[i])) < 2 * DBL_EPSILON); ATF_CHECK(cos(-d_pi_odd[i]) == -1.0); ATF_CHECK(fabs(tan(-d_pi_odd[i])) < 2 * DBL_EPSILON); ATF_CHECK(fabs(sin(d_pi_odd[i] * 2)) < 2 * DBL_EPSILON); ATF_CHECK(cos(d_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabs(tan(d_pi_odd[i] * 2)) < 2 * DBL_EPSILON); ATF_CHECK(fabs(sin(-d_pi_odd[i] * 2)) < 2 * DBL_EPSILON); ATF_CHECK(cos(-d_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabs(tan(-d_pi_odd[i] * 2)) < 2 * DBL_EPSILON); } -#if LDBL_MANT_DIG > 53 +#if LDBL_MANT_DIG == 64 /* XXX: || LDBL_MANT_DIG == 113 */ for (i = 0; i < nitems(ld_pi_odd); i++) { ATF_CHECK(fabsl(sinl(ld_pi_odd[i])) < LDBL_EPSILON); ATF_CHECK(cosl(ld_pi_odd[i]) == -1.0); ATF_CHECK(fabsl(tanl(ld_pi_odd[i])) < LDBL_EPSILON); ATF_CHECK(fabsl(sinl(-ld_pi_odd[i])) < LDBL_EPSILON); ATF_CHECK(cosl(-ld_pi_odd[i]) == -1.0); ATF_CHECK(fabsl(tanl(-ld_pi_odd[i])) < LDBL_EPSILON); ATF_CHECK(fabsl(sinl(ld_pi_odd[i] * 2)) < LDBL_EPSILON); ATF_CHECK(cosl(ld_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabsl(tanl(ld_pi_odd[i] * 2)) < LDBL_EPSILON); ATF_CHECK(fabsl(sinl(-ld_pi_odd[i] * 2)) < LDBL_EPSILON); ATF_CHECK(cosl(-ld_pi_odd[i] * 2) == 1.0); ATF_CHECK(fabsl(tanl(-ld_pi_odd[i] * 2)) < LDBL_EPSILON); } #endif } ATF_TC(accuracy); ATF_TC_HEAD(accuracy, tc) { atf_tc_set_md_var(tc, "descr", "tests the accuracy of these functions over the primary range"); } ATF_TC_BODY(accuracy, tc) { /* For small args, sin(x) = tan(x) = x, and cos(x) = 1. */ testall(sin, 0xd.50ee515fe4aea16p-114L, 0xd.50ee515fe4aea16p-114L, ALL_STD_EXCEPT, FE_INEXACT); testall(tan, 0xd.50ee515fe4aea16p-114L, 0xd.50ee515fe4aea16p-114L, ALL_STD_EXCEPT, FE_INEXACT); testall(cos, 0xd.50ee515fe4aea16p-114L, 1.0, ALL_STD_EXCEPT, FE_INEXACT); /* * These tests should pass for f32, d64, and ld80 as long as * the error is <= 0.75 ulp (round to nearest) */ #if LDBL_MANT_DIG <= 64 #define testacc testall #else #define testacc testdf #endif testacc(sin, 0.17255452780841205174L, 0.17169949801444412683L, ALL_STD_EXCEPT, FE_INEXACT); testacc(sin, -0.75431944555904520893L, -0.68479288156557286353L, ALL_STD_EXCEPT, FE_INEXACT); testacc(cos, 0.70556358769838947292L, 0.76124620693117771850L, ALL_STD_EXCEPT, FE_INEXACT); testacc(cos, -0.34061437849088045332L, 0.94254960031831729956L, ALL_STD_EXCEPT, FE_INEXACT); testacc(tan, -0.15862817413325692897L, -0.15997221861309522115L, ALL_STD_EXCEPT, FE_INEXACT); testacc(tan, 0.38374784931303813530L, 0.40376500259976759951L, ALL_STD_EXCEPT, FE_INEXACT); /* * XXX missing: * - tests for ld128 * - tests for other rounding modes (probably won't pass for now) * - tests for large numbers that get reduced to hi+lo with lo!=0 */ } #endif ATF_TP_ADD_TCS(tp) { ATF_TP_ADD_TC(tp, special); #ifndef __i386__ ATF_TP_ADD_TC(tp, accuracy); ATF_TP_ADD_TC(tp, reduction); #endif return (atf_no_error()); } Index: projects/runtime-coverage/share/mk/bsd.dep.mk =================================================================== --- projects/runtime-coverage/share/mk/bsd.dep.mk (revision 321584) +++ projects/runtime-coverage/share/mk/bsd.dep.mk (revision 321585) @@ -1,342 +1,342 @@ # $FreeBSD$ # # The include file handles Makefile dependencies. # # # +++ variables +++ # # CLEANDEPENDDIRS Additional directories to remove for the cleandepend # target. # # CLEANDEPENDFILES Additional files to remove for the cleandepend target. # # CTAGS A tags file generation program [gtags] # # CTAGSFLAGS Options for ctags(1) [not set] # # DEPENDFILE dependencies file [.depend] # # GTAGSFLAGS Options for gtags(1) [-o] # # HTAGSFLAGS Options for htags(1) [not set] # # SRCS List of source files (c, c++, assembler) # # DPSRCS List of source files which are needed for generating # dependencies, ${SRCS} are always part of it. # # +++ targets +++ # # cleandepend: # remove ${CLEANDEPENDFILES}; remove ${CLEANDEPENDDIRS} and all # contents. # # depend: # Make the dependencies for the source files, and store # them in the file ${DEPENDFILE}. # # tags: # In "ctags" mode, create a tags file for the source files. # In "gtags" mode, create a (GLOBAL) gtags file for the # source files. If HTML is defined, htags(1) is also run # after gtags(1). .if !target(____) .error bsd.dep.mk cannot be included directly. .endif CTAGS?= gtags CTAGSFLAGS?= GTAGSFLAGS?= -o HTAGSFLAGS?= .if ${MK_DIRDEPS_BUILD} == "no" .MAKE.DEPENDFILE= ${DEPENDFILE} .endif CLEANDEPENDFILES+= ${DEPENDFILE} ${DEPENDFILE}.* .if ${MK_META_MODE} == "yes" CLEANDEPENDFILES+= *.meta .endif # Keep `tags' here, before SRCS are mangled below for `depend'. .if !target(tags) && defined(SRCS) && !defined(NO_TAGS) tags: ${SRCS} .if ${CTAGS:T} == "gtags" @cd ${.CURDIR} && ${CTAGS} ${GTAGSFLAGS} ${.OBJDIR} .if defined(HTML) @cd ${.CURDIR} && htags ${HTAGSFLAGS} -d ${.OBJDIR} ${.OBJDIR} .endif .else @${CTAGS} ${CTAGSFLAGS} -f /dev/stdout \ ${.ALLSRC:N*.h} | sed "s;${.CURDIR}/;;" > ${.TARGET} .endif .endif .if !empty(.MAKE.MODE:Mmeta) && empty(.MAKE.MODE:Mnofilemon) _meta_filemon= 1 .endif # By default META_MODE is disabled in bmake if there is no OBJDIR # unless .MAKE.MODE contains "curdirOk=[^0nNfF]" .if defined(_meta_filemon) && ${.OBJDIR} == ${.CURDIR} && \ (empty(.MAKE.MODE:tl:Mcurdirok=*) || \ !empty(.MAKE.MODE:tl:Mcurdirok=[0NnFf]*)) .undef _meta_filemon .endif # Skip reading .depend when not needed to speed up tree-walks and simple # lookups. See _SKIP_BUILD logic in bsd.init.mk for more details. # Also skip generating or including .depend.* files if in meta+filemon mode # since it will track dependencies itself. OBJS_DEPEND_GUESS is still used # for _meta_filemon but not for _SKIP_DEPEND. -.if defined(_SKIP_BUILD) +.if !defined(NO_SKIP_DEPEND) && defined(_SKIP_BUILD) _SKIP_DEPEND= 1 .endif .if ${MK_DIRDEPS_BUILD} == "no" .if defined(_SKIP_DEPEND) || defined(_meta_filemon) .MAKE.DEPENDFILE= /dev/null .endif .endif .if defined(SRCS) CLEANFILES?= .for _S in ${SRCS:N*.[dhly]} OBJS_DEPEND_GUESS.${_S:R}.o+= ${_S} .endfor # Lexical analyzers .for _LSRC in ${SRCS:M*.l:N*/*} .for _LC in ${_LSRC:R}.c ${_LC}: ${_LSRC} ${LEX} ${LFLAGS} -o${.TARGET} ${.ALLSRC} OBJS_DEPEND_GUESS.${_LC:R}.o+= ${_LC} SRCS:= ${SRCS:S/${_LSRC}/${_LC}/} CLEANFILES+= ${_LC} .endfor .endfor # Yacc grammars .for _YSRC in ${SRCS:M*.y:N*/*} .for _YC in ${_YSRC:R}.c SRCS:= ${SRCS:S/${_YSRC}/${_YC}/} CLEANFILES+= ${_YC} .if !empty(YFLAGS:M-d) && !empty(SRCS:My.tab.h) .ORDER: ${_YC} y.tab.h y.tab.h: .NOMETA ${_YC} y.tab.h: ${_YSRC} ${YACC} ${YFLAGS} ${.ALLSRC} cp y.tab.c ${_YC} CLEANFILES+= y.tab.c y.tab.h .elif !empty(YFLAGS:M-d) .for _YH in ${_YC:R}.h .ORDER: ${_YC} ${_YH} ${_YH}: .NOMETA ${_YC} ${_YH}: ${_YSRC} ${YACC} ${YFLAGS} -o ${_YC} ${.ALLSRC} SRCS+= ${_YH} CLEANFILES+= ${_YH} .endfor .else ${_YC}: ${_YSRC} ${YACC} ${YFLAGS} -o ${_YC} ${.ALLSRC} .endif OBJS_DEPEND_GUESS.${_YC:R}.o+= ${_YC} .endfor .endfor # DTrace probe definitions .if ${SRCS:M*.d} CFLAGS+= -I${.OBJDIR} .endif .for _DSRC in ${SRCS:M*.d:N*/*} .for _D in ${_DSRC:R} SRCS+= ${_D}.h ${_D}.h: ${_DSRC} ${DTRACE} ${DTRACEFLAGS} -h -s ${.ALLSRC} SRCS:= ${SRCS:S/^${_DSRC}$//} OBJS+= ${_D}.o CLEANFILES+= ${_D}.h ${_D}.o ${_D}.o: ${_DSRC} ${OBJS:S/^${_D}.o$//} @rm -f ${.TARGET} ${DTRACE} ${DTRACEFLAGS} -G -o ${.TARGET} -s ${.ALLSRC:N*.h} .if defined(LIB) CLEANFILES+= ${_D}.pico ${_D}.po ${_D}.pico: ${_DSRC} ${SOBJS:S/^${_D}.pico$//} @rm -f ${.TARGET} ${DTRACE} ${DTRACEFLAGS} -G -o ${.TARGET} -s ${.ALLSRC:N*.h} ${_D}.po: ${_DSRC} ${POBJS:S/^${_D}.po$//} @rm -f ${.TARGET} ${DTRACE} ${DTRACEFLAGS} -G -o ${.TARGET} -s ${.ALLSRC:N*.h} .endif .endfor .endfor .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP .endif # Handle OBJS=../somefile.o hacks. Just replace '/' rather than use :T to # avoid collisions. DEPEND_FILTER= C,/,_,g DEPENDSRCS= ${SRCS:M*.[cSC]} ${SRCS:M*.cxx} ${SRCS:M*.cpp} ${SRCS:M*.cc} .if !empty(DEPENDSRCS) DEPENDOBJS+= ${DEPENDSRCS:R:S,$,.o,} .endif DEPENDFILES_OBJS= ${DEPENDOBJS:O:u:${DEPEND_FILTER}:C/^/${DEPENDFILE}./} .if defined(_SKIP_DEPEND) # Don't bother statting any .meta files for .depend* ${DEPENDOBJS}: .NOMETA ${DEPENDFILE}: .NOMETA # Unset these to avoid looping/statting on them later. .undef DEPENDSRCS .undef DEPENDOBJS .undef DEPENDFILES_OBJS .endif # defined(_SKIP_DEPEND) DEPEND_CFLAGS+= -MD ${DEPEND_MP} -MF${DEPENDFILE}.${.TARGET:${DEPEND_FILTER}} DEPEND_CFLAGS+= -MT${.TARGET} .if !defined(_meta_filemon) .if !empty(DEPEND_CFLAGS) # Only add in DEPEND_CFLAGS for CFLAGS on files we expect from DEPENDOBJS # as those are the only ones we will include. DEPEND_CFLAGS_CONDITION= "${DEPENDOBJS:${DEPEND_FILTER}:M${.TARGET:${DEPEND_FILTER}}}" != "" CFLAGS+= ${${DEPEND_CFLAGS_CONDITION}:?${DEPEND_CFLAGS}:} .endif .for __depend_obj in ${DEPENDFILES_OBJS} .if ${MAKE_VERSION} < 20160220 .sinclude "${.OBJDIR}/${__depend_obj}" .else .dinclude "${.OBJDIR}/${__depend_obj}" .endif .endfor .endif # !defined(_meta_filemon) .endif # defined(SRCS) .if ${MK_DIRDEPS_BUILD} == "yes" && ${.MAKE.DEPENDFILE} != "/dev/null" # Prevent meta.autodep.mk from tracking "local dependencies". .depend: .include # If using filemon then _EXTRADEPEND is skipped since it is not needed. .if defined(_meta_filemon) # this depend: bypasses that below # the dependency helps when bootstrapping depend: beforedepend ${DPSRCS} ${SRCS} afterdepend beforedepend: afterdepend: beforedepend .endif .endif # Guess some dependencies for when no ${DEPENDFILE}.OBJ is generated yet. # For meta+filemon the .meta file is checked for since it is the dependency # file used. .for __obj in ${DEPENDOBJS:O:u} # If the obj has any '/', then replace with '_'. For meta files, this is # mimicing what bmake's meta_name() does and adding in the full path # as well to ensure that the expected meta file is read. .if ${__obj:M*/*} _meta_obj= ${.OBJDIR:C,/,_,g}_${__obj:C,/,_,g}.meta .else _meta_obj= ${__obj}.meta .endif _dep_obj= ${DEPENDFILE}.${__obj:${DEPEND_FILTER}} .if defined(_meta_filemon) _depfile= ${.OBJDIR}/${_meta_obj} .else _depfile= ${.OBJDIR}/${_dep_obj} .endif .if !exists(${_depfile}) ${__obj}: ${OBJS_DEPEND_GUESS} ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .elif defined(_meta_filemon) # For meta mode we still need to know which file to depend on to avoid # ambiguous suffix transformation rules from .PATH. Meta mode does not # use .depend files. We really only need source files, not headers since # they are typically in SRCS/beforebuild already. For target-specific # guesses do include headers though since they may not be in SRCS. ${__obj}: ${OBJS_DEPEND_GUESS:N*.h} ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .endif # !exists(${_depfile}) .endfor # Always run 'make depend' to generate dependencies early and to avoid the # need for manually running it. The dirdeps build should only do this in # sub-makes though since MAKELEVEL0 is for dirdeps calculations. .if ${MK_DIRDEPS_BUILD} == "no" || ${.MAKE.LEVEL} > 0 beforebuild: depend .endif .if !target(depend) .if defined(SRCS) depend: beforedepend ${DEPENDFILE} afterdepend # Tell bmake not to look for generated files via .PATH .NOPATH: ${DEPENDFILE} ${DEPENDFILES_OBJS} DPSRCS+= ${SRCS} # A .depend file will only be generated if there are commands in # beforedepend/_EXTRADEPEND/afterdepend The _EXTRADEPEND target is # ignored if using meta+filemon since it handles all dependencies. The other # targets are kept as they be used for generating something. The target is # kept to allow 'make depend' to generate files. ${DEPENDFILE}: ${DPSRCS} .if !defined(_SKIP_DEPEND) .if exists(${.OBJDIR}/${DEPENDFILE}) || \ ((commands(beforedepend) || \ (!defined(_meta_filemon) && commands(_EXTRADEPEND)) || \ commands(afterdepend)) && !empty(.MAKE.MODE:Mmeta)) rm -f ${DEPENDFILE} .endif .endif .if !defined(_meta_filemon) && target(_EXTRADEPEND) _EXTRADEPEND: .USE ${DEPENDFILE}: _EXTRADEPEND .endif .ORDER: ${DEPENDFILE} afterdepend .else depend: beforedepend afterdepend .endif .if !target(beforedepend) beforedepend: .else .ORDER: beforedepend ${DEPENDFILE} .ORDER: beforedepend afterdepend .endif .if !target(afterdepend) afterdepend: .endif .endif .if defined(SRCS) .if ${CTAGS:T} == "gtags" CLEANDEPENDFILES+= GPATH GRTAGS GSYMS GTAGS .if defined(HTML) CLEANDEPENDDIRS+= HTML .endif .else CLEANDEPENDFILES+= tags .endif .endif .if !target(cleandepend) cleandepend: .if !empty(CLEANDEPENDFILES) rm -f ${CLEANDEPENDFILES} .endif .if !empty(CLEANDEPENDDIRS) rm -rf ${CLEANDEPENDDIRS} .endif .endif .ORDER: cleandepend all .ORDER: cleandepend depend .if !target(checkdpadd) && (defined(DPADD) || defined(LDADD)) _LDADD_FROM_DPADD= ${DPADD:R:T:C;^lib(.*)$;-l\1;g} # Ignore -Wl,--start-group/-Wl,--end-group as it might be required in the # LDADD list due to unresolved symbols _LDADD_CANONICALIZED= ${LDADD:N:R:T:C;^lib(.*)$;-l\1;g:N-Wl,--[es]*-group} checkdpadd: .if ${_LDADD_FROM_DPADD} != ${_LDADD_CANONICALIZED} @echo ${.CURDIR} @echo "DPADD -> ${_LDADD_FROM_DPADD}" @echo "LDADD -> ${_LDADD_CANONICALIZED}" .endif .endif Index: projects/runtime-coverage/share/mk/bsd.stand.mk =================================================================== --- projects/runtime-coverage/share/mk/bsd.stand.mk (revision 321584) +++ projects/runtime-coverage/share/mk/bsd.stand.mk (revision 321585) @@ -1,26 +1,29 @@ # $FreeBSD$ # # Common definitons for programs building in the stand-alone environment # and/or using libstand. # CFLAGS+= -ffreestanding -Wformat CFLAGS+= ${CFLAGS_NO_SIMD} -D_STANDALONE .if ${MACHINE_CPUARCH} == "riscv" CFLAGS+= -mno-float .elif ${MACHINE_CPUARCH} != "aarch64" CFLAGS+= -msoft-float .endif .if ${MACHINE_CPUARCH} == "i386" CFLAGS.gcc+= -mpreferred-stack-boundary=2 .endif .if ${MACHINE_CPUARCH} == "amd64" CFLAGS+= -fPIC -mno-red-zone .endif .if ${MACHINE_CPUARCH} == "aarch64" CFLAGS+= -fPIC -mgeneral-regs-only .endif +.if ${MACHINE_CPUARCH} == "arm" +CFLAGS+= -fPIC +.endif .if ${MACHINE_CPUARCH} == "mips" CFLAGS+= -G0 -fno-pic -mno-abicalls .endif Index: projects/runtime-coverage/sys/arm/include/ucontext.h =================================================================== --- projects/runtime-coverage/sys/arm/include/ucontext.h (revision 321584) +++ projects/runtime-coverage/sys/arm/include/ucontext.h (revision 321585) @@ -1,87 +1,87 @@ /* $NetBSD: mcontext.h,v 1.4 2003/10/08 22:43:01 thorpej Exp $ */ /*- * Copyright (c) 2001, 2002 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Klaus Klein and by Jason R. Thorpe of Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_MCONTEXT_H_ #define _MACHINE_MCONTEXT_H_ /* * General register state */ #define _NGREG 17 typedef unsigned int __greg_t; typedef __greg_t __gregset_t[_NGREG]; #define _REG_R0 0 #define _REG_R1 1 #define _REG_R2 2 #define _REG_R3 3 #define _REG_R4 4 #define _REG_R5 5 #define _REG_R6 6 #define _REG_R7 7 #define _REG_R8 8 #define _REG_R9 9 #define _REG_R10 10 #define _REG_R11 11 #define _REG_R12 12 #define _REG_R13 13 #define _REG_R14 14 #define _REG_R15 15 #define _REG_CPSR 16 /* Convenience synonyms */ #define _REG_FP _REG_R11 #define _REG_SP _REG_R13 #define _REG_LR _REG_R14 #define _REG_PC _REG_R15 /* * Floating point register state */ typedef struct { __uint64_t mcv_reg[32]; __uint32_t mcv_fpscr; } mcontext_vfp_t; typedef struct { __gregset_t __gregs; /* * Originally, rest of this structure was named __fpu, 35 * 4 bytes * long, never accessed from kernel. */ - size_t mc_vfp_size; + __size_t mc_vfp_size; void *mc_vfp_ptr; unsigned int mc_spare[33]; } mcontext_t; #define UC_ #endif /* !_MACHINE_MCONTEXT_H_ */ Index: projects/runtime-coverage/sys/arm64/arm64/trap.c =================================================================== --- projects/runtime-coverage/sys/arm64/arm64/trap.c (revision 321584) +++ projects/runtime-coverage/sys/arm64/arm64/trap.c (revision 321585) @@ -1,432 +1,448 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include #endif #ifdef VFP #include #endif #ifdef KDB #include #endif #ifdef DDB #include #endif extern register_t fsu_intr_fault; /* Called from exception.S */ void do_el1h_sync(struct thread *, struct trapframe *); void do_el0_sync(struct thread *, struct trapframe *); void do_el0_error(struct trapframe *); static void print_registers(struct trapframe *frame); int (*dtrace_invop_jump_addr)(struct trapframe *); static __inline void call_trapsignal(struct thread *td, int sig, int code, void *addr) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = addr; trapsignal(td, &ksi); } int cpu_fetch_syscall_args(struct thread *td) { struct proc *p; register_t *ap; struct syscall_args *sa; int nap; nap = 8; p = td->td_proc; ap = td->td_frame->tf_x; sa = &td->td_sa; sa->code = td->td_frame->tf_x[8]; if (sa->code == SYS_syscall || sa->code == SYS___syscall) { sa->code = *ap++; nap--; } if (p->p_sysent->sv_mask) sa->code &= p->p_sysent->sv_mask; if (sa->code >= p->p_sysent->sv_size) sa->callp = &p->p_sysent->sv_table[0]; else sa->callp = &p->p_sysent->sv_table[sa->code]; sa->narg = sa->callp->sy_narg; memcpy(sa->args, ap, nap * sizeof(register_t)); if (sa->narg > nap) panic("ARM64TODO: Could we have more than 8 args?"); td->td_retval[0] = 0; td->td_retval[1] = 0; return (0); } #include "../../kern/subr_syscall.c" static void svc_handler(struct thread *td, struct trapframe *frame) { int error; if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) { error = syscallenter(td); syscallret(td, error); } else { call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr); userret(td, frame); } } static void data_abort(struct thread *td, struct trapframe *frame, uint64_t esr, uint64_t far, int lower) { struct vm_map *map; struct proc *p; struct pcb *pcb; vm_prot_t ftype; vm_offset_t va; int error, sig, ucode; /* * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive * and Store-Exclusive instruction usage restrictions", state * of the exclusive monitors after data abort exception is unknown. */ clrex(); #ifdef KDB if (kdb_active) { kdb_reenter(); return; } #endif pcb = td->td_pcb; /* * Special case for fuswintr and suswintr. These can't sleep so * handle them early on in the trap handler. */ if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) { frame->tf_elr = pcb->pcb_onfault; return; } p = td->td_proc; if (lower) map = &p->p_vmspace->vm_map; else { /* The top bit tells us which range to use */ if (far >= VM_MAXUSER_ADDRESS) { map = kernel_map; } else { map = &p->p_vmspace->vm_map; if (map == NULL) map = kernel_map; } } if (pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) return; KASSERT(td->td_md.md_spinlock_count == 0, ("data abort with spinlock held")); if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) { print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); panic("data abort in critical section or under mutex"); } va = trunc_page(far); ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; /* Fault in the page. */ error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); if (error != KERN_SUCCESS) { if (lower) { sig = SIGSEGV; if (error == KERN_PROTECTION_FAILURE) ucode = SEGV_ACCERR; else ucode = SEGV_MAPERR; call_trapsignal(td, sig, ucode, (void *)far); } else { if (td->td_intr_nesting_level == 0 && pcb->pcb_onfault != 0) { frame->tf_x[0] = error; frame->tf_elr = pcb->pcb_onfault; return; } printf("Fatal data abort:\n"); print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); #ifdef KDB if (debugger_on_panic || kdb_active) if (kdb_trap(ESR_ELx_EXCEPTION(esr), 0, frame)) return; #endif panic("vm_fault failed: %lx", frame->tf_elr); } } if (lower) userret(td, frame); } static void print_registers(struct trapframe *frame) { u_int reg; for (reg = 0; reg < nitems(frame->tf_x); reg++) { printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg, frame->tf_x[reg]); } printf(" sp: %16lx\n", frame->tf_sp); printf(" lr: %16lx\n", frame->tf_lr); printf(" elr: %16lx\n", frame->tf_elr); printf("spsr: %8x\n", frame->tf_spsr); } void do_el1h_sync(struct thread *td, struct trapframe *frame) { + struct trapframe *oframe; uint32_t exception; uint64_t esr, far; /* Read the esr register to get the exception details */ esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); #ifdef KDTRACE_HOOKS if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) return; #endif CTR4(KTR_TRAP, "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); + oframe = td->td_frame; + + switch (exception) { + case EXCP_BRK: + case EXCP_WATCHPT_EL1: + case EXCP_SOFTSTP_EL1: + break; + default: + td->td_frame = frame; + break; + } + switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: #ifdef VFP if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) { vfp_restore_state(); } else #endif { print_registers(frame); printf(" esr: %.8lx\n", esr); panic("VFP exception in the kernel"); } break; case EXCP_INSN_ABORT: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); intr_enable(); data_abort(td, frame, esr, far, 0); break; case EXCP_BRK: #ifdef KDTRACE_HOOKS if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ dtrace_invop_jump_addr != 0) { dtrace_invop_jump_addr(frame); break; } #endif /* FALLTHROUGH */ case EXCP_WATCHPT_EL1: case EXCP_SOFTSTP_EL1: #ifdef KDB - kdb_trap(exception, 0, frame); + kdb_trap(exception, 0, + (td->td_frame != NULL) ? td->td_frame : frame); #else panic("No debugger in kernel.\n"); #endif break; default: print_registers(frame); panic("Unknown kernel exception %x esr_el1 %lx\n", exception, esr); } + + td->td_frame = oframe; } /* * The attempted execution of an instruction bit pattern that has no allocated * instruction results in an exception with an unknown reason. */ static void el0_excp_unknown(struct trapframe *frame, uint64_t far) { struct thread *td; td = curthread; call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far); userret(td, frame); } void do_el0_sync(struct thread *td, struct trapframe *frame) { uint32_t exception; uint64_t esr, far; /* Check we have a sane environment when entering from userland */ KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, ("Invalid pcpu address from userland: %p (tpidr %lx)", get_pcpu(), READ_SPECIALREG(tpidr_el1))); esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); switch (exception) { case EXCP_UNKNOWN: case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); } intr_enable(); CTR4(KTR_TRAP, "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: #ifdef VFP vfp_restore_state(); #else panic("VFP exception in userland"); #endif break; case EXCP_SVC: svc_handler(td, frame); break; case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: data_abort(td, frame, esr, far, 1); break; case EXCP_UNKNOWN: el0_excp_unknown(frame, far); break; case EXCP_SP_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp); userret(td, frame); break; case EXCP_PC_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_BRK: call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_MSR: call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_SOFTSTP_EL0: td->td_frame->tf_spsr &= ~PSR_SS; td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; WRITE_SPECIALREG(MDSCR_EL1, READ_SPECIALREG(MDSCR_EL1) & ~DBG_MDSCR_SS); call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)frame->tf_elr); userret(td, frame); break; default: call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr); userret(td, frame); break; } KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Kernel VFP flags set while entering userspace")); KASSERT( td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate, ("Kernel VFP state in use when entering userspace")); } void do_el0_error(struct trapframe *frame) { panic("ARM64TODO: do_el0_error"); } Index: projects/runtime-coverage/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c =================================================================== --- projects/runtime-coverage/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c (revision 321584) +++ projects/runtime-coverage/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c (revision 321585) @@ -1,7378 +1,7388 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011, 2017 by Delphix. All rights reserved. * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2013 Martin Matuska . All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright 2013 Saso Kiselkov. All rights reserved. * Copyright (c) 2014 Integros [integros.com] * Copyright 2016 Toomas Soome */ /* * SPA: Storage Pool Allocator * * This file contains all the routines used when modifying on-disk SPA state. * This includes opening, importing, destroying, exporting a pool, and syncing a * pool. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef _KERNEL #include #include #include #endif /* _KERNEL */ #include "zfs_prop.h" #include "zfs_comutil.h" /* Check hostid on import? */ static int check_hostid = 1; /* * The interval, in seconds, at which failed configuration cache file writes * should be retried. */ static int zfs_ccw_retry_interval = 300; SYSCTL_DECL(_vfs_zfs); SYSCTL_INT(_vfs_zfs, OID_AUTO, check_hostid, CTLFLAG_RWTUN, &check_hostid, 0, "Check hostid on import?"); TUNABLE_INT("vfs.zfs.ccw_retry_interval", &zfs_ccw_retry_interval); SYSCTL_INT(_vfs_zfs, OID_AUTO, ccw_retry_interval, CTLFLAG_RW, &zfs_ccw_retry_interval, 0, "Configuration cache file write, retry after failure, interval (seconds)"); typedef enum zti_modes { ZTI_MODE_FIXED, /* value is # of threads (min 1) */ ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ ZTI_MODE_NULL, /* don't create a taskq */ ZTI_NMODES } zti_modes_t; #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } #define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } #define ZTI_N(n) ZTI_P(n, 1) #define ZTI_ONE ZTI_N(1) typedef struct zio_taskq_info { zti_modes_t zti_mode; uint_t zti_value; uint_t zti_count; } zio_taskq_info_t; static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { "issue", "issue_high", "intr", "intr_high" }; /* * This table defines the taskq settings for each ZFS I/O type. When * initializing a pool, we use this table to create an appropriately sized * taskq. Some operations are low volume and therefore have a small, static * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE * macros. Other operations process a large amount of data; the ZTI_BATCH * macro causes us to create a taskq oriented for throughput. Some operations * are so high frequency and short-lived that the taskq itself can become a a * point of lock contention. The ZTI_P(#, #) macro indicates that we need an * additional degree of parallelism specified by the number of threads per- * taskq and the number of taskqs; when dispatching an event in this case, the * particular taskq is chosen at random. * * The different taskq priorities are to handle the different contexts (issue * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that * need to be handled with minimum delay. */ const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ { ZTI_N(8), ZTI_NULL, ZTI_P(12, 8), ZTI_NULL }, /* READ */ { ZTI_BATCH, ZTI_N(5), ZTI_N(8), ZTI_N(5) }, /* WRITE */ { ZTI_P(12, 8), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ }; static sysevent_t *spa_event_create(spa_t *spa, vdev_t *vd, const char *name); static void spa_event_post(sysevent_t *ev); static void spa_sync_version(void *arg, dmu_tx_t *tx); static void spa_sync_props(void *arg, dmu_tx_t *tx); static boolean_t spa_has_active_shared_spare(spa_t *spa); static int spa_load_impl(spa_t *spa, uint64_t, nvlist_t *config, spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig, char **ereport); static void spa_vdev_resilver_done(spa_t *spa); uint_t zio_taskq_batch_pct = 75; /* 1 thread per cpu in pset */ #ifdef PSRSET_BIND id_t zio_taskq_psrset_bind = PS_NONE; #endif #ifdef SYSDC boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ uint_t zio_taskq_basedc = 80; /* base duty cycle */ #endif boolean_t spa_create_process = B_TRUE; /* no process ==> no sysdc */ extern int zfs_sync_pass_deferred_free; /* * This (illegal) pool name is used when temporarily importing a spa_t in order * to get the vdev stats associated with the imported devices. */ #define TRYIMPORT_NAME "$import" /* * ========================================================================== * SPA properties routines * ========================================================================== */ /* * Add a (source=src, propname=propval) list to an nvlist. */ static void spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, uint64_t intval, zprop_source_t src) { const char *propname = zpool_prop_to_name(prop); nvlist_t *propval; VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); if (strval != NULL) VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); else VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0); VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); nvlist_free(propval); } /* * Get property values from the spa configuration. */ static void spa_prop_get_config(spa_t *spa, nvlist_t **nvp) { vdev_t *rvd = spa->spa_root_vdev; dsl_pool_t *pool = spa->spa_dsl_pool; uint64_t size, alloc, cap, version; zprop_source_t src = ZPROP_SRC_NONE; spa_config_dirent_t *dp; metaslab_class_t *mc = spa_normal_class(spa); ASSERT(MUTEX_HELD(&spa->spa_props_lock)); if (rvd != NULL) { alloc = metaslab_class_get_alloc(spa_normal_class(spa)); size = metaslab_class_get_space(spa_normal_class(spa)); spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, size - alloc, src); spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, metaslab_class_fragmentation(mc), src); spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, metaslab_class_expandable_space(mc), src); spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, (spa_mode(spa) == FREAD), src); cap = (size == 0) ? 0 : (alloc * 100 / size); spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, ddt_get_pool_dedup_ratio(spa), src); spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, rvd->vdev_state, src); version = spa_version(spa); if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) src = ZPROP_SRC_DEFAULT; else src = ZPROP_SRC_LOCAL; spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, version, src); } if (pool != NULL) { /* * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, * when opening pools before this version freedir will be NULL. */ if (pool->dp_free_dir != NULL) { spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, src); } else { spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 0, src); } if (pool->dp_leak_dir != NULL) { spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, src); } else { spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 0, src); } } spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); if (spa->spa_comment != NULL) { spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 0, ZPROP_SRC_LOCAL); } if (spa->spa_root != NULL) spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 0, ZPROP_SRC_LOCAL); if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); } else { spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); } if ((dp = list_head(&spa->spa_config_list)) != NULL) { if (dp->scd_path == NULL) { spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, "none", 0, ZPROP_SRC_LOCAL); } else if (strcmp(dp->scd_path, spa_config_path) != 0) { spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, dp->scd_path, 0, ZPROP_SRC_LOCAL); } } } /* * Get zpool property values. */ int spa_prop_get(spa_t *spa, nvlist_t **nvp) { objset_t *mos = spa->spa_meta_objset; zap_cursor_t zc; zap_attribute_t za; int err; VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0); mutex_enter(&spa->spa_props_lock); /* * Get properties from the spa config. */ spa_prop_get_config(spa, nvp); /* If no pool property object, no more prop to get. */ if (mos == NULL || spa->spa_pool_props_object == 0) { mutex_exit(&spa->spa_props_lock); return (0); } /* * Get properties from the MOS pool property object. */ for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); (err = zap_cursor_retrieve(&zc, &za)) == 0; zap_cursor_advance(&zc)) { uint64_t intval = 0; char *strval = NULL; zprop_source_t src = ZPROP_SRC_DEFAULT; zpool_prop_t prop; if ((prop = zpool_name_to_prop(za.za_name)) == ZPROP_INVAL) continue; switch (za.za_integer_length) { case 8: /* integer property */ if (za.za_first_integer != zpool_prop_default_numeric(prop)) src = ZPROP_SRC_LOCAL; if (prop == ZPOOL_PROP_BOOTFS) { dsl_pool_t *dp; dsl_dataset_t *ds = NULL; dp = spa_get_dsl(spa); dsl_pool_config_enter(dp, FTAG); if (err = dsl_dataset_hold_obj(dp, za.za_first_integer, FTAG, &ds)) { dsl_pool_config_exit(dp, FTAG); break; } strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); dsl_dataset_name(ds, strval); dsl_dataset_rele(ds, FTAG); dsl_pool_config_exit(dp, FTAG); } else { strval = NULL; intval = za.za_first_integer; } spa_prop_add_list(*nvp, prop, strval, intval, src); if (strval != NULL) kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); break; case 1: /* string property */ strval = kmem_alloc(za.za_num_integers, KM_SLEEP); err = zap_lookup(mos, spa->spa_pool_props_object, za.za_name, 1, za.za_num_integers, strval); if (err) { kmem_free(strval, za.za_num_integers); break; } spa_prop_add_list(*nvp, prop, strval, 0, src); kmem_free(strval, za.za_num_integers); break; default: break; } } zap_cursor_fini(&zc); mutex_exit(&spa->spa_props_lock); out: if (err && err != ENOENT) { nvlist_free(*nvp); *nvp = NULL; return (err); } return (0); } /* * Validate the given pool properties nvlist and modify the list * for the property values to be set. */ static int spa_prop_validate(spa_t *spa, nvlist_t *props) { nvpair_t *elem; int error = 0, reset_bootfs = 0; uint64_t objnum = 0; boolean_t has_feature = B_FALSE; elem = NULL; while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { uint64_t intval; char *strval, *slash, *check, *fname; const char *propname = nvpair_name(elem); zpool_prop_t prop = zpool_name_to_prop(propname); switch (prop) { case ZPROP_INVAL: if (!zpool_prop_feature(propname)) { error = SET_ERROR(EINVAL); break; } /* * Sanitize the input. */ if (nvpair_type(elem) != DATA_TYPE_UINT64) { error = SET_ERROR(EINVAL); break; } if (nvpair_value_uint64(elem, &intval) != 0) { error = SET_ERROR(EINVAL); break; } if (intval != 0) { error = SET_ERROR(EINVAL); break; } fname = strchr(propname, '@') + 1; if (zfeature_lookup_name(fname, NULL) != 0) { error = SET_ERROR(EINVAL); break; } has_feature = B_TRUE; break; case ZPOOL_PROP_VERSION: error = nvpair_value_uint64(elem, &intval); if (!error && (intval < spa_version(spa) || intval > SPA_VERSION_BEFORE_FEATURES || has_feature)) error = SET_ERROR(EINVAL); break; case ZPOOL_PROP_DELEGATION: case ZPOOL_PROP_AUTOREPLACE: case ZPOOL_PROP_LISTSNAPS: case ZPOOL_PROP_AUTOEXPAND: error = nvpair_value_uint64(elem, &intval); if (!error && intval > 1) error = SET_ERROR(EINVAL); break; case ZPOOL_PROP_BOOTFS: /* * If the pool version is less than SPA_VERSION_BOOTFS, * or the pool is still being created (version == 0), * the bootfs property cannot be set. */ if (spa_version(spa) < SPA_VERSION_BOOTFS) { error = SET_ERROR(ENOTSUP); break; } /* * Make sure the vdev config is bootable */ if (!vdev_is_bootable(spa->spa_root_vdev)) { error = SET_ERROR(ENOTSUP); break; } reset_bootfs = 1; error = nvpair_value_string(elem, &strval); if (!error) { objset_t *os; uint64_t propval; if (strval == NULL || strval[0] == '\0') { objnum = zpool_prop_default_numeric( ZPOOL_PROP_BOOTFS); break; } if (error = dmu_objset_hold(strval, FTAG, &os)) break; /* * Must be ZPL, and its property settings * must be supported by GRUB (compression * is not gzip, and large blocks are not used). */ if (dmu_objset_type(os) != DMU_OST_ZFS) { error = SET_ERROR(ENOTSUP); } else if ((error = dsl_prop_get_int_ds(dmu_objset_ds(os), zfs_prop_to_name(ZFS_PROP_COMPRESSION), &propval)) == 0 && !BOOTFS_COMPRESS_VALID(propval)) { error = SET_ERROR(ENOTSUP); } else { objnum = dmu_objset_id(os); } dmu_objset_rele(os, FTAG); } break; case ZPOOL_PROP_FAILUREMODE: error = nvpair_value_uint64(elem, &intval); if (!error && (intval < ZIO_FAILURE_MODE_WAIT || intval > ZIO_FAILURE_MODE_PANIC)) error = SET_ERROR(EINVAL); /* * This is a special case which only occurs when * the pool has completely failed. This allows * the user to change the in-core failmode property * without syncing it out to disk (I/Os might * currently be blocked). We do this by returning * EIO to the caller (spa_prop_set) to trick it * into thinking we encountered a property validation * error. */ if (!error && spa_suspended(spa)) { spa->spa_failmode = intval; error = SET_ERROR(EIO); } break; case ZPOOL_PROP_CACHEFILE: if ((error = nvpair_value_string(elem, &strval)) != 0) break; if (strval[0] == '\0') break; if (strcmp(strval, "none") == 0) break; if (strval[0] != '/') { error = SET_ERROR(EINVAL); break; } slash = strrchr(strval, '/'); ASSERT(slash != NULL); if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || strcmp(slash, "/..") == 0) error = SET_ERROR(EINVAL); break; case ZPOOL_PROP_COMMENT: if ((error = nvpair_value_string(elem, &strval)) != 0) break; for (check = strval; *check != '\0'; check++) { /* * The kernel doesn't have an easy isprint() * check. For this kernel check, we merely * check ASCII apart from DEL. Fix this if * there is an easy-to-use kernel isprint(). */ if (*check >= 0x7f) { error = SET_ERROR(EINVAL); break; } } if (strlen(strval) > ZPROP_MAX_COMMENT) error = E2BIG; break; case ZPOOL_PROP_DEDUPDITTO: if (spa_version(spa) < SPA_VERSION_DEDUP) error = SET_ERROR(ENOTSUP); else error = nvpair_value_uint64(elem, &intval); if (error == 0 && intval != 0 && intval < ZIO_DEDUPDITTO_MIN) error = SET_ERROR(EINVAL); break; } if (error) break; } if (!error && reset_bootfs) { error = nvlist_remove(props, zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); if (!error) { error = nvlist_add_uint64(props, zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); } } return (error); } void spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) { char *cachefile; spa_config_dirent_t *dp; if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), &cachefile) != 0) return; dp = kmem_alloc(sizeof (spa_config_dirent_t), KM_SLEEP); if (cachefile[0] == '\0') dp->scd_path = spa_strdup(spa_config_path); else if (strcmp(cachefile, "none") == 0) dp->scd_path = NULL; else dp->scd_path = spa_strdup(cachefile); list_insert_head(&spa->spa_config_list, dp); if (need_sync) spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); } int spa_prop_set(spa_t *spa, nvlist_t *nvp) { int error; nvpair_t *elem = NULL; boolean_t need_sync = B_FALSE; if ((error = spa_prop_validate(spa, nvp)) != 0) return (error); while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); if (prop == ZPOOL_PROP_CACHEFILE || prop == ZPOOL_PROP_ALTROOT || prop == ZPOOL_PROP_READONLY) continue; if (prop == ZPOOL_PROP_VERSION || prop == ZPROP_INVAL) { uint64_t ver; if (prop == ZPOOL_PROP_VERSION) { VERIFY(nvpair_value_uint64(elem, &ver) == 0); } else { ASSERT(zpool_prop_feature(nvpair_name(elem))); ver = SPA_VERSION_FEATURES; need_sync = B_TRUE; } /* Save time if the version is already set. */ if (ver == spa_version(spa)) continue; /* * In addition to the pool directory object, we might * create the pool properties object, the features for * read object, the features for write object, or the * feature descriptions object. */ error = dsl_sync_task(spa->spa_name, NULL, spa_sync_version, &ver, 6, ZFS_SPACE_CHECK_RESERVED); if (error) return (error); continue; } need_sync = B_TRUE; break; } if (need_sync) { return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, nvp, 6, ZFS_SPACE_CHECK_RESERVED)); } return (0); } /* * If the bootfs property value is dsobj, clear it. */ void spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) { if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { VERIFY(zap_remove(spa->spa_meta_objset, spa->spa_pool_props_object, zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); spa->spa_bootfs = 0; } } /*ARGSUSED*/ static int spa_change_guid_check(void *arg, dmu_tx_t *tx) { uint64_t *newguid = arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; vdev_t *rvd = spa->spa_root_vdev; uint64_t vdev_state; spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); vdev_state = rvd->vdev_state; spa_config_exit(spa, SCL_STATE, FTAG); if (vdev_state != VDEV_STATE_HEALTHY) return (SET_ERROR(ENXIO)); ASSERT3U(spa_guid(spa), !=, *newguid); return (0); } static void spa_change_guid_sync(void *arg, dmu_tx_t *tx) { uint64_t *newguid = arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; uint64_t oldguid; vdev_t *rvd = spa->spa_root_vdev; oldguid = spa_guid(spa); spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); rvd->vdev_guid = *newguid; rvd->vdev_guid_sum += (*newguid - oldguid); vdev_config_dirty(rvd); spa_config_exit(spa, SCL_STATE, FTAG); spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", oldguid, *newguid); } /* * Change the GUID for the pool. This is done so that we can later * re-import a pool built from a clone of our own vdevs. We will modify * the root vdev's guid, our own pool guid, and then mark all of our * vdevs dirty. Note that we must make sure that all our vdevs are * online when we do this, or else any vdevs that weren't present * would be orphaned from our pool. We are also going to issue a * sysevent to update any watchers. */ int spa_change_guid(spa_t *spa) { int error; uint64_t guid; mutex_enter(&spa->spa_vdev_top_lock); mutex_enter(&spa_namespace_lock); guid = spa_generate_guid(NULL); error = dsl_sync_task(spa->spa_name, spa_change_guid_check, spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); if (error == 0) { spa_config_sync(spa, B_FALSE, B_TRUE); spa_event_notify(spa, NULL, ESC_ZFS_POOL_REGUID); } mutex_exit(&spa_namespace_lock); mutex_exit(&spa->spa_vdev_top_lock); return (error); } /* * ========================================================================== * SPA state manipulation (open/create/destroy/import/export) * ========================================================================== */ static int spa_error_entry_compare(const void *a, const void *b) { spa_error_entry_t *sa = (spa_error_entry_t *)a; spa_error_entry_t *sb = (spa_error_entry_t *)b; int ret; ret = bcmp(&sa->se_bookmark, &sb->se_bookmark, sizeof (zbookmark_phys_t)); if (ret < 0) return (-1); else if (ret > 0) return (1); else return (0); } /* * Utility function which retrieves copies of the current logs and * re-initializes them in the process. */ void spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) { ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t)); bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t)); avl_create(&spa->spa_errlist_scrub, spa_error_entry_compare, sizeof (spa_error_entry_t), offsetof(spa_error_entry_t, se_avl)); avl_create(&spa->spa_errlist_last, spa_error_entry_compare, sizeof (spa_error_entry_t), offsetof(spa_error_entry_t, se_avl)); } static void spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) { const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; enum zti_modes mode = ztip->zti_mode; uint_t value = ztip->zti_value; uint_t count = ztip->zti_count; spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; char name[32]; uint_t flags = 0; boolean_t batch = B_FALSE; if (mode == ZTI_MODE_NULL) { tqs->stqs_count = 0; tqs->stqs_taskq = NULL; return; } ASSERT3U(count, >, 0); tqs->stqs_count = count; tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); switch (mode) { case ZTI_MODE_FIXED: ASSERT3U(value, >=, 1); value = MAX(value, 1); break; case ZTI_MODE_BATCH: batch = B_TRUE; flags |= TASKQ_THREADS_CPU_PCT; value = zio_taskq_batch_pct; break; default: panic("unrecognized mode for %s_%s taskq (%u:%u) in " "spa_activate()", zio_type_name[t], zio_taskq_types[q], mode, value); break; } for (uint_t i = 0; i < count; i++) { taskq_t *tq; if (count > 1) { (void) snprintf(name, sizeof (name), "%s_%s_%u", zio_type_name[t], zio_taskq_types[q], i); } else { (void) snprintf(name, sizeof (name), "%s_%s", zio_type_name[t], zio_taskq_types[q]); } #ifdef SYSDC if (zio_taskq_sysdc && spa->spa_proc != &p0) { if (batch) flags |= TASKQ_DC_BATCH; tq = taskq_create_sysdc(name, value, 50, INT_MAX, spa->spa_proc, zio_taskq_basedc, flags); } else { #endif pri_t pri = maxclsyspri; /* * The write issue taskq can be extremely CPU * intensive. Run it at slightly lower priority * than the other taskqs. * FreeBSD notes: * - numerically higher priorities are lower priorities; * - if priorities divided by four (RQ_PPQ) are equal * then a difference between them is insignificant. */ if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) #ifdef illumos pri--; #else pri += 4; #endif tq = taskq_create_proc(name, value, pri, 50, INT_MAX, spa->spa_proc, flags); #ifdef SYSDC } #endif tqs->stqs_taskq[i] = tq; } } static void spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) { spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; if (tqs->stqs_taskq == NULL) { ASSERT0(tqs->stqs_count); return; } for (uint_t i = 0; i < tqs->stqs_count; i++) { ASSERT3P(tqs->stqs_taskq[i], !=, NULL); taskq_destroy(tqs->stqs_taskq[i]); } kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); tqs->stqs_taskq = NULL; } /* * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. * Note that a type may have multiple discrete taskqs to avoid lock contention * on the taskq itself. In that case we choose which taskq at random by using * the low bits of gethrtime(). */ void spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) { spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; taskq_t *tq; ASSERT3P(tqs->stqs_taskq, !=, NULL); ASSERT3U(tqs->stqs_count, !=, 0); if (tqs->stqs_count == 1) { tq = tqs->stqs_taskq[0]; } else { #ifdef _KERNEL tq = tqs->stqs_taskq[cpu_ticks() % tqs->stqs_count]; #else tq = tqs->stqs_taskq[gethrtime() % tqs->stqs_count]; #endif } taskq_dispatch_ent(tq, func, arg, flags, ent); } static void spa_create_zio_taskqs(spa_t *spa) { for (int t = 0; t < ZIO_TYPES; t++) { for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { spa_taskqs_init(spa, t, q); } } } #ifdef _KERNEL #ifdef SPA_PROCESS static void spa_thread(void *arg) { callb_cpr_t cprinfo; spa_t *spa = arg; user_t *pu = PTOU(curproc); CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, spa->spa_name); ASSERT(curproc != &p0); (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), "zpool-%s", spa->spa_name); (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); #ifdef PSRSET_BIND /* bind this thread to the requested psrset */ if (zio_taskq_psrset_bind != PS_NONE) { pool_lock(); mutex_enter(&cpu_lock); mutex_enter(&pidlock); mutex_enter(&curproc->p_lock); if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 0, NULL, NULL) == 0) { curthread->t_bind_pset = zio_taskq_psrset_bind; } else { cmn_err(CE_WARN, "Couldn't bind process for zfs pool \"%s\" to " "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); } mutex_exit(&curproc->p_lock); mutex_exit(&pidlock); mutex_exit(&cpu_lock); pool_unlock(); } #endif #ifdef SYSDC if (zio_taskq_sysdc) { sysdc_thread_enter(curthread, 100, 0); } #endif spa->spa_proc = curproc; spa->spa_did = curthread->t_did; spa_create_zio_taskqs(spa); mutex_enter(&spa->spa_proc_lock); ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); spa->spa_proc_state = SPA_PROC_ACTIVE; cv_broadcast(&spa->spa_proc_cv); CALLB_CPR_SAFE_BEGIN(&cprinfo); while (spa->spa_proc_state == SPA_PROC_ACTIVE) cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); spa->spa_proc_state = SPA_PROC_GONE; spa->spa_proc = &p0; cv_broadcast(&spa->spa_proc_cv); CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ mutex_enter(&curproc->p_lock); lwp_exit(); } #endif /* SPA_PROCESS */ #endif /* * Activate an uninitialized pool. */ static void spa_activate(spa_t *spa, int mode) { ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); spa->spa_state = POOL_STATE_ACTIVE; spa->spa_mode = mode; spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops); spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops); /* Try to create a covering process */ mutex_enter(&spa->spa_proc_lock); ASSERT(spa->spa_proc_state == SPA_PROC_NONE); ASSERT(spa->spa_proc == &p0); spa->spa_did = 0; #ifdef SPA_PROCESS /* Only create a process if we're going to be around a while. */ if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, NULL, 0) == 0) { spa->spa_proc_state = SPA_PROC_CREATED; while (spa->spa_proc_state == SPA_PROC_CREATED) { cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); } ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); ASSERT(spa->spa_proc != &p0); ASSERT(spa->spa_did != 0); } else { #ifdef _KERNEL cmn_err(CE_WARN, "Couldn't create process for zfs pool \"%s\"\n", spa->spa_name); #endif } } #endif /* SPA_PROCESS */ mutex_exit(&spa->spa_proc_lock); /* If we didn't create a process, we need to create our taskqs. */ ASSERT(spa->spa_proc == &p0); if (spa->spa_proc == &p0) { spa_create_zio_taskqs(spa); } /* * Start TRIM thread. */ trim_thread_create(spa); list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), offsetof(vdev_t, vdev_config_dirty_node)); list_create(&spa->spa_evicting_os_list, sizeof (objset_t), offsetof(objset_t, os_evicting_node)); list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), offsetof(vdev_t, vdev_state_dirty_node)); txg_list_create(&spa->spa_vdev_txg_list, spa, offsetof(struct vdev, vdev_txg_node)); avl_create(&spa->spa_errlist_scrub, spa_error_entry_compare, sizeof (spa_error_entry_t), offsetof(spa_error_entry_t, se_avl)); avl_create(&spa->spa_errlist_last, spa_error_entry_compare, sizeof (spa_error_entry_t), offsetof(spa_error_entry_t, se_avl)); } /* * Opposite of spa_activate(). */ static void spa_deactivate(spa_t *spa) { ASSERT(spa->spa_sync_on == B_FALSE); ASSERT(spa->spa_dsl_pool == NULL); ASSERT(spa->spa_root_vdev == NULL); ASSERT(spa->spa_async_zio_root == NULL); ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); /* * Stop TRIM thread in case spa_unload() wasn't called directly * before spa_deactivate(). */ trim_thread_destroy(spa); spa_evicting_os_wait(spa); txg_list_destroy(&spa->spa_vdev_txg_list); list_destroy(&spa->spa_config_dirty_list); list_destroy(&spa->spa_evicting_os_list); list_destroy(&spa->spa_state_dirty_list); for (int t = 0; t < ZIO_TYPES; t++) { for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { spa_taskqs_fini(spa, t, q); } } metaslab_class_destroy(spa->spa_normal_class); spa->spa_normal_class = NULL; metaslab_class_destroy(spa->spa_log_class); spa->spa_log_class = NULL; /* * If this was part of an import or the open otherwise failed, we may * still have errors left in the queues. Empty them just in case. */ spa_errlog_drain(spa); avl_destroy(&spa->spa_errlist_scrub); avl_destroy(&spa->spa_errlist_last); spa->spa_state = POOL_STATE_UNINITIALIZED; mutex_enter(&spa->spa_proc_lock); if (spa->spa_proc_state != SPA_PROC_NONE) { ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); spa->spa_proc_state = SPA_PROC_DEACTIVATE; cv_broadcast(&spa->spa_proc_cv); while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { ASSERT(spa->spa_proc != &p0); cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); } ASSERT(spa->spa_proc_state == SPA_PROC_GONE); spa->spa_proc_state = SPA_PROC_NONE; } ASSERT(spa->spa_proc == &p0); mutex_exit(&spa->spa_proc_lock); #ifdef SPA_PROCESS /* * We want to make sure spa_thread() has actually exited the ZFS * module, so that the module can't be unloaded out from underneath * it. */ if (spa->spa_did != 0) { thread_join(spa->spa_did); spa->spa_did = 0; } #endif /* SPA_PROCESS */ } /* * Verify a pool configuration, and construct the vdev tree appropriately. This * will create all the necessary vdevs in the appropriate layout, with each vdev * in the CLOSED state. This will prep the pool before open/creation/import. * All vdev validation is done by the vdev_alloc() routine. */ static int spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id, int atype) { nvlist_t **child; uint_t children; int error; if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) return (error); if ((*vdp)->vdev_ops->vdev_op_leaf) return (0); error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child, &children); if (error == ENOENT) return (0); if (error) { vdev_free(*vdp); *vdp = NULL; return (SET_ERROR(EINVAL)); } for (int c = 0; c < children; c++) { vdev_t *vd; if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, atype)) != 0) { vdev_free(*vdp); *vdp = NULL; return (error); } } ASSERT(*vdp != NULL); return (0); } /* * Opposite of spa_load(). */ static void spa_unload(spa_t *spa) { int i; ASSERT(MUTEX_HELD(&spa_namespace_lock)); /* * Stop TRIM thread. */ trim_thread_destroy(spa); /* * Stop async tasks. */ spa_async_suspend(spa); /* * Stop syncing. */ if (spa->spa_sync_on) { txg_sync_stop(spa->spa_dsl_pool); spa->spa_sync_on = B_FALSE; } /* * Even though vdev_free() also calls vdev_metaslab_fini, we need * to call it earlier, before we wait for async i/o to complete. * This ensures that there is no async metaslab prefetching, by * calling taskq_wait(mg_taskq). */ if (spa->spa_root_vdev != NULL) { spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) vdev_metaslab_fini(spa->spa_root_vdev->vdev_child[c]); spa_config_exit(spa, SCL_ALL, FTAG); } /* * Wait for any outstanding async I/O to complete. */ if (spa->spa_async_zio_root != NULL) { for (int i = 0; i < max_ncpus; i++) (void) zio_wait(spa->spa_async_zio_root[i]); kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); spa->spa_async_zio_root = NULL; } bpobj_close(&spa->spa_deferred_bpobj); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); /* * Close all vdevs. */ if (spa->spa_root_vdev) vdev_free(spa->spa_root_vdev); ASSERT(spa->spa_root_vdev == NULL); /* * Close the dsl pool. */ if (spa->spa_dsl_pool) { dsl_pool_close(spa->spa_dsl_pool); spa->spa_dsl_pool = NULL; spa->spa_meta_objset = NULL; } ddt_unload(spa); /* * Drop and purge level 2 cache */ spa_l2cache_drop(spa); for (i = 0; i < spa->spa_spares.sav_count; i++) vdev_free(spa->spa_spares.sav_vdevs[i]); if (spa->spa_spares.sav_vdevs) { kmem_free(spa->spa_spares.sav_vdevs, spa->spa_spares.sav_count * sizeof (void *)); spa->spa_spares.sav_vdevs = NULL; } if (spa->spa_spares.sav_config) { nvlist_free(spa->spa_spares.sav_config); spa->spa_spares.sav_config = NULL; } spa->spa_spares.sav_count = 0; for (i = 0; i < spa->spa_l2cache.sav_count; i++) { vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); vdev_free(spa->spa_l2cache.sav_vdevs[i]); } if (spa->spa_l2cache.sav_vdevs) { kmem_free(spa->spa_l2cache.sav_vdevs, spa->spa_l2cache.sav_count * sizeof (void *)); spa->spa_l2cache.sav_vdevs = NULL; } if (spa->spa_l2cache.sav_config) { nvlist_free(spa->spa_l2cache.sav_config); spa->spa_l2cache.sav_config = NULL; } spa->spa_l2cache.sav_count = 0; spa->spa_async_suspended = 0; if (spa->spa_comment != NULL) { spa_strfree(spa->spa_comment); spa->spa_comment = NULL; } spa_config_exit(spa, SCL_ALL, FTAG); } /* * Load (or re-load) the current list of vdevs describing the active spares for * this pool. When this is called, we have some form of basic information in * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and * then re-generate a more complete list including status information. */ static void spa_load_spares(spa_t *spa) { nvlist_t **spares; uint_t nspares; int i; vdev_t *vd, *tvd; ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); /* * First, close and free any existing spare vdevs. */ for (i = 0; i < spa->spa_spares.sav_count; i++) { vd = spa->spa_spares.sav_vdevs[i]; /* Undo the call to spa_activate() below */ if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, B_FALSE)) != NULL && tvd->vdev_isspare) spa_spare_remove(tvd); vdev_close(vd); vdev_free(vd); } if (spa->spa_spares.sav_vdevs) kmem_free(spa->spa_spares.sav_vdevs, spa->spa_spares.sav_count * sizeof (void *)); if (spa->spa_spares.sav_config == NULL) nspares = 0; else VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); spa->spa_spares.sav_count = (int)nspares; spa->spa_spares.sav_vdevs = NULL; if (nspares == 0) return; /* * Construct the array of vdevs, opening them to get status in the * process. For each spare, there is potentially two different vdev_t * structures associated with it: one in the list of spares (used only * for basic validation purposes) and one in the active vdev * configuration (if it's spared in). During this phase we open and * validate each vdev on the spare list. If the vdev also exists in the * active configuration, then we also mark this vdev as an active spare. */ spa->spa_spares.sav_vdevs = kmem_alloc(nspares * sizeof (void *), KM_SLEEP); for (i = 0; i < spa->spa_spares.sav_count; i++) { VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, VDEV_ALLOC_SPARE) == 0); ASSERT(vd != NULL); spa->spa_spares.sav_vdevs[i] = vd; if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, B_FALSE)) != NULL) { if (!tvd->vdev_isspare) spa_spare_add(tvd); /* * We only mark the spare active if we were successfully * able to load the vdev. Otherwise, importing a pool * with a bad active spare would result in strange * behavior, because multiple pool would think the spare * is actively in use. * * There is a vulnerability here to an equally bizarre * circumstance, where a dead active spare is later * brought back to life (onlined or otherwise). Given * the rarity of this scenario, and the extra complexity * it adds, we ignore the possibility. */ if (!vdev_is_dead(tvd)) spa_spare_activate(tvd); } vd->vdev_top = vd; vd->vdev_aux = &spa->spa_spares; if (vdev_open(vd) != 0) continue; if (vdev_validate_aux(vd) == 0) spa_spare_add(vd); } /* * Recompute the stashed list of spares, with status information * this time. */ VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), KM_SLEEP); for (i = 0; i < spa->spa_spares.sav_count; i++) spares[i] = vdev_config_generate(spa, spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0); for (i = 0; i < spa->spa_spares.sav_count; i++) nvlist_free(spares[i]); kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); } /* * Load (or re-load) the current list of vdevs describing the active l2cache for * this pool. When this is called, we have some form of basic information in * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and * then re-generate a more complete list including status information. * Devices which are already active have their details maintained, and are * not re-opened. */ static void spa_load_l2cache(spa_t *spa) { nvlist_t **l2cache; uint_t nl2cache; int i, j, oldnvdevs; uint64_t guid; vdev_t *vd, **oldvdevs, **newvdevs; spa_aux_vdev_t *sav = &spa->spa_l2cache; ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); if (sav->sav_config != NULL) { VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); } else { nl2cache = 0; newvdevs = NULL; } oldvdevs = sav->sav_vdevs; oldnvdevs = sav->sav_count; sav->sav_vdevs = NULL; sav->sav_count = 0; /* * Process new nvlist of vdevs. */ for (i = 0; i < nl2cache; i++) { VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID, &guid) == 0); newvdevs[i] = NULL; for (j = 0; j < oldnvdevs; j++) { vd = oldvdevs[j]; if (vd != NULL && guid == vd->vdev_guid) { /* * Retain previous vdev for add/remove ops. */ newvdevs[i] = vd; oldvdevs[j] = NULL; break; } } if (newvdevs[i] == NULL) { /* * Create new vdev */ VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, VDEV_ALLOC_L2CACHE) == 0); ASSERT(vd != NULL); newvdevs[i] = vd; /* * Commit this vdev as an l2cache device, * even if it fails to open. */ spa_l2cache_add(vd); vd->vdev_top = vd; vd->vdev_aux = sav; spa_l2cache_activate(vd); if (vdev_open(vd) != 0) continue; (void) vdev_validate_aux(vd); if (!vdev_is_dead(vd)) l2arc_add_vdev(spa, vd); } } /* * Purge vdevs that were dropped */ for (i = 0; i < oldnvdevs; i++) { uint64_t pool; vd = oldvdevs[i]; if (vd != NULL) { ASSERT(vd->vdev_isl2cache); if (spa_l2cache_exists(vd->vdev_guid, &pool) && pool != 0ULL && l2arc_vdev_present(vd)) l2arc_remove_vdev(vd); vdev_clear_stats(vd); vdev_free(vd); } } if (oldvdevs) kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); if (sav->sav_config == NULL) goto out; sav->sav_vdevs = newvdevs; sav->sav_count = (int)nl2cache; /* * Recompute the stashed list of l2cache devices, with status * information this time. */ VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); for (i = 0; i < sav->sav_count; i++) l2cache[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); VERIFY(nvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0); out: for (i = 0; i < sav->sav_count; i++) nvlist_free(l2cache[i]); if (sav->sav_count) kmem_free(l2cache, sav->sav_count * sizeof (void *)); } static int load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) { dmu_buf_t *db; char *packed = NULL; size_t nvsize = 0; int error; *value = NULL; error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); if (error != 0) return (error); nvsize = *(uint64_t *)db->db_data; dmu_buf_rele(db, FTAG); packed = kmem_alloc(nvsize, KM_SLEEP); error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, DMU_READ_PREFETCH); if (error == 0) error = nvlist_unpack(packed, nvsize, value, 0); kmem_free(packed, nvsize); return (error); } /* * Checks to see if the given vdev could not be opened, in which case we post a * sysevent to notify the autoreplace code that the device has been removed. */ static void spa_check_removed(vdev_t *vd) { for (int c = 0; c < vd->vdev_children; c++) spa_check_removed(vd->vdev_child[c]); if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && !vd->vdev_ishole) { zfs_post_autoreplace(vd->vdev_spa, vd); spa_event_notify(vd->vdev_spa, vd, ESC_ZFS_VDEV_CHECK); } } static void spa_config_valid_zaps(vdev_t *vd, vdev_t *mvd) { ASSERT3U(vd->vdev_children, ==, mvd->vdev_children); vd->vdev_top_zap = mvd->vdev_top_zap; vd->vdev_leaf_zap = mvd->vdev_leaf_zap; for (uint64_t i = 0; i < vd->vdev_children; i++) { spa_config_valid_zaps(vd->vdev_child[i], mvd->vdev_child[i]); } } /* * Validate the current config against the MOS config */ static boolean_t spa_config_valid(spa_t *spa, nvlist_t *config) { vdev_t *mrvd, *rvd = spa->spa_root_vdev; nvlist_t *nv; VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nv) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); VERIFY(spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD) == 0); ASSERT3U(rvd->vdev_children, ==, mrvd->vdev_children); /* * If we're doing a normal import, then build up any additional * diagnostic information about missing devices in this config. * We'll pass this up to the user for further processing. */ if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { nvlist_t **child, *nv; uint64_t idx = 0; child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t **), KM_SLEEP); VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); for (int c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; vdev_t *mtvd = mrvd->vdev_child[c]; if (tvd->vdev_ops == &vdev_missing_ops && mtvd->vdev_ops != &vdev_missing_ops && mtvd->vdev_islog) child[idx++] = vdev_config_generate(spa, mtvd, B_FALSE, 0); } if (idx) { VERIFY(nvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, child, idx) == 0); VERIFY(nvlist_add_nvlist(spa->spa_load_info, ZPOOL_CONFIG_MISSING_DEVICES, nv) == 0); for (int i = 0; i < idx; i++) nvlist_free(child[i]); } nvlist_free(nv); kmem_free(child, rvd->vdev_children * sizeof (char **)); } /* * Compare the root vdev tree with the information we have * from the MOS config (mrvd). Check each top-level vdev * with the corresponding MOS config top-level (mtvd). */ for (int c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; vdev_t *mtvd = mrvd->vdev_child[c]; /* * Resolve any "missing" vdevs in the current configuration. * If we find that the MOS config has more accurate information * about the top-level vdev then use that vdev instead. */ if (tvd->vdev_ops == &vdev_missing_ops && mtvd->vdev_ops != &vdev_missing_ops) { if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) continue; /* * Device specific actions. */ if (mtvd->vdev_islog) { spa_set_log_state(spa, SPA_LOG_CLEAR); } else { /* * XXX - once we have 'readonly' pool * support we should be able to handle * missing data devices by transitioning * the pool to readonly. */ continue; } /* * Swap the missing vdev with the data we were * able to obtain from the MOS config. */ vdev_remove_child(rvd, tvd); vdev_remove_child(mrvd, mtvd); vdev_add_child(rvd, mtvd); vdev_add_child(mrvd, tvd); spa_config_exit(spa, SCL_ALL, FTAG); vdev_load(mtvd); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); vdev_reopen(rvd); } else { if (mtvd->vdev_islog) { /* * Load the slog device's state from the MOS * config since it's possible that the label * does not contain the most up-to-date * information. */ vdev_load_log_state(tvd, mtvd); vdev_reopen(tvd); } /* * Per-vdev ZAP info is stored exclusively in the MOS. */ spa_config_valid_zaps(tvd, mtvd); } } vdev_free(mrvd); spa_config_exit(spa, SCL_ALL, FTAG); /* * Ensure we were able to validate the config. */ return (rvd->vdev_guid_sum == spa->spa_uberblock.ub_guid_sum); } /* * Check for missing log devices */ static boolean_t spa_check_logs(spa_t *spa) { boolean_t rv = B_FALSE; dsl_pool_t *dp = spa_get_dsl(spa); switch (spa->spa_log_state) { case SPA_LOG_MISSING: /* need to recheck in case slog has been restored */ case SPA_LOG_UNKNOWN: rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); if (rv) spa_set_log_state(spa, SPA_LOG_MISSING); break; } return (rv); } static boolean_t spa_passivate_log(spa_t *spa) { vdev_t *rvd = spa->spa_root_vdev; boolean_t slog_found = B_FALSE; ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); if (!spa_has_slogs(spa)) return (B_FALSE); for (int c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; metaslab_group_t *mg = tvd->vdev_mg; if (tvd->vdev_islog) { metaslab_group_passivate(mg); slog_found = B_TRUE; } } return (slog_found); } static void spa_activate_log(spa_t *spa) { vdev_t *rvd = spa->spa_root_vdev; ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); for (int c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; metaslab_group_t *mg = tvd->vdev_mg; if (tvd->vdev_islog) metaslab_group_activate(mg); } } int spa_offline_log(spa_t *spa) { int error; error = dmu_objset_find(spa_name(spa), zil_vdev_offline, NULL, DS_FIND_CHILDREN); if (error == 0) { /* * We successfully offlined the log device, sync out the * current txg so that the "stubby" block can be removed * by zil_sync(). */ txg_wait_synced(spa->spa_dsl_pool, 0); } return (error); } static void spa_aux_check_removed(spa_aux_vdev_t *sav) { int i; for (i = 0; i < sav->sav_count; i++) spa_check_removed(sav->sav_vdevs[i]); } void spa_claim_notify(zio_t *zio) { spa_t *spa = zio->io_spa; if (zio->io_error) return; mutex_enter(&spa->spa_props_lock); /* any mutex will do */ if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) spa->spa_claim_max_txg = zio->io_bp->blk_birth; mutex_exit(&spa->spa_props_lock); } typedef struct spa_load_error { uint64_t sle_meta_count; uint64_t sle_data_count; } spa_load_error_t; static void spa_load_verify_done(zio_t *zio) { blkptr_t *bp = zio->io_bp; spa_load_error_t *sle = zio->io_private; dmu_object_type_t type = BP_GET_TYPE(bp); int error = zio->io_error; spa_t *spa = zio->io_spa; abd_free(zio->io_abd); if (error) { if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && type != DMU_OT_INTENT_LOG) atomic_inc_64(&sle->sle_meta_count); else atomic_inc_64(&sle->sle_data_count); } mutex_enter(&spa->spa_scrub_lock); spa->spa_scrub_inflight--; cv_broadcast(&spa->spa_scrub_io_cv); mutex_exit(&spa->spa_scrub_lock); } /* * Maximum number of concurrent scrub i/os to create while verifying * a pool while importing it. */ int spa_load_verify_maxinflight = 10000; boolean_t spa_load_verify_metadata = B_TRUE; boolean_t spa_load_verify_data = B_TRUE; SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_load_verify_maxinflight, CTLFLAG_RWTUN, &spa_load_verify_maxinflight, 0, "Maximum number of concurrent scrub I/Os to create while verifying a " "pool while importing it"); SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_load_verify_metadata, CTLFLAG_RWTUN, &spa_load_verify_metadata, 0, "Check metadata on import?"); SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_load_verify_data, CTLFLAG_RWTUN, &spa_load_verify_data, 0, "Check user data on import?"); /*ARGSUSED*/ static int spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { if (bp == NULL || BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) return (0); /* * Note: normally this routine will not be called if * spa_load_verify_metadata is not set. However, it may be useful * to manually set the flag after the traversal has begun. */ if (!spa_load_verify_metadata) return (0); if (!BP_IS_METADATA(bp) && !spa_load_verify_data) return (0); zio_t *rio = arg; size_t size = BP_GET_PSIZE(bp); mutex_enter(&spa->spa_scrub_lock); while (spa->spa_scrub_inflight >= spa_load_verify_maxinflight) cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); spa->spa_scrub_inflight++; mutex_exit(&spa->spa_scrub_lock); zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); return (0); } /* ARGSUSED */ int verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) { if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); return (0); } static int spa_load_verify(spa_t *spa) { zio_t *rio; spa_load_error_t sle = { 0 }; zpool_rewind_policy_t policy; boolean_t verify_ok = B_FALSE; int error = 0; zpool_get_rewind_policy(spa->spa_config, &policy); if (policy.zrp_request & ZPOOL_NEVER_REWIND) return (0); dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); error = dmu_objset_find_dp(spa->spa_dsl_pool, spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, DS_FIND_CHILDREN); dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); if (error != 0) return (error); rio = zio_root(spa, NULL, &sle, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); if (spa_load_verify_metadata) { error = traverse_pool(spa, spa->spa_verify_min_txg, TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA, spa_load_verify_cb, rio); } (void) zio_wait(rio); spa->spa_load_meta_errors = sle.sle_meta_count; spa->spa_load_data_errors = sle.sle_data_count; if (!error && sle.sle_meta_count <= policy.zrp_maxmeta && sle.sle_data_count <= policy.zrp_maxdata) { int64_t loss = 0; verify_ok = B_TRUE; spa->spa_load_txg = spa->spa_uberblock.ub_txg; spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; VERIFY(nvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0); VERIFY(nvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, loss) == 0); VERIFY(nvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0); } else { spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; } if (error) { if (error != ENXIO && error != EIO) error = SET_ERROR(EIO); return (error); } return (verify_ok ? 0 : EIO); } /* * Find a value in the pool props object. */ static void spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) { (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); } /* * Find a value in the pool directory object. */ static int spa_dir_prop(spa_t *spa, const char *name, uint64_t *val) { return (zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, name, sizeof (uint64_t), 1, val)); } static int spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) { vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); return (err); } /* * Fix up config after a partly-completed split. This is done with the * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off * pool have that entry in their config, but only the splitting one contains * a list of all the guids of the vdevs that are being split off. * * This function determines what to do with that list: either rejoin * all the disks to the pool, or complete the splitting process. To attempt * the rejoin, each disk that is offlined is marked online again, and * we do a reopen() call. If the vdev label for every disk that was * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) * then we call vdev_split() on each disk, and complete the split. * * Otherwise we leave the config alone, with all the vdevs in place in * the original pool. */ static void spa_try_repair(spa_t *spa, nvlist_t *config) { uint_t extracted; uint64_t *glist; uint_t i, gcount; nvlist_t *nvl; vdev_t **vd; boolean_t attempt_reopen; if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) return; /* check that the config is complete */ if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, &glist, &gcount) != 0) return; vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); /* attempt to online all the vdevs & validate */ attempt_reopen = B_TRUE; for (i = 0; i < gcount; i++) { if (glist[i] == 0) /* vdev is hole */ continue; vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); if (vd[i] == NULL) { /* * Don't bother attempting to reopen the disks; * just do the split. */ attempt_reopen = B_FALSE; } else { /* attempt to re-online it */ vd[i]->vdev_offline = B_FALSE; } } if (attempt_reopen) { vdev_reopen(spa->spa_root_vdev); /* check each device to see what state it's in */ for (extracted = 0, i = 0; i < gcount; i++) { if (vd[i] != NULL && vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) break; ++extracted; } } /* * If every disk has been moved to the new pool, or if we never * even attempted to look at them, then we split them off for * good. */ if (!attempt_reopen || gcount == extracted) { for (i = 0; i < gcount; i++) if (vd[i] != NULL) vdev_split(vd[i]); vdev_reopen(spa->spa_root_vdev); } kmem_free(vd, gcount * sizeof (vdev_t *)); } static int spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig) { nvlist_t *config = spa->spa_config; char *ereport = FM_EREPORT_ZFS_POOL; char *comment; int error; uint64_t pool_guid; nvlist_t *nvl; if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) return (SET_ERROR(EINVAL)); ASSERT(spa->spa_comment == NULL); if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) spa->spa_comment = spa_strdup(comment); /* * Versioning wasn't explicitly added to the label until later, so if * it's not present treat it as the initial version. */ if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, &spa->spa_ubsync.ub_version) != 0) spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &spa->spa_config_txg); if ((state == SPA_LOAD_IMPORT || state == SPA_LOAD_TRYIMPORT) && spa_guid_exists(pool_guid, 0)) { error = SET_ERROR(EEXIST); } else { spa->spa_config_guid = pool_guid; if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) { VERIFY(nvlist_dup(nvl, &spa->spa_config_splitting, KM_SLEEP) == 0); } nvlist_free(spa->spa_load_info); spa->spa_load_info = fnvlist_alloc(); gethrestime(&spa->spa_loaded_ts); error = spa_load_impl(spa, pool_guid, config, state, type, mosconfig, &ereport); } /* * Don't count references from objsets that are already closed * and are making their way through the eviction process. */ spa_evicting_os_wait(spa); spa->spa_minref = refcount_count(&spa->spa_refcount); if (error) { if (error != EEXIST) { spa->spa_loaded_ts.tv_sec = 0; spa->spa_loaded_ts.tv_nsec = 0; } if (error != EBADF) { zfs_ereport_post(ereport, spa, NULL, NULL, 0, 0); } } spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; spa->spa_ena = 0; return (error); } /* * Count the number of per-vdev ZAPs associated with all of the vdevs in the * vdev tree rooted in the given vd, and ensure that each ZAP is present in the * spa's per-vdev ZAP list. */ static uint64_t vdev_count_verify_zaps(vdev_t *vd) { spa_t *spa = vd->vdev_spa; uint64_t total = 0; if (vd->vdev_top_zap != 0) { total++; ASSERT0(zap_lookup_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps, vd->vdev_top_zap)); } if (vd->vdev_leaf_zap != 0) { total++; ASSERT0(zap_lookup_int(spa->spa_meta_objset, spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); } for (uint64_t i = 0; i < vd->vdev_children; i++) { total += vdev_count_verify_zaps(vd->vdev_child[i]); } return (total); } /* * Load an existing storage pool, using the pool's builtin spa_config as a * source of configuration information. */ static int spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, spa_load_state_t state, spa_import_type_t type, boolean_t mosconfig, char **ereport) { int error = 0; nvlist_t *nvroot = NULL; nvlist_t *label; vdev_t *rvd; uberblock_t *ub = &spa->spa_uberblock; uint64_t children, config_cache_txg = spa->spa_config_txg; int orig_mode = spa->spa_mode; int parse; uint64_t obj; boolean_t missing_feat_write = B_FALSE; /* * If this is an untrusted config, access the pool in read-only mode. * This prevents things like resilvering recently removed devices. */ if (!mosconfig) spa->spa_mode = FREAD; ASSERT(MUTEX_HELD(&spa_namespace_lock)); spa->spa_load_state = state; if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot)) return (SET_ERROR(EINVAL)); parse = (type == SPA_IMPORT_EXISTING ? VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); /* * Create "The Godfather" zio to hold all async IOs */ spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); for (int i = 0; i < max_ncpus; i++) { spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER); } /* * Parse the configuration into a vdev tree. We explicitly set the * value that will be returned by spa_version() since parsing the * configuration requires knowing the version number. */ spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, parse); spa_config_exit(spa, SCL_ALL, FTAG); if (error != 0) return (error); ASSERT(spa->spa_root_vdev == rvd); ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); if (type != SPA_IMPORT_ASSEMBLE) { ASSERT(spa_guid(spa) == pool_guid); } /* * Try to open all vdevs, loading each label in the process. */ spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = vdev_open(rvd); spa_config_exit(spa, SCL_ALL, FTAG); if (error != 0) return (error); /* * We need to validate the vdev labels against the configuration that * we have in hand, which is dependent on the setting of mosconfig. If * mosconfig is true then we're validating the vdev labels based on * that config. Otherwise, we're validating against the cached config * (zpool.cache) that was read when we loaded the zfs module, and then * later we will recursively call spa_load() and validate against * the vdev config. * * If we're assembling a new pool that's been split off from an * existing pool, the labels haven't yet been updated so we skip * validation for now. */ if (type != SPA_IMPORT_ASSEMBLE) { spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = vdev_validate(rvd, mosconfig); spa_config_exit(spa, SCL_ALL, FTAG); if (error != 0) return (error); if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) return (SET_ERROR(ENXIO)); } /* * Find the best uberblock. */ vdev_uberblock_load(rvd, ub, &label); /* * If we weren't able to find a single valid uberblock, return failure. */ if (ub->ub_txg == 0) { nvlist_free(label); return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); } /* * If the pool has an unsupported version we can't open it. */ if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { nvlist_free(label); return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); } if (ub->ub_version >= SPA_VERSION_FEATURES) { nvlist_t *features; /* * If we weren't able to find what's necessary for reading the * MOS in the label, return failure. */ if (label == NULL || nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, &features) != 0) { nvlist_free(label); return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); } /* * Update our in-core representation with the definitive values * from the label. */ nvlist_free(spa->spa_label_features); VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0); } nvlist_free(label); /* * Look through entries in the label nvlist's features_for_read. If * there is a feature listed there which we don't understand then we * cannot open a pool. */ if (ub->ub_version >= SPA_VERSION_FEATURES) { nvlist_t *unsup_feat; VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) == 0); for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, NULL); nvp != NULL; nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { if (!zfeature_is_supported(nvpair_name(nvp))) { VERIFY(nvlist_add_string(unsup_feat, nvpair_name(nvp), "") == 0); } } if (!nvlist_empty(unsup_feat)) { VERIFY(nvlist_add_nvlist(spa->spa_load_info, ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0); nvlist_free(unsup_feat); return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, ENOTSUP)); } nvlist_free(unsup_feat); } /* * If the vdev guid sum doesn't match the uberblock, we have an * incomplete configuration. We first check to see if the pool * is aware of the complete config (i.e ZPOOL_CONFIG_VDEV_CHILDREN). * If it is, defer the vdev_guid_sum check till later so we * can handle missing vdevs. */ if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN, &children) != 0 && mosconfig && type != SPA_IMPORT_ASSEMBLE && rvd->vdev_guid_sum != ub->ub_guid_sum) return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_try_repair(spa, config); spa_config_exit(spa, SCL_ALL, FTAG); nvlist_free(spa->spa_config_splitting); spa->spa_config_splitting = NULL; } /* * Initialize internal SPA structures. */ spa->spa_state = POOL_STATE_ACTIVE; spa->spa_ubsync = spa->spa_uberblock; spa->spa_verify_min_txg = spa->spa_extreme_rewind ? TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; spa->spa_first_txg = spa->spa_last_ubsync_txg ? spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; spa->spa_claim_max_txg = spa->spa_first_txg; spa->spa_prev_software_version = ub->ub_software_version; error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); if (error) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (spa_version(spa) >= SPA_VERSION_FEATURES) { boolean_t missing_feat_read = B_FALSE; nvlist_t *unsup_feat, *enabled_feat; if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, &spa->spa_feat_for_read_obj) != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, &spa->spa_feat_for_write_obj) != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, &spa->spa_feat_desc_obj) != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } enabled_feat = fnvlist_alloc(); unsup_feat = fnvlist_alloc(); if (!spa_features_check(spa, B_FALSE, unsup_feat, enabled_feat)) missing_feat_read = B_TRUE; if (spa_writeable(spa) || state == SPA_LOAD_TRYIMPORT) { if (!spa_features_check(spa, B_TRUE, unsup_feat, enabled_feat)) { missing_feat_write = B_TRUE; } } fnvlist_add_nvlist(spa->spa_load_info, ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); if (!nvlist_empty(unsup_feat)) { fnvlist_add_nvlist(spa->spa_load_info, ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); } fnvlist_free(enabled_feat); fnvlist_free(unsup_feat); if (!missing_feat_read) { fnvlist_add_boolean(spa->spa_load_info, ZPOOL_CONFIG_CAN_RDONLY); } /* * If the state is SPA_LOAD_TRYIMPORT, our objective is * twofold: to determine whether the pool is available for * import in read-write mode and (if it is not) whether the * pool is available for import in read-only mode. If the pool * is available for import in read-write mode, it is displayed * as available in userland; if it is not available for import * in read-only mode, it is displayed as unavailable in * userland. If the pool is available for import in read-only * mode but not read-write mode, it is displayed as unavailable * in userland with a special note that the pool is actually * available for open in read-only mode. * * As a result, if the state is SPA_LOAD_TRYIMPORT and we are * missing a feature for write, we must first determine whether * the pool can be opened read-only before returning to * userland in order to know whether to display the * abovementioned note. */ if (missing_feat_read || (missing_feat_write && spa_writeable(spa))) { return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, ENOTSUP)); } /* * Load refcounts for ZFS features from disk into an in-memory * cache during SPA initialization. */ for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { uint64_t refcount; error = feature_get_refcount_from_disk(spa, &spa_feature_table[i], &refcount); if (error == 0) { spa->spa_feat_refcount_cache[i] = refcount; } else if (error == ENOTSUP) { spa->spa_feat_refcount_cache[i] = SPA_FEATURE_DISABLED; } else { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } } } if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, &spa->spa_feat_enabled_txg_obj) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } spa->spa_is_initializing = B_TRUE; error = dsl_pool_open(spa->spa_dsl_pool); spa->spa_is_initializing = B_FALSE; if (error != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (!mosconfig) { uint64_t hostid; nvlist_t *policy = NULL, *nvconfig; if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (!spa_is_root(spa) && nvlist_lookup_uint64(nvconfig, ZPOOL_CONFIG_HOSTID, &hostid) == 0) { char *hostname; unsigned long myhostid = 0; VERIFY(nvlist_lookup_string(nvconfig, ZPOOL_CONFIG_HOSTNAME, &hostname) == 0); #ifdef _KERNEL myhostid = zone_get_hostid(NULL); #else /* _KERNEL */ /* * We're emulating the system's hostid in userland, so * we can't use zone_get_hostid(). */ (void) ddi_strtoul(hw_serial, NULL, 10, &myhostid); #endif /* _KERNEL */ if (check_hostid && hostid != 0 && myhostid != 0 && hostid != myhostid) { nvlist_free(nvconfig); cmn_err(CE_WARN, "pool '%s' could not be " "loaded as it was last accessed by " "another system (host: %s hostid: 0x%lx). " "See: http://illumos.org/msg/ZFS-8000-EY", spa_name(spa), hostname, (unsigned long)hostid); return (SET_ERROR(EBADF)); } } if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_REWIND_POLICY, &policy) == 0) VERIFY(nvlist_add_nvlist(nvconfig, ZPOOL_REWIND_POLICY, policy) == 0); spa_config_set(spa, nvconfig); spa_unload(spa); spa_deactivate(spa); spa_activate(spa, orig_mode); return (spa_load(spa, state, SPA_IMPORT_EXISTING, B_TRUE)); } /* Grab the secret checksum salt from the MOS. */ error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, sizeof (spa->spa_cksum_salt.zcs_bytes), spa->spa_cksum_salt.zcs_bytes); if (error == ENOENT) { /* Generate a new salt for subsequent use */ (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, sizeof (spa->spa_cksum_salt.zcs_bytes)); } else if (error != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); if (error != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); /* * Load the bit that tells us to use the new accounting function * (raid-z deflation). If we have an older pool, this will not * be present. */ error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, &spa->spa_creation_version); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); /* * Load the persistent error log. If we have an older pool, this will * not be present. */ error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, &spa->spa_errlog_scrub); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); /* * Load the history object. If we have an older pool, this * will not be present. */ error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); /* * Load the per-vdev ZAP map. If we have an older pool, this will not * be present; in this case, defer its creation to a later time to * avoid dirtying the MOS this early / out of sync context. See * spa_sync_config_object. */ /* The sentinel is only available in the MOS config. */ nvlist_t *mos_config; if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, &spa->spa_all_vdev_zaps); if (error == ENOENT) { VERIFY(!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); spa->spa_avz_action = AVZ_ACTION_INITIALIZE; ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); } else if (error != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { /* * An older version of ZFS overwrote the sentinel value, so * we have orphaned per-vdev ZAPs in the MOS. Defer their * destruction to later; see spa_sync_config_object. */ spa->spa_avz_action = AVZ_ACTION_DESTROY; /* * We're assuming that no vdevs have had their ZAPs created * before this. Better be sure of it. */ ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); } nvlist_free(mos_config); /* * If we're assembling the pool from the split-off vdevs of * an existing pool, we don't want to attach the spares & cache * devices. */ /* * Load any hot spares for this pool. */ error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); if (load_nvlist(spa, spa->spa_spares.sav_object, &spa->spa_spares.sav_config) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_spares(spa); spa_config_exit(spa, SCL_ALL, FTAG); } else if (error == 0) { spa->spa_spares.sav_sync = B_TRUE; } /* * Load any level 2 ARC devices for this pool. */ error = spa_dir_prop(spa, DMU_POOL_L2CACHE, &spa->spa_l2cache.sav_object); if (error != 0 && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); if (load_nvlist(spa, spa->spa_l2cache.sav_object, &spa->spa_l2cache.sav_config) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_l2cache(spa); spa_config_exit(spa, SCL_ALL, FTAG); } else if (error == 0) { spa->spa_l2cache.sav_sync = B_TRUE; } spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object); if (error && error != ENOENT) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (error == 0) { uint64_t autoreplace; spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); spa_prop_find(spa, ZPOOL_PROP_DEDUPDITTO, &spa->spa_dedup_ditto); spa->spa_autoreplace = (autoreplace != 0); } /* * If the 'autoreplace' property is set, then post a resource notifying * the ZFS DE that it should not issue any faults for unopenable * devices. We also iterate over the vdevs, and post a sysevent for any * unopenable vdevs so that the normal autoreplace handler can take * over. */ if (spa->spa_autoreplace && state != SPA_LOAD_TRYIMPORT) { spa_check_removed(spa->spa_root_vdev); /* * For the import case, this is done in spa_import(), because * at this point we're using the spare definitions from * the MOS config, not necessarily from the userland config. */ if (state != SPA_LOAD_IMPORT) { spa_aux_check_removed(&spa->spa_spares); spa_aux_check_removed(&spa->spa_l2cache); } } /* * Load the vdev state for all toplevel vdevs. */ vdev_load(rvd); /* * Propagate the leaf DTLs we just loaded all the way up the tree. */ spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); vdev_dtl_reassess(rvd, 0, 0, B_FALSE); spa_config_exit(spa, SCL_ALL, FTAG); /* * Load the DDTs (dedup tables). */ error = ddt_load(spa); if (error != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); spa_update_dspace(spa); /* * Validate the config, using the MOS config to fill in any * information which might be missing. If we fail to validate * the config then declare the pool unfit for use. If we're * assembling a pool from a split, the log is not transferred * over. */ if (type != SPA_IMPORT_ASSEMBLE) { nvlist_t *nvconfig; if (load_nvlist(spa, spa->spa_config_object, &nvconfig) != 0) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); if (!spa_config_valid(spa, nvconfig)) { nvlist_free(nvconfig); return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); } nvlist_free(nvconfig); /* * Now that we've validated the config, check the state of the * root vdev. If it can't be opened, it indicates one or * more toplevel vdevs are faulted. */ if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) return (SET_ERROR(ENXIO)); if (spa_writeable(spa) && spa_check_logs(spa)) { *ereport = FM_EREPORT_ZFS_LOG_REPLAY; return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, ENXIO)); } } if (missing_feat_write) { ASSERT(state == SPA_LOAD_TRYIMPORT); /* * At this point, we know that we can open the pool in * read-only mode but not read-write mode. We now have enough * information and can return to userland. */ return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, ENOTSUP)); } /* * We've successfully opened the pool, verify that we're ready * to start pushing transactions. */ if (state != SPA_LOAD_TRYIMPORT) { if (error = spa_load_verify(spa)) return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); } if (spa_writeable(spa) && (state == SPA_LOAD_RECOVER || spa->spa_load_max_txg == UINT64_MAX)) { dmu_tx_t *tx; int need_update = B_FALSE; dsl_pool_t *dp = spa_get_dsl(spa); ASSERT(state != SPA_LOAD_TRYIMPORT); /* * Claim log blocks that haven't been committed yet. * This must all happen in a single txg. * Note: spa_claim_max_txg is updated by spa_claim_notify(), * invoked from zil_claim_log_block()'s i/o done callback. * Price of rollback is that we abandon the log. */ spa->spa_claiming = B_TRUE; tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, zil_claim, tx, DS_FIND_CHILDREN); dmu_tx_commit(tx); spa->spa_claiming = B_FALSE; spa_set_log_state(spa, SPA_LOG_GOOD); spa->spa_sync_on = B_TRUE; txg_sync_start(spa->spa_dsl_pool); /* * Wait for all claims to sync. We sync up to the highest * claimed log block birth time so that claimed log blocks * don't appear to be from the future. spa_claim_max_txg * will have been set for us by either zil_check_log_chain() * (invoked from spa_check_logs()) or zil_claim() above. */ txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); /* * If the config cache is stale, or we have uninitialized * metaslabs (see spa_vdev_add()), then update the config. * * If this is a verbatim import, trust the current * in-core spa_config and update the disk labels. */ if (config_cache_txg != spa->spa_config_txg || state == SPA_LOAD_IMPORT || state == SPA_LOAD_RECOVER || (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) need_update = B_TRUE; for (int c = 0; c < rvd->vdev_children; c++) if (rvd->vdev_child[c]->vdev_ms_array == 0) need_update = B_TRUE; /* * Update the config cache asychronously in case we're the * root pool, in which case the config cache isn't writable yet. */ if (need_update) spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); /* * Check all DTLs to see if anything needs resilvering. */ if (!dsl_scan_resilvering(spa->spa_dsl_pool) && vdev_resilver_needed(rvd, NULL, NULL)) spa_async_request(spa, SPA_ASYNC_RESILVER); /* * Log the fact that we booted up (so that we can detect if * we rebooted in the middle of an operation). */ spa_history_log_version(spa, "open"); /* * Delete any inconsistent datasets. */ (void) dmu_objset_find(spa_name(spa), dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); /* * Clean up any stale temporary dataset userrefs. */ dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); } return (0); } static int spa_load_retry(spa_t *spa, spa_load_state_t state, int mosconfig) { int mode = spa->spa_mode; spa_unload(spa); spa_deactivate(spa); spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; spa_activate(spa, mode); spa_async_suspend(spa); return (spa_load(spa, state, SPA_IMPORT_EXISTING, mosconfig)); } /* * If spa_load() fails this function will try loading prior txg's. If * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this * function will not rewind the pool and will return the same error as * spa_load(). */ static int spa_load_best(spa_t *spa, spa_load_state_t state, int mosconfig, uint64_t max_request, int rewind_flags) { nvlist_t *loadinfo = NULL; nvlist_t *config = NULL; int load_error, rewind_error; uint64_t safe_rewind_txg; uint64_t min_txg; if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { spa->spa_load_max_txg = spa->spa_load_txg; spa_set_log_state(spa, SPA_LOG_CLEAR); } else { spa->spa_load_max_txg = max_request; if (max_request != UINT64_MAX) spa->spa_extreme_rewind = B_TRUE; } load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING, mosconfig); if (load_error == 0) return (0); if (spa->spa_root_vdev != NULL) config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; if (rewind_flags & ZPOOL_NEVER_REWIND) { nvlist_free(config); return (load_error); } if (state == SPA_LOAD_RECOVER) { /* Price of rolling back is discarding txgs, including log */ spa_set_log_state(spa, SPA_LOG_CLEAR); } else { /* * If we aren't rolling back save the load info from our first * import attempt so that we can restore it after attempting * to rewind. */ loadinfo = spa->spa_load_info; spa->spa_load_info = fnvlist_alloc(); } spa->spa_load_max_txg = spa->spa_last_ubsync_txg; safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? TXG_INITIAL : safe_rewind_txg; /* * Continue as long as we're finding errors, we're still within * the acceptable rewind range, and we're still finding uberblocks */ while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { if (spa->spa_load_max_txg < safe_rewind_txg) spa->spa_extreme_rewind = B_TRUE; rewind_error = spa_load_retry(spa, state, mosconfig); } spa->spa_extreme_rewind = B_FALSE; spa->spa_load_max_txg = UINT64_MAX; if (config && (rewind_error || state != SPA_LOAD_RECOVER)) spa_config_set(spa, config); if (state == SPA_LOAD_RECOVER) { ASSERT3P(loadinfo, ==, NULL); return (rewind_error); } else { /* Store the rewind info as part of the initial load info */ fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, spa->spa_load_info); /* Restore the initial load info */ fnvlist_free(spa->spa_load_info); spa->spa_load_info = loadinfo; return (load_error); } } /* * Pool Open/Import * * The import case is identical to an open except that the configuration is sent * down from userland, instead of grabbed from the configuration cache. For the * case of an open, the pool configuration will exist in the * POOL_STATE_UNINITIALIZED state. * * The stats information (gen/count/ustats) is used to gather vdev statistics at * the same time open the pool, without having to keep around the spa_t in some * ambiguous state. */ static int spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, nvlist_t **config) { spa_t *spa; spa_load_state_t state = SPA_LOAD_OPEN; int error; int locked = B_FALSE; int firstopen = B_FALSE; *spapp = NULL; /* * As disgusting as this is, we need to support recursive calls to this * function because dsl_dir_open() is called during spa_load(), and ends * up calling spa_open() again. The real fix is to figure out how to * avoid dsl_dir_open() calling this in the first place. */ if (mutex_owner(&spa_namespace_lock) != curthread) { mutex_enter(&spa_namespace_lock); locked = B_TRUE; } if ((spa = spa_lookup(pool)) == NULL) { if (locked) mutex_exit(&spa_namespace_lock); return (SET_ERROR(ENOENT)); } if (spa->spa_state == POOL_STATE_UNINITIALIZED) { zpool_rewind_policy_t policy; firstopen = B_TRUE; zpool_get_rewind_policy(nvpolicy ? nvpolicy : spa->spa_config, &policy); if (policy.zrp_request & ZPOOL_DO_REWIND) state = SPA_LOAD_RECOVER; spa_activate(spa, spa_mode_global); if (state != SPA_LOAD_RECOVER) spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; error = spa_load_best(spa, state, B_FALSE, policy.zrp_txg, policy.zrp_request); if (error == EBADF) { /* * If vdev_validate() returns failure (indicated by * EBADF), it indicates that one of the vdevs indicates * that the pool has been exported or destroyed. If * this is the case, the config cache is out of sync and * we should remove the pool from the namespace. */ spa_unload(spa); spa_deactivate(spa); spa_config_sync(spa, B_TRUE, B_TRUE); spa_remove(spa); if (locked) mutex_exit(&spa_namespace_lock); return (SET_ERROR(ENOENT)); } if (error) { /* * We can't open the pool, but we still have useful * information: the state of each vdev after the * attempted vdev_open(). Return this to the user. */ if (config != NULL && spa->spa_config) { VERIFY(nvlist_dup(spa->spa_config, config, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); } spa_unload(spa); spa_deactivate(spa); spa->spa_last_open_failed = error; if (locked) mutex_exit(&spa_namespace_lock); *spapp = NULL; return (error); } } spa_open_ref(spa, tag); if (config != NULL) *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); /* * If we've recovered the pool, pass back any information we * gathered while doing the load. */ if (state == SPA_LOAD_RECOVER) { VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); } if (locked) { spa->spa_last_open_failed = 0; spa->spa_last_ubsync_txg = 0; spa->spa_load_txg = 0; mutex_exit(&spa_namespace_lock); #ifdef __FreeBSD__ #ifdef _KERNEL if (firstopen) zvol_create_minors(spa->spa_name); #endif #endif } *spapp = spa; return (0); } int spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy, nvlist_t **config) { return (spa_open_common(name, spapp, tag, policy, config)); } int spa_open(const char *name, spa_t **spapp, void *tag) { return (spa_open_common(name, spapp, tag, NULL, NULL)); } /* * Lookup the given spa_t, incrementing the inject count in the process, * preventing it from being exported or destroyed. */ spa_t * spa_inject_addref(char *name) { spa_t *spa; mutex_enter(&spa_namespace_lock); if ((spa = spa_lookup(name)) == NULL) { mutex_exit(&spa_namespace_lock); return (NULL); } spa->spa_inject_ref++; mutex_exit(&spa_namespace_lock); return (spa); } void spa_inject_delref(spa_t *spa) { mutex_enter(&spa_namespace_lock); spa->spa_inject_ref--; mutex_exit(&spa_namespace_lock); } /* * Add spares device information to the nvlist. */ static void spa_add_spares(spa_t *spa, nvlist_t *config) { nvlist_t **spares; uint_t i, nspares; nvlist_t *nvroot; uint64_t guid; vdev_stat_t *vs; uint_t vsc; uint64_t pool; ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); if (spa->spa_spares.sav_count == 0) return; VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); if (nspares != 0) { VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); VERIFY(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); /* * Go through and find any spares which have since been * repurposed as an active spare. If this is the case, update * their status appropriately. */ for (i = 0; i < nspares; i++) { VERIFY(nvlist_lookup_uint64(spares[i], ZPOOL_CONFIG_GUID, &guid) == 0); if (spa_spare_exists(guid, &pool, NULL) && pool != 0ULL) { VERIFY(nvlist_lookup_uint64_array( spares[i], ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc) == 0); vs->vs_state = VDEV_STATE_CANT_OPEN; vs->vs_aux = VDEV_AUX_SPARED; } } } } /* * Add l2cache device information to the nvlist, including vdev stats. */ static void spa_add_l2cache(spa_t *spa, nvlist_t *config) { nvlist_t **l2cache; uint_t i, j, nl2cache; nvlist_t *nvroot; uint64_t guid; vdev_t *vd; vdev_stat_t *vs; uint_t vsc; ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); if (spa->spa_l2cache.sav_count == 0) return; VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); if (nl2cache != 0) { VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); VERIFY(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); /* * Update level 2 cache device stats. */ for (i = 0; i < nl2cache; i++) { VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID, &guid) == 0); vd = NULL; for (j = 0; j < spa->spa_l2cache.sav_count; j++) { if (guid == spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { vd = spa->spa_l2cache.sav_vdevs[j]; break; } } ASSERT(vd != NULL); VERIFY(nvlist_lookup_uint64_array(l2cache[i], ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc) == 0); vdev_get_stats(vd, vs); } } } static void spa_add_feature_stats(spa_t *spa, nvlist_t *config) { nvlist_t *features; zap_cursor_t zc; zap_attribute_t za; ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); VERIFY(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP) == 0); /* We may be unable to read features if pool is suspended. */ if (spa_suspended(spa)) goto out; if (spa->spa_feat_for_read_obj != 0) { for (zap_cursor_init(&zc, spa->spa_meta_objset, spa->spa_feat_for_read_obj); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { ASSERT(za.za_integer_length == sizeof (uint64_t) && za.za_num_integers == 1); VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name, za.za_first_integer)); } zap_cursor_fini(&zc); } if (spa->spa_feat_for_write_obj != 0) { for (zap_cursor_init(&zc, spa->spa_meta_objset, spa->spa_feat_for_write_obj); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { ASSERT(za.za_integer_length == sizeof (uint64_t) && za.za_num_integers == 1); VERIFY3U(0, ==, nvlist_add_uint64(features, za.za_name, za.za_first_integer)); } zap_cursor_fini(&zc); } out: VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, features) == 0); nvlist_free(features); } int spa_get_stats(const char *name, nvlist_t **config, char *altroot, size_t buflen) { int error; spa_t *spa; *config = NULL; error = spa_open_common(name, &spa, FTAG, NULL, config); if (spa != NULL) { /* * This still leaves a window of inconsistency where the spares * or l2cache devices could change and the config would be * self-inconsistent. */ spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); if (*config != NULL) { uint64_t loadtimes[2]; loadtimes[0] = spa->spa_loaded_ts.tv_sec; loadtimes[1] = spa->spa_loaded_ts.tv_nsec; VERIFY(nvlist_add_uint64_array(*config, ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0); VERIFY(nvlist_add_uint64(*config, ZPOOL_CONFIG_ERRCOUNT, spa_get_errlog_size(spa)) == 0); if (spa_suspended(spa)) VERIFY(nvlist_add_uint64(*config, ZPOOL_CONFIG_SUSPENDED, spa->spa_failmode) == 0); spa_add_spares(spa, *config); spa_add_l2cache(spa, *config); spa_add_feature_stats(spa, *config); } } /* * We want to get the alternate root even for faulted pools, so we cheat * and call spa_lookup() directly. */ if (altroot) { if (spa == NULL) { mutex_enter(&spa_namespace_lock); spa = spa_lookup(name); if (spa) spa_altroot(spa, altroot, buflen); else altroot[0] = '\0'; spa = NULL; mutex_exit(&spa_namespace_lock); } else { spa_altroot(spa, altroot, buflen); } } if (spa != NULL) { spa_config_exit(spa, SCL_CONFIG, FTAG); spa_close(spa, FTAG); } return (error); } /* * Validate that the auxiliary device array is well formed. We must have an * array of nvlists, each which describes a valid leaf vdev. If this is an * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be * specified, as long as they are well-formed. */ static int spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, spa_aux_vdev_t *sav, const char *config, uint64_t version, vdev_labeltype_t label) { nvlist_t **dev; uint_t i, ndev; vdev_t *vd; int error; ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); /* * It's acceptable to have no devs specified. */ if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) return (0); if (ndev == 0) return (SET_ERROR(EINVAL)); /* * Make sure the pool is formatted with a version that supports this * device type. */ if (spa_version(spa) < version) return (SET_ERROR(ENOTSUP)); /* * Set the pending device list so we correctly handle device in-use * checking. */ sav->sav_pending = dev; sav->sav_npending = ndev; for (i = 0; i < ndev; i++) { if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, mode)) != 0) goto out; if (!vd->vdev_ops->vdev_op_leaf) { vdev_free(vd); error = SET_ERROR(EINVAL); goto out; } /* * The L2ARC currently only supports disk devices in * kernel context. For user-level testing, we allow it. */ #ifdef _KERNEL if ((strcmp(config, ZPOOL_CONFIG_L2CACHE) == 0) && strcmp(vd->vdev_ops->vdev_op_type, VDEV_TYPE_DISK) != 0) { error = SET_ERROR(ENOTBLK); vdev_free(vd); goto out; } #endif vd->vdev_top = vd; if ((error = vdev_open(vd)) == 0 && (error = vdev_label_init(vd, crtxg, label)) == 0) { VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, vd->vdev_guid) == 0); } vdev_free(vd); if (error && (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) goto out; else error = 0; } out: sav->sav_pending = NULL; sav->sav_npending = 0; return (error); } static int spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) { int error; ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, VDEV_LABEL_SPARE)) != 0) { return (error); } return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, VDEV_LABEL_L2CACHE)); } static void spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, const char *config) { int i; if (sav->sav_config != NULL) { nvlist_t **olddevs; uint_t oldndevs; nvlist_t **newdevs; /* * Generate new dev list by concatentating with the * current dev list. */ VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config, &olddevs, &oldndevs) == 0); newdevs = kmem_alloc(sizeof (void *) * (ndevs + oldndevs), KM_SLEEP); for (i = 0; i < oldndevs; i++) VERIFY(nvlist_dup(olddevs[i], &newdevs[i], KM_SLEEP) == 0); for (i = 0; i < ndevs; i++) VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], KM_SLEEP) == 0); VERIFY(nvlist_remove(sav->sav_config, config, DATA_TYPE_NVLIST_ARRAY) == 0); VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, newdevs, ndevs + oldndevs) == 0); for (i = 0; i < oldndevs + ndevs; i++) nvlist_free(newdevs[i]); kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); } else { /* * Generate a new dev list. */ VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, devs, ndevs) == 0); } } /* * Stop and drop level 2 ARC devices */ void spa_l2cache_drop(spa_t *spa) { vdev_t *vd; int i; spa_aux_vdev_t *sav = &spa->spa_l2cache; for (i = 0; i < sav->sav_count; i++) { uint64_t pool; vd = sav->sav_vdevs[i]; ASSERT(vd != NULL); if (spa_l2cache_exists(vd->vdev_guid, &pool) && pool != 0ULL && l2arc_vdev_present(vd)) l2arc_remove_vdev(vd); } } /* * Pool Creation */ int spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, nvlist_t *zplprops) { spa_t *spa; char *altroot = NULL; vdev_t *rvd; dsl_pool_t *dp; dmu_tx_t *tx; int error = 0; uint64_t txg = TXG_INITIAL; nvlist_t **spares, **l2cache; uint_t nspares, nl2cache; uint64_t version, obj; boolean_t has_features; /* * If this pool already exists, return failure. */ mutex_enter(&spa_namespace_lock); if (spa_lookup(pool) != NULL) { mutex_exit(&spa_namespace_lock); return (SET_ERROR(EEXIST)); } /* * Allocate a new spa_t structure. */ (void) nvlist_lookup_string(props, zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); spa = spa_add(pool, NULL, altroot); spa_activate(spa, spa_mode_global); if (props && (error = spa_prop_validate(spa, props))) { spa_deactivate(spa); spa_remove(spa); mutex_exit(&spa_namespace_lock); return (error); } has_features = B_FALSE; for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); elem != NULL; elem = nvlist_next_nvpair(props, elem)) { if (zpool_prop_feature(nvpair_name(elem))) has_features = B_TRUE; } if (has_features || nvlist_lookup_uint64(props, zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { version = SPA_VERSION; } ASSERT(SPA_VERSION_IS_SUPPORTED(version)); spa->spa_first_txg = txg; spa->spa_uberblock.ub_txg = txg - 1; spa->spa_uberblock.ub_version = version; spa->spa_ubsync = spa->spa_uberblock; spa->spa_load_state = SPA_LOAD_CREATE; /* * Create "The Godfather" zio to hold all async IOs */ spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); for (int i = 0; i < max_ncpus; i++) { spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER); } /* * Create the root vdev. */ spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); ASSERT(error != 0 || rvd != NULL); ASSERT(error != 0 || spa->spa_root_vdev == rvd); if (error == 0 && !zfs_allocatable_devs(nvroot)) error = SET_ERROR(EINVAL); if (error == 0 && (error = vdev_create(rvd, txg, B_FALSE)) == 0 && (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { for (int c = 0; c < rvd->vdev_children; c++) { vdev_ashift_optimize(rvd->vdev_child[c]); vdev_metaslab_set_size(rvd->vdev_child[c]); vdev_expand(rvd->vdev_child[c], txg); } } spa_config_exit(spa, SCL_ALL, FTAG); if (error != 0) { spa_unload(spa); spa_deactivate(spa); spa_remove(spa); mutex_exit(&spa_namespace_lock); return (error); } /* * Get the list of spares, if specified. */ if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) { VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_spares(spa); spa_config_exit(spa, SCL_ALL, FTAG); spa->spa_spares.sav_sync = B_TRUE; } /* * Get the list of level 2 cache devices, if specified. */ if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0) { VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_l2cache(spa); spa_config_exit(spa, SCL_ALL, FTAG); spa->spa_l2cache.sav_sync = B_TRUE; } spa->spa_is_initializing = B_TRUE; spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, txg); spa->spa_meta_objset = dp->dp_meta_objset; spa->spa_is_initializing = B_FALSE; /* * Create DDTs (dedup tables). */ ddt_create(spa); spa_update_dspace(spa); tx = dmu_tx_create_assigned(dp, txg); /* * Create the pool config object. */ spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); if (zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { cmn_err(CE_PANIC, "failed to add pool config"); } if (spa_version(spa) >= SPA_VERSION_FEATURES) spa_feature_create_zap_objects(spa, tx); if (zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, sizeof (uint64_t), 1, &version, tx) != 0) { cmn_err(CE_PANIC, "failed to add pool version"); } /* Newly created pools with the right version are always deflated. */ if (version >= SPA_VERSION_RAIDZ_DEFLATE) { spa->spa_deflate = TRUE; if (zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { cmn_err(CE_PANIC, "failed to add deflate"); } } /* * Create the deferred-free bpobj. Turn off compression * because sync-to-convergence takes longer if the blocksize * keeps changing. */ obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); dmu_object_set_compress(spa->spa_meta_objset, obj, ZIO_COMPRESS_OFF, tx); if (zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, sizeof (uint64_t), 1, &obj, tx) != 0) { cmn_err(CE_PANIC, "failed to add bpobj"); } VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj)); /* * Create the pool's history object. */ if (version >= SPA_VERSION_ZPOOL_HISTORY) spa_history_create_obj(spa, tx); /* * Generate some random noise for salted checksums to operate on. */ (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, sizeof (spa->spa_cksum_salt.zcs_bytes)); /* * Set pool properties. */ spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); if (props != NULL) { spa_configfile_set(spa, props, B_FALSE); spa_sync_props(props, tx); } dmu_tx_commit(tx); spa->spa_sync_on = B_TRUE; txg_sync_start(spa->spa_dsl_pool); /* * We explicitly wait for the first transaction to complete so that our * bean counters are appropriately updated. */ txg_wait_synced(spa->spa_dsl_pool, txg); spa_config_sync(spa, B_FALSE, B_TRUE); spa_event_notify(spa, NULL, ESC_ZFS_POOL_CREATE); spa_history_log_version(spa, "create"); /* * Don't count references from objsets that are already closed * and are making their way through the eviction process. */ spa_evicting_os_wait(spa); spa->spa_minref = refcount_count(&spa->spa_refcount); spa->spa_load_state = SPA_LOAD_NONE; mutex_exit(&spa_namespace_lock); return (0); } #ifdef _KERNEL #ifdef illumos /* * Get the root pool information from the root disk, then import the root pool * during the system boot up time. */ extern int vdev_disk_read_rootlabel(char *, char *, nvlist_t **); static nvlist_t * spa_generate_rootconf(char *devpath, char *devid, uint64_t *guid) { nvlist_t *config; nvlist_t *nvtop, *nvroot; uint64_t pgid; if (vdev_disk_read_rootlabel(devpath, devid, &config) != 0) return (NULL); /* * Add this top-level vdev to the child array. */ VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtop) == 0); VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pgid) == 0); VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID, guid) == 0); /* * Put this pool's top-level vdevs into a root vdev. */ VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_GUID, pgid) == 0); VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, &nvtop, 1) == 0); /* * Replace the existing vdev_tree with the new root vdev in * this pool's configuration (remove the old, add the new). */ VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, nvroot) == 0); nvlist_free(nvroot); return (config); } /* * Walk the vdev tree and see if we can find a device with "better" * configuration. A configuration is "better" if the label on that * device has a more recent txg. */ static void spa_alt_rootvdev(vdev_t *vd, vdev_t **avd, uint64_t *txg) { for (int c = 0; c < vd->vdev_children; c++) spa_alt_rootvdev(vd->vdev_child[c], avd, txg); if (vd->vdev_ops->vdev_op_leaf) { nvlist_t *label; uint64_t label_txg; if (vdev_disk_read_rootlabel(vd->vdev_physpath, vd->vdev_devid, &label) != 0) return; VERIFY(nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_TXG, &label_txg) == 0); /* * Do we have a better boot device? */ if (label_txg > *txg) { *txg = label_txg; *avd = vd; } nvlist_free(label); } } /* * Import a root pool. * * For x86. devpath_list will consist of devid and/or physpath name of * the vdev (e.g. "id1,sd@SSEAGATE..." or "/pci@1f,0/ide@d/disk@0,0:a"). * The GRUB "findroot" command will return the vdev we should boot. * * For Sparc, devpath_list consists the physpath name of the booting device * no matter the rootpool is a single device pool or a mirrored pool. * e.g. * "/pci@1f,0/ide@d/disk@0,0:a" */ int spa_import_rootpool(char *devpath, char *devid) { spa_t *spa; vdev_t *rvd, *bvd, *avd = NULL; nvlist_t *config, *nvtop; uint64_t guid, txg; char *pname; int error; /* * Read the label from the boot device and generate a configuration. */ config = spa_generate_rootconf(devpath, devid, &guid); #if defined(_OBP) && defined(_KERNEL) if (config == NULL) { if (strstr(devpath, "/iscsi/ssd") != NULL) { /* iscsi boot */ get_iscsi_bootpath_phy(devpath); config = spa_generate_rootconf(devpath, devid, &guid); } } #endif if (config == NULL) { cmn_err(CE_NOTE, "Cannot read the pool label from '%s'", devpath); return (SET_ERROR(EIO)); } VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, &pname) == 0); VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &txg) == 0); mutex_enter(&spa_namespace_lock); if ((spa = spa_lookup(pname)) != NULL) { /* * Remove the existing root pool from the namespace so that we * can replace it with the correct config we just read in. */ spa_remove(spa); } spa = spa_add(pname, config, NULL); spa->spa_is_root = B_TRUE; spa->spa_import_flags = ZFS_IMPORT_VERBATIM; /* * Build up a vdev tree based on the boot device's label config. */ VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtop) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = spa_config_parse(spa, &rvd, nvtop, NULL, 0, VDEV_ALLOC_ROOTPOOL); spa_config_exit(spa, SCL_ALL, FTAG); if (error) { mutex_exit(&spa_namespace_lock); nvlist_free(config); cmn_err(CE_NOTE, "Can not parse the config for pool '%s'", pname); return (error); } /* * Get the boot vdev. */ if ((bvd = vdev_lookup_by_guid(rvd, guid)) == NULL) { cmn_err(CE_NOTE, "Can not find the boot vdev for guid %llu", (u_longlong_t)guid); error = SET_ERROR(ENOENT); goto out; } /* * Determine if there is a better boot device. */ avd = bvd; spa_alt_rootvdev(rvd, &avd, &txg); if (avd != bvd) { cmn_err(CE_NOTE, "The boot device is 'degraded'. Please " "try booting from '%s'", avd->vdev_path); error = SET_ERROR(EINVAL); goto out; } /* * If the boot device is part of a spare vdev then ensure that * we're booting off the active spare. */ if (bvd->vdev_parent->vdev_ops == &vdev_spare_ops && !bvd->vdev_isspare) { cmn_err(CE_NOTE, "The boot device is currently spared. Please " "try booting from '%s'", bvd->vdev_parent-> vdev_child[bvd->vdev_parent->vdev_children - 1]->vdev_path); error = SET_ERROR(EINVAL); goto out; } error = 0; out: spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); vdev_free(rvd); spa_config_exit(spa, SCL_ALL, FTAG); mutex_exit(&spa_namespace_lock); nvlist_free(config); return (error); } #else /* !illumos */ extern int vdev_geom_read_pool_label(const char *name, nvlist_t ***configs, uint64_t *count); static nvlist_t * spa_generate_rootconf(const char *name) { nvlist_t **configs, **tops; nvlist_t *config; nvlist_t *best_cfg, *nvtop, *nvroot; uint64_t *holes; uint64_t best_txg; uint64_t nchildren; uint64_t pgid; uint64_t count; uint64_t i; uint_t nholes; if (vdev_geom_read_pool_label(name, &configs, &count) != 0) return (NULL); ASSERT3U(count, !=, 0); best_txg = 0; for (i = 0; i < count; i++) { uint64_t txg; VERIFY(nvlist_lookup_uint64(configs[i], ZPOOL_CONFIG_POOL_TXG, &txg) == 0); if (txg > best_txg) { best_txg = txg; best_cfg = configs[i]; } } nchildren = 1; nvlist_lookup_uint64(best_cfg, ZPOOL_CONFIG_VDEV_CHILDREN, &nchildren); holes = NULL; nvlist_lookup_uint64_array(best_cfg, ZPOOL_CONFIG_HOLE_ARRAY, &holes, &nholes); tops = kmem_zalloc(nchildren * sizeof(void *), KM_SLEEP); for (i = 0; i < nchildren; i++) { if (i >= count) break; if (configs[i] == NULL) continue; VERIFY(nvlist_lookup_nvlist(configs[i], ZPOOL_CONFIG_VDEV_TREE, &nvtop) == 0); nvlist_dup(nvtop, &tops[i], KM_SLEEP); } for (i = 0; holes != NULL && i < nholes; i++) { if (i >= nchildren) continue; if (tops[holes[i]] != NULL) continue; nvlist_alloc(&tops[holes[i]], NV_UNIQUE_NAME, KM_SLEEP); VERIFY(nvlist_add_string(tops[holes[i]], ZPOOL_CONFIG_TYPE, VDEV_TYPE_HOLE) == 0); VERIFY(nvlist_add_uint64(tops[holes[i]], ZPOOL_CONFIG_ID, holes[i]) == 0); VERIFY(nvlist_add_uint64(tops[holes[i]], ZPOOL_CONFIG_GUID, 0) == 0); } for (i = 0; i < nchildren; i++) { if (tops[i] != NULL) continue; nvlist_alloc(&tops[i], NV_UNIQUE_NAME, KM_SLEEP); VERIFY(nvlist_add_string(tops[i], ZPOOL_CONFIG_TYPE, VDEV_TYPE_MISSING) == 0); VERIFY(nvlist_add_uint64(tops[i], ZPOOL_CONFIG_ID, i) == 0); VERIFY(nvlist_add_uint64(tops[i], ZPOOL_CONFIG_GUID, 0) == 0); } /* * Create pool config based on the best vdev config. */ nvlist_dup(best_cfg, &config, KM_SLEEP); /* * Put this pool's top-level vdevs into a root vdev. */ VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pgid) == 0); VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_GUID, pgid) == 0); VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, tops, nchildren) == 0); /* * Replace the existing vdev_tree with the new root vdev in * this pool's configuration (remove the old, add the new). */ VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, nvroot) == 0); /* * Drop vdev config elements that should not be present at pool level. */ nvlist_remove(config, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64); nvlist_remove(config, ZPOOL_CONFIG_TOP_GUID, DATA_TYPE_UINT64); for (i = 0; i < count; i++) nvlist_free(configs[i]); kmem_free(configs, count * sizeof(void *)); for (i = 0; i < nchildren; i++) nvlist_free(tops[i]); kmem_free(tops, nchildren * sizeof(void *)); nvlist_free(nvroot); return (config); } int spa_import_rootpool(const char *name) { spa_t *spa; vdev_t *rvd, *bvd, *avd = NULL; nvlist_t *config, *nvtop; uint64_t txg; char *pname; int error; /* * Read the label from the boot device and generate a configuration. */ config = spa_generate_rootconf(name); mutex_enter(&spa_namespace_lock); if (config != NULL) { VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, &pname) == 0 && strcmp(name, pname) == 0); VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &txg) == 0); if ((spa = spa_lookup(pname)) != NULL) { /* + * The pool could already be imported, + * e.g., after reboot -r. + */ + if (spa->spa_state == POOL_STATE_ACTIVE) { + mutex_exit(&spa_namespace_lock); + nvlist_free(config); + return (0); + } + + /* * Remove the existing root pool from the namespace so * that we can replace it with the correct config * we just read in. */ spa_remove(spa); } spa = spa_add(pname, config, NULL); /* * Set spa_ubsync.ub_version as it can be used in vdev_alloc() * via spa_version(). */ if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, &spa->spa_ubsync.ub_version) != 0) spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; } else if ((spa = spa_lookup(name)) == NULL) { mutex_exit(&spa_namespace_lock); nvlist_free(config); cmn_err(CE_NOTE, "Cannot find the pool label for '%s'", name); return (EIO); } else { VERIFY(nvlist_dup(spa->spa_config, &config, KM_SLEEP) == 0); } spa->spa_is_root = B_TRUE; spa->spa_import_flags = ZFS_IMPORT_VERBATIM; /* * Build up a vdev tree based on the boot device's label config. */ VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtop) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); error = spa_config_parse(spa, &rvd, nvtop, NULL, 0, VDEV_ALLOC_ROOTPOOL); spa_config_exit(spa, SCL_ALL, FTAG); if (error) { mutex_exit(&spa_namespace_lock); nvlist_free(config); cmn_err(CE_NOTE, "Can not parse the config for pool '%s'", pname); return (error); } spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); vdev_free(rvd); spa_config_exit(spa, SCL_ALL, FTAG); mutex_exit(&spa_namespace_lock); nvlist_free(config); return (0); } #endif /* illumos */ #endif /* _KERNEL */ /* * Import a non-root pool into the system. */ int spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) { spa_t *spa; char *altroot = NULL; spa_load_state_t state = SPA_LOAD_IMPORT; zpool_rewind_policy_t policy; uint64_t mode = spa_mode_global; uint64_t readonly = B_FALSE; int error; nvlist_t *nvroot; nvlist_t **spares, **l2cache; uint_t nspares, nl2cache; /* * If a pool with this name exists, return failure. */ mutex_enter(&spa_namespace_lock); if (spa_lookup(pool) != NULL) { mutex_exit(&spa_namespace_lock); return (SET_ERROR(EEXIST)); } /* * Create and initialize the spa structure. */ (void) nvlist_lookup_string(props, zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); (void) nvlist_lookup_uint64(props, zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); if (readonly) mode = FREAD; spa = spa_add(pool, config, altroot); spa->spa_import_flags = flags; /* * Verbatim import - Take a pool and insert it into the namespace * as if it had been loaded at boot. */ if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { if (props != NULL) spa_configfile_set(spa, props, B_FALSE); spa_config_sync(spa, B_FALSE, B_TRUE); spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT); mutex_exit(&spa_namespace_lock); return (0); } spa_activate(spa, mode); /* * Don't start async tasks until we know everything is healthy. */ spa_async_suspend(spa); zpool_get_rewind_policy(config, &policy); if (policy.zrp_request & ZPOOL_DO_REWIND) state = SPA_LOAD_RECOVER; /* * Pass off the heavy lifting to spa_load(). Pass TRUE for mosconfig * because the user-supplied config is actually the one to trust when * doing an import. */ if (state != SPA_LOAD_RECOVER) spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; error = spa_load_best(spa, state, B_TRUE, policy.zrp_txg, policy.zrp_request); /* * Propagate anything learned while loading the pool and pass it * back to caller (i.e. rewind info, missing devices, etc). */ VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); /* * Toss any existing sparelist, as it doesn't have any validity * anymore, and conflicts with spa_has_spare(). */ if (spa->spa_spares.sav_config) { nvlist_free(spa->spa_spares.sav_config); spa->spa_spares.sav_config = NULL; spa_load_spares(spa); } if (spa->spa_l2cache.sav_config) { nvlist_free(spa->spa_l2cache.sav_config); spa->spa_l2cache.sav_config = NULL; spa_load_l2cache(spa); } VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); if (error == 0) error = spa_validate_aux(spa, nvroot, -1ULL, VDEV_ALLOC_SPARE); if (error == 0) error = spa_validate_aux(spa, nvroot, -1ULL, VDEV_ALLOC_L2CACHE); spa_config_exit(spa, SCL_ALL, FTAG); if (props != NULL) spa_configfile_set(spa, props, B_FALSE); if (error != 0 || (props && spa_writeable(spa) && (error = spa_prop_set(spa, props)))) { spa_unload(spa); spa_deactivate(spa); spa_remove(spa); mutex_exit(&spa_namespace_lock); return (error); } spa_async_resume(spa); /* * Override any spares and level 2 cache devices as specified by * the user, as these may have correct device names/devids, etc. */ if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) { if (spa->spa_spares.sav_config) VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_spares(spa); spa_config_exit(spa, SCL_ALL, FTAG); spa->spa_spares.sav_sync = B_TRUE; } if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0) { if (spa->spa_l2cache.sav_config) VERIFY(nvlist_remove(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa_load_l2cache(spa); spa_config_exit(spa, SCL_ALL, FTAG); spa->spa_l2cache.sav_sync = B_TRUE; } /* * Check for any removed devices. */ if (spa->spa_autoreplace) { spa_aux_check_removed(&spa->spa_spares); spa_aux_check_removed(&spa->spa_l2cache); } if (spa_writeable(spa)) { /* * Update the config cache to include the newly-imported pool. */ spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); } /* * It's possible that the pool was expanded while it was exported. * We kick off an async task to handle this for us. */ spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); spa_history_log_version(spa, "import"); spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT); mutex_exit(&spa_namespace_lock); #ifdef __FreeBSD__ #ifdef _KERNEL zvol_create_minors(pool); #endif #endif return (0); } nvlist_t * spa_tryimport(nvlist_t *tryconfig) { nvlist_t *config = NULL; char *poolname; spa_t *spa; uint64_t state; int error; if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) return (NULL); if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) return (NULL); /* * Create and initialize the spa structure. */ mutex_enter(&spa_namespace_lock); spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); spa_activate(spa, FREAD); /* * Pass off the heavy lifting to spa_load(). * Pass TRUE for mosconfig because the user-supplied config * is actually the one to trust when doing an import. */ error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING, B_TRUE); /* * If 'tryconfig' was at least parsable, return the current config. */ if (spa->spa_root_vdev != NULL) { config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, spa->spa_uberblock.ub_timestamp) == 0); VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); /* * If the bootfs property exists on this pool then we * copy it out so that external consumers can tell which * pools are bootable. */ if ((!error || error == EEXIST) && spa->spa_bootfs) { char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); /* * We have to play games with the name since the * pool was opened as TRYIMPORT_NAME. */ if (dsl_dsobj_to_dsname(spa_name(spa), spa->spa_bootfs, tmpname) == 0) { char *cp; char *dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); cp = strchr(tmpname, '/'); if (cp == NULL) { (void) strlcpy(dsname, tmpname, MAXPATHLEN); } else { (void) snprintf(dsname, MAXPATHLEN, "%s/%s", poolname, ++cp); } VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, dsname) == 0); kmem_free(dsname, MAXPATHLEN); } kmem_free(tmpname, MAXPATHLEN); } /* * Add the list of hot spares and level 2 cache devices. */ spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); spa_add_spares(spa, config); spa_add_l2cache(spa, config); spa_config_exit(spa, SCL_CONFIG, FTAG); } spa_unload(spa); spa_deactivate(spa); spa_remove(spa); mutex_exit(&spa_namespace_lock); return (config); } /* * Pool export/destroy * * The act of destroying or exporting a pool is very simple. We make sure there * is no more pending I/O and any references to the pool are gone. Then, we * update the pool state and sync all the labels to disk, removing the * configuration from the cache afterwards. If the 'hardforce' flag is set, then * we don't sync the labels or remove the configuration cache. */ static int spa_export_common(char *pool, int new_state, nvlist_t **oldconfig, boolean_t force, boolean_t hardforce) { spa_t *spa; if (oldconfig) *oldconfig = NULL; if (!(spa_mode_global & FWRITE)) return (SET_ERROR(EROFS)); mutex_enter(&spa_namespace_lock); if ((spa = spa_lookup(pool)) == NULL) { mutex_exit(&spa_namespace_lock); return (SET_ERROR(ENOENT)); } /* * Put a hold on the pool, drop the namespace lock, stop async tasks, * reacquire the namespace lock, and see if we can export. */ spa_open_ref(spa, FTAG); mutex_exit(&spa_namespace_lock); spa_async_suspend(spa); mutex_enter(&spa_namespace_lock); spa_close(spa, FTAG); /* * The pool will be in core if it's openable, * in which case we can modify its state. */ if (spa->spa_state != POOL_STATE_UNINITIALIZED && spa->spa_sync_on) { /* * Objsets may be open only because they're dirty, so we * have to force it to sync before checking spa_refcnt. */ txg_wait_synced(spa->spa_dsl_pool, 0); spa_evicting_os_wait(spa); /* * A pool cannot be exported or destroyed if there are active * references. If we are resetting a pool, allow references by * fault injection handlers. */ if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0 && new_state != POOL_STATE_UNINITIALIZED)) { spa_async_resume(spa); mutex_exit(&spa_namespace_lock); return (SET_ERROR(EBUSY)); } /* * A pool cannot be exported if it has an active shared spare. * This is to prevent other pools stealing the active spare * from an exported pool. At user's own will, such pool can * be forcedly exported. */ if (!force && new_state == POOL_STATE_EXPORTED && spa_has_active_shared_spare(spa)) { spa_async_resume(spa); mutex_exit(&spa_namespace_lock); return (SET_ERROR(EXDEV)); } /* * We want this to be reflected on every label, * so mark them all dirty. spa_unload() will do the * final sync that pushes these changes out. */ if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa->spa_state = new_state; spa->spa_final_txg = spa_last_synced_txg(spa) + TXG_DEFER_SIZE + 1; vdev_config_dirty(spa->spa_root_vdev); spa_config_exit(spa, SCL_ALL, FTAG); } } spa_event_notify(spa, NULL, ESC_ZFS_POOL_DESTROY); if (spa->spa_state != POOL_STATE_UNINITIALIZED) { spa_unload(spa); spa_deactivate(spa); } if (oldconfig && spa->spa_config) VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0); if (new_state != POOL_STATE_UNINITIALIZED) { if (!hardforce) spa_config_sync(spa, B_TRUE, B_TRUE); spa_remove(spa); } mutex_exit(&spa_namespace_lock); return (0); } /* * Destroy a storage pool. */ int spa_destroy(char *pool) { return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, B_FALSE, B_FALSE)); } /* * Export a storage pool. */ int spa_export(char *pool, nvlist_t **oldconfig, boolean_t force, boolean_t hardforce) { return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, force, hardforce)); } /* * Similar to spa_export(), this unloads the spa_t without actually removing it * from the namespace in any way. */ int spa_reset(char *pool) { return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, B_FALSE, B_FALSE)); } /* * ========================================================================== * Device manipulation * ========================================================================== */ /* * Add a device to a storage pool. */ int spa_vdev_add(spa_t *spa, nvlist_t *nvroot) { uint64_t txg, id; int error; vdev_t *rvd = spa->spa_root_vdev; vdev_t *vd, *tvd; nvlist_t **spares, **l2cache; uint_t nspares, nl2cache; ASSERT(spa_writeable(spa)); txg = spa_vdev_enter(spa); if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, VDEV_ALLOC_ADD)) != 0) return (spa_vdev_exit(spa, NULL, txg, error)); spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) nspares = 0; if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) != 0) nl2cache = 0; if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) return (spa_vdev_exit(spa, vd, txg, EINVAL)); if (vd->vdev_children != 0 && (error = vdev_create(vd, txg, B_FALSE)) != 0) return (spa_vdev_exit(spa, vd, txg, error)); /* * We must validate the spares and l2cache devices after checking the * children. Otherwise, vdev_inuse() will blindly overwrite the spare. */ if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) return (spa_vdev_exit(spa, vd, txg, error)); /* * Transfer each new top-level vdev from vd to rvd. */ for (int c = 0; c < vd->vdev_children; c++) { /* * Set the vdev id to the first hole, if one exists. */ for (id = 0; id < rvd->vdev_children; id++) { if (rvd->vdev_child[id]->vdev_ishole) { vdev_free(rvd->vdev_child[id]); break; } } tvd = vd->vdev_child[c]; vdev_remove_child(vd, tvd); tvd->vdev_id = id; vdev_add_child(rvd, tvd); vdev_config_dirty(tvd); } if (nspares != 0) { spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, ZPOOL_CONFIG_SPARES); spa_load_spares(spa); spa->spa_spares.sav_sync = B_TRUE; } if (nl2cache != 0) { spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, ZPOOL_CONFIG_L2CACHE); spa_load_l2cache(spa); spa->spa_l2cache.sav_sync = B_TRUE; } /* * We have to be careful when adding new vdevs to an existing pool. * If other threads start allocating from these vdevs before we * sync the config cache, and we lose power, then upon reboot we may * fail to open the pool because there are DVAs that the config cache * can't translate. Therefore, we first add the vdevs without * initializing metaslabs; sync the config cache (via spa_vdev_exit()); * and then let spa_config_update() initialize the new metaslabs. * * spa_load() checks for added-but-not-initialized vdevs, so that * if we lose power at any point in this sequence, the remaining * steps will be completed the next time we load the pool. */ (void) spa_vdev_exit(spa, vd, txg, 0); mutex_enter(&spa_namespace_lock); spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); spa_event_notify(spa, NULL, ESC_ZFS_VDEV_ADD); mutex_exit(&spa_namespace_lock); return (0); } /* * Attach a device to a mirror. The arguments are the path to any device * in the mirror, and the nvroot for the new device. If the path specifies * a device that is not mirrored, we automatically insert the mirror vdev. * * If 'replacing' is specified, the new device is intended to replace the * existing device; in this case the two devices are made into their own * mirror using the 'replacing' vdev, which is functionally identical to * the mirror vdev (it actually reuses all the same ops) but has a few * extra rules: you can't attach to it after it's been created, and upon * completion of resilvering, the first disk (the one being replaced) * is automatically detached. */ int spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing) { uint64_t txg, dtl_max_txg; vdev_t *rvd = spa->spa_root_vdev; vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; vdev_ops_t *pvops; char *oldvdpath, *newvdpath; int newvd_isspare; int error; ASSERT(spa_writeable(spa)); txg = spa_vdev_enter(spa); oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); if (oldvd == NULL) return (spa_vdev_exit(spa, NULL, txg, ENODEV)); if (!oldvd->vdev_ops->vdev_op_leaf) return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); pvd = oldvd->vdev_parent; if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, VDEV_ALLOC_ATTACH)) != 0) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); if (newrootvd->vdev_children != 1) return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); newvd = newrootvd->vdev_child[0]; if (!newvd->vdev_ops->vdev_op_leaf) return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); if ((error = vdev_create(newrootvd, txg, replacing)) != 0) return (spa_vdev_exit(spa, newrootvd, txg, error)); /* * Spares can't replace logs */ if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); if (!replacing) { /* * For attach, the only allowable parent is a mirror or the root * vdev. */ if (pvd->vdev_ops != &vdev_mirror_ops && pvd->vdev_ops != &vdev_root_ops) return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); pvops = &vdev_mirror_ops; } else { /* * Active hot spares can only be replaced by inactive hot * spares. */ if (pvd->vdev_ops == &vdev_spare_ops && oldvd->vdev_isspare && !spa_has_spare(spa, newvd->vdev_guid)) return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); /* * If the source is a hot spare, and the parent isn't already a * spare, then we want to create a new hot spare. Otherwise, we * want to create a replacing vdev. The user is not allowed to * attach to a spared vdev child unless the 'isspare' state is * the same (spare replaces spare, non-spare replaces * non-spare). */ if (pvd->vdev_ops == &vdev_replacing_ops && spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); } else if (pvd->vdev_ops == &vdev_spare_ops && newvd->vdev_isspare != oldvd->vdev_isspare) { return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); } if (newvd->vdev_isspare) pvops = &vdev_spare_ops; else pvops = &vdev_replacing_ops; } /* * Make sure the new device is big enough. */ if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); /* * The new device cannot have a higher alignment requirement * than the top-level vdev. */ if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) return (spa_vdev_exit(spa, newrootvd, txg, EDOM)); /* * If this is an in-place replacement, update oldvd's path and devid * to make it distinguishable from newvd, and unopenable from now on. */ if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { spa_strfree(oldvd->vdev_path); oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, KM_SLEEP); (void) sprintf(oldvd->vdev_path, "%s/%s", newvd->vdev_path, "old"); if (oldvd->vdev_devid != NULL) { spa_strfree(oldvd->vdev_devid); oldvd->vdev_devid = NULL; } } /* mark the device being resilvered */ newvd->vdev_resilver_txg = txg; /* * If the parent is not a mirror, or if we're replacing, insert the new * mirror/replacing/spare vdev above oldvd. */ if (pvd->vdev_ops != pvops) pvd = vdev_add_parent(oldvd, pvops); ASSERT(pvd->vdev_top->vdev_parent == rvd); ASSERT(pvd->vdev_ops == pvops); ASSERT(oldvd->vdev_parent == pvd); /* * Extract the new device from its root and add it to pvd. */ vdev_remove_child(newrootvd, newvd); newvd->vdev_id = pvd->vdev_children; newvd->vdev_crtxg = oldvd->vdev_crtxg; vdev_add_child(pvd, newvd); tvd = newvd->vdev_top; ASSERT(pvd->vdev_top == tvd); ASSERT(tvd->vdev_parent == rvd); vdev_config_dirty(tvd); /* * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account * for any dmu_sync-ed blocks. It will propagate upward when * spa_vdev_exit() calls vdev_dtl_reassess(). */ dtl_max_txg = txg + TXG_CONCURRENT_STATES; vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL, dtl_max_txg - TXG_INITIAL); if (newvd->vdev_isspare) { spa_spare_activate(newvd); spa_event_notify(spa, newvd, ESC_ZFS_VDEV_SPARE); } oldvdpath = spa_strdup(oldvd->vdev_path); newvdpath = spa_strdup(newvd->vdev_path); newvd_isspare = newvd->vdev_isspare; /* * Mark newvd's DTL dirty in this txg. */ vdev_dirty(tvd, VDD_DTL, newvd, txg); /* * Schedule the resilver to restart in the future. We do this to * ensure that dmu_sync-ed blocks have been stitched into the * respective datasets. */ dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg); if (spa->spa_bootfs) spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH); spa_event_notify(spa, newvd, ESC_ZFS_VDEV_ATTACH); /* * Commit the config */ (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); spa_history_log_internal(spa, "vdev attach", NULL, "%s vdev=%s %s vdev=%s", replacing && newvd_isspare ? "spare in" : replacing ? "replace" : "attach", newvdpath, replacing ? "for" : "to", oldvdpath); spa_strfree(oldvdpath); spa_strfree(newvdpath); return (0); } /* * Detach a device from a mirror or replacing vdev. * * If 'replace_done' is specified, only detach if the parent * is a replacing vdev. */ int spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) { uint64_t txg; int error; vdev_t *rvd = spa->spa_root_vdev; vdev_t *vd, *pvd, *cvd, *tvd; boolean_t unspare = B_FALSE; uint64_t unspare_guid = 0; char *vdpath; ASSERT(spa_writeable(spa)); txg = spa_vdev_enter(spa); vd = spa_lookup_by_guid(spa, guid, B_FALSE); if (vd == NULL) return (spa_vdev_exit(spa, NULL, txg, ENODEV)); if (!vd->vdev_ops->vdev_op_leaf) return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); pvd = vd->vdev_parent; /* * If the parent/child relationship is not as expected, don't do it. * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing * vdev that's replacing B with C. The user's intent in replacing * is to go from M(A,B) to M(A,C). If the user decides to cancel * the replace by detaching C, the expected behavior is to end up * M(A,B). But suppose that right after deciding to detach C, * the replacement of B completes. We would have M(A,C), and then * ask to detach C, which would leave us with just A -- not what * the user wanted. To prevent this, we make sure that the * parent/child relationship hasn't changed -- in this example, * that C's parent is still the replacing vdev R. */ if (pvd->vdev_guid != pguid && pguid != 0) return (spa_vdev_exit(spa, NULL, txg, EBUSY)); /* * Only 'replacing' or 'spare' vdevs can be replaced. */ if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && pvd->vdev_ops != &vdev_spare_ops) return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); ASSERT(pvd->vdev_ops != &vdev_spare_ops || spa_version(spa) >= SPA_VERSION_SPARES); /* * Only mirror, replacing, and spare vdevs support detach. */ if (pvd->vdev_ops != &vdev_replacing_ops && pvd->vdev_ops != &vdev_mirror_ops && pvd->vdev_ops != &vdev_spare_ops) return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); /* * If this device has the only valid copy of some data, * we cannot safely detach it. */ if (vdev_dtl_required(vd)) return (spa_vdev_exit(spa, NULL, txg, EBUSY)); ASSERT(pvd->vdev_children >= 2); /* * If we are detaching the second disk from a replacing vdev, then * check to see if we changed the original vdev's path to have "/old" * at the end in spa_vdev_attach(). If so, undo that change now. */ if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && vd->vdev_path != NULL) { size_t len = strlen(vd->vdev_path); for (int c = 0; c < pvd->vdev_children; c++) { cvd = pvd->vdev_child[c]; if (cvd == vd || cvd->vdev_path == NULL) continue; if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && strcmp(cvd->vdev_path + len, "/old") == 0) { spa_strfree(cvd->vdev_path); cvd->vdev_path = spa_strdup(vd->vdev_path); break; } } } /* * If we are detaching the original disk from a spare, then it implies * that the spare should become a real disk, and be removed from the * active spare list for the pool. */ if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0 && pvd->vdev_child[pvd->vdev_children - 1]->vdev_isspare) unspare = B_TRUE; /* * Erase the disk labels so the disk can be used for other things. * This must be done after all other error cases are handled, * but before we disembowel vd (so we can still do I/O to it). * But if we can't do it, don't treat the error as fatal -- * it may be that the unwritability of the disk is the reason * it's being detached! */ error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); /* * Remove vd from its parent and compact the parent's children. */ vdev_remove_child(pvd, vd); vdev_compact_children(pvd); /* * Remember one of the remaining children so we can get tvd below. */ cvd = pvd->vdev_child[pvd->vdev_children - 1]; /* * If we need to remove the remaining child from the list of hot spares, * do it now, marking the vdev as no longer a spare in the process. * We must do this before vdev_remove_parent(), because that can * change the GUID if it creates a new toplevel GUID. For a similar * reason, we must remove the spare now, in the same txg as the detach; * otherwise someone could attach a new sibling, change the GUID, and * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. */ if (unspare) { ASSERT(cvd->vdev_isspare); spa_spare_remove(cvd); unspare_guid = cvd->vdev_guid; (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); cvd->vdev_unspare = B_TRUE; } /* * If the parent mirror/replacing vdev only has one child, * the parent is no longer needed. Remove it from the tree. */ if (pvd->vdev_children == 1) { if (pvd->vdev_ops == &vdev_spare_ops) cvd->vdev_unspare = B_FALSE; vdev_remove_parent(cvd); } /* * We don't set tvd until now because the parent we just removed * may have been the previous top-level vdev. */ tvd = cvd->vdev_top; ASSERT(tvd->vdev_parent == rvd); /* * Reevaluate the parent vdev state. */ vdev_propagate_state(cvd); /* * If the 'autoexpand' property is set on the pool then automatically * try to expand the size of the pool. For example if the device we * just detached was smaller than the others, it may be possible to * add metaslabs (i.e. grow the pool). We need to reopen the vdev * first so that we can obtain the updated sizes of the leaf vdevs. */ if (spa->spa_autoexpand) { vdev_reopen(tvd); vdev_expand(tvd, txg); } vdev_config_dirty(tvd); /* * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that * vd->vdev_detached is set and free vd's DTL object in syncing context. * But first make sure we're not on any *other* txg's DTL list, to * prevent vd from being accessed after it's freed. */ vdpath = spa_strdup(vd->vdev_path); for (int t = 0; t < TXG_SIZE; t++) (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); vd->vdev_detached = B_TRUE; vdev_dirty(tvd, VDD_DTL, vd, txg); spa_event_notify(spa, vd, ESC_ZFS_VDEV_REMOVE); /* hang on to the spa before we release the lock */ spa_open_ref(spa, FTAG); error = spa_vdev_exit(spa, vd, txg, 0); spa_history_log_internal(spa, "detach", NULL, "vdev=%s", vdpath); spa_strfree(vdpath); /* * If this was the removal of the original device in a hot spare vdev, * then we want to go through and remove the device from the hot spare * list of every other pool. */ if (unspare) { spa_t *altspa = NULL; mutex_enter(&spa_namespace_lock); while ((altspa = spa_next(altspa)) != NULL) { if (altspa->spa_state != POOL_STATE_ACTIVE || altspa == spa) continue; spa_open_ref(altspa, FTAG); mutex_exit(&spa_namespace_lock); (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); mutex_enter(&spa_namespace_lock); spa_close(altspa, FTAG); } mutex_exit(&spa_namespace_lock); /* search the rest of the vdevs for spares to remove */ spa_vdev_resilver_done(spa); } /* all done with the spa; OK to release */ mutex_enter(&spa_namespace_lock); spa_close(spa, FTAG); mutex_exit(&spa_namespace_lock); return (error); } /* * Split a set of devices from their mirrors, and create a new pool from them. */ int spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, nvlist_t *props, boolean_t exp) { int error = 0; uint64_t txg, *glist; spa_t *newspa; uint_t c, children, lastlog; nvlist_t **child, *nvl, *tmp; dmu_tx_t *tx; char *altroot = NULL; vdev_t *rvd, **vml = NULL; /* vdev modify list */ boolean_t activate_slog; ASSERT(spa_writeable(spa)); txg = spa_vdev_enter(spa); /* clear the log and flush everything up to now */ activate_slog = spa_passivate_log(spa); (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); error = spa_offline_log(spa); txg = spa_vdev_config_enter(spa); if (activate_slog) spa_activate_log(spa); if (error != 0) return (spa_vdev_exit(spa, NULL, txg, error)); /* check new spa name before going any further */ if (spa_lookup(newname) != NULL) return (spa_vdev_exit(spa, NULL, txg, EEXIST)); /* * scan through all the children to ensure they're all mirrors */ if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, &children) != 0) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); /* first, check to ensure we've got the right child count */ rvd = spa->spa_root_vdev; lastlog = 0; for (c = 0; c < rvd->vdev_children; c++) { vdev_t *vd = rvd->vdev_child[c]; /* don't count the holes & logs as children */ if (vd->vdev_islog || vd->vdev_ishole) { if (lastlog == 0) lastlog = c; continue; } lastlog = 0; } if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); /* next, ensure no spare or cache devices are part of the split */ if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); /* then, loop over each vdev and validate it */ for (c = 0; c < children; c++) { uint64_t is_hole = 0; (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, &is_hole); if (is_hole != 0) { if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || spa->spa_root_vdev->vdev_child[c]->vdev_islog) { continue; } else { error = SET_ERROR(EINVAL); break; } } /* which disk is going to be split? */ if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, &glist[c]) != 0) { error = SET_ERROR(EINVAL); break; } /* look it up in the spa */ vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); if (vml[c] == NULL) { error = SET_ERROR(ENODEV); break; } /* make sure there's nothing stopping the split */ if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || vml[c]->vdev_islog || vml[c]->vdev_ishole || vml[c]->vdev_isspare || vml[c]->vdev_isl2cache || !vdev_writeable(vml[c]) || vml[c]->vdev_children != 0 || vml[c]->vdev_state != VDEV_STATE_HEALTHY || c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { error = SET_ERROR(EINVAL); break; } if (vdev_dtl_required(vml[c])) { error = SET_ERROR(EBUSY); break; } /* we need certain info from the top level */ VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, vml[c]->vdev_top->vdev_ms_array) == 0); VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, vml[c]->vdev_top->vdev_ms_shift) == 0); VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, vml[c]->vdev_top->vdev_asize) == 0); VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, vml[c]->vdev_top->vdev_ashift) == 0); /* transfer per-vdev ZAPs */ ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); VERIFY0(nvlist_add_uint64(child[c], ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); VERIFY0(nvlist_add_uint64(child[c], ZPOOL_CONFIG_VDEV_TOP_ZAP, vml[c]->vdev_parent->vdev_top_zap)); } if (error != 0) { kmem_free(vml, children * sizeof (vdev_t *)); kmem_free(glist, children * sizeof (uint64_t)); return (spa_vdev_exit(spa, NULL, txg, error)); } /* stop writers from using the disks */ for (c = 0; c < children; c++) { if (vml[c] != NULL) vml[c]->vdev_offline = B_TRUE; } vdev_reopen(spa->spa_root_vdev); /* * Temporarily record the splitting vdevs in the spa config. This * will disappear once the config is regenerated. */ VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children) == 0); kmem_free(glist, children * sizeof (uint64_t)); mutex_enter(&spa->spa_props_lock); VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl) == 0); mutex_exit(&spa->spa_props_lock); spa->spa_config_splitting = nvl; vdev_config_dirty(spa->spa_root_vdev); /* configure and create the new pool */ VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, spa_generate_guid(NULL)) == 0); VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); (void) nvlist_lookup_string(props, zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); /* add the new pool to the namespace */ newspa = spa_add(newname, config, altroot); newspa->spa_avz_action = AVZ_ACTION_REBUILD; newspa->spa_config_txg = spa->spa_config_txg; spa_set_log_state(newspa, SPA_LOG_CLEAR); /* release the spa config lock, retaining the namespace lock */ spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); if (zio_injection_enabled) zio_handle_panic_injection(spa, FTAG, 1); spa_activate(newspa, spa_mode_global); spa_async_suspend(newspa); #ifndef illumos /* mark that we are creating new spa by splitting */ newspa->spa_splitting_newspa = B_TRUE; #endif /* create the new pool from the disks of the original pool */ error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE, B_TRUE); #ifndef illumos newspa->spa_splitting_newspa = B_FALSE; #endif if (error) goto out; /* if that worked, generate a real config for the new pool */ if (newspa->spa_root_vdev != NULL) { VERIFY(nvlist_alloc(&newspa->spa_config_splitting, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, B_TRUE)); } /* set the props */ if (props != NULL) { spa_configfile_set(newspa, props, B_FALSE); error = spa_prop_set(newspa, props); if (error) goto out; } /* flush everything */ txg = spa_vdev_config_enter(newspa); vdev_config_dirty(newspa->spa_root_vdev); (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); if (zio_injection_enabled) zio_handle_panic_injection(spa, FTAG, 2); spa_async_resume(newspa); /* finally, update the original pool's config */ txg = spa_vdev_config_enter(spa); tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); error = dmu_tx_assign(tx, TXG_WAIT); if (error != 0) dmu_tx_abort(tx); for (c = 0; c < children; c++) { if (vml[c] != NULL) { vdev_split(vml[c]); if (error == 0) spa_history_log_internal(spa, "detach", tx, "vdev=%s", vml[c]->vdev_path); vdev_free(vml[c]); } } spa->spa_avz_action = AVZ_ACTION_REBUILD; vdev_config_dirty(spa->spa_root_vdev); spa->spa_config_splitting = NULL; nvlist_free(nvl); if (error == 0) dmu_tx_commit(tx); (void) spa_vdev_exit(spa, NULL, txg, 0); if (zio_injection_enabled) zio_handle_panic_injection(spa, FTAG, 3); /* split is complete; log a history record */ spa_history_log_internal(newspa, "split", NULL, "from pool %s", spa_name(spa)); kmem_free(vml, children * sizeof (vdev_t *)); /* if we're not going to mount the filesystems in userland, export */ if (exp) error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, B_FALSE, B_FALSE); return (error); out: spa_unload(newspa); spa_deactivate(newspa); spa_remove(newspa); txg = spa_vdev_config_enter(spa); /* re-online all offlined disks */ for (c = 0; c < children; c++) { if (vml[c] != NULL) vml[c]->vdev_offline = B_FALSE; } vdev_reopen(spa->spa_root_vdev); nvlist_free(spa->spa_config_splitting); spa->spa_config_splitting = NULL; (void) spa_vdev_exit(spa, NULL, txg, error); kmem_free(vml, children * sizeof (vdev_t *)); return (error); } static nvlist_t * spa_nvlist_lookup_by_guid(nvlist_t **nvpp, int count, uint64_t target_guid) { for (int i = 0; i < count; i++) { uint64_t guid; VERIFY(nvlist_lookup_uint64(nvpp[i], ZPOOL_CONFIG_GUID, &guid) == 0); if (guid == target_guid) return (nvpp[i]); } return (NULL); } static void spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count, nvlist_t *dev_to_remove) { nvlist_t **newdev = NULL; if (count > 1) newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP); for (int i = 0, j = 0; i < count; i++) { if (dev[i] == dev_to_remove) continue; VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0); } VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0); VERIFY(nvlist_add_nvlist_array(config, name, newdev, count - 1) == 0); for (int i = 0; i < count - 1; i++) nvlist_free(newdev[i]); if (count > 1) kmem_free(newdev, (count - 1) * sizeof (void *)); } /* * Evacuate the device. */ static int spa_vdev_remove_evacuate(spa_t *spa, vdev_t *vd) { uint64_t txg; int error = 0; ASSERT(MUTEX_HELD(&spa_namespace_lock)); ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); ASSERT(vd == vd->vdev_top); /* * Evacuate the device. We don't hold the config lock as writer * since we need to do I/O but we do keep the * spa_namespace_lock held. Once this completes the device * should no longer have any blocks allocated on it. */ if (vd->vdev_islog) { if (vd->vdev_stat.vs_alloc != 0) error = spa_offline_log(spa); } else { error = SET_ERROR(ENOTSUP); } if (error) return (error); /* * The evacuation succeeded. Remove any remaining MOS metadata * associated with this vdev, and wait for these changes to sync. */ ASSERT0(vd->vdev_stat.vs_alloc); txg = spa_vdev_config_enter(spa); vd->vdev_removing = B_TRUE; vdev_dirty_leaves(vd, VDD_DTL, txg); vdev_config_dirty(vd); spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); return (0); } /* * Complete the removal by cleaning up the namespace. */ static void spa_vdev_remove_from_namespace(spa_t *spa, vdev_t *vd) { vdev_t *rvd = spa->spa_root_vdev; uint64_t id = vd->vdev_id; boolean_t last_vdev = (id == (rvd->vdev_children - 1)); ASSERT(MUTEX_HELD(&spa_namespace_lock)); ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); ASSERT(vd == vd->vdev_top); /* * Only remove any devices which are empty. */ if (vd->vdev_stat.vs_alloc != 0) return; (void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); if (list_link_active(&vd->vdev_state_dirty_node)) vdev_state_clean(vd); if (list_link_active(&vd->vdev_config_dirty_node)) vdev_config_clean(vd); vdev_free(vd); if (last_vdev) { vdev_compact_children(rvd); } else { vd = vdev_alloc_common(spa, id, 0, &vdev_hole_ops); vdev_add_child(rvd, vd); } vdev_config_dirty(rvd); /* * Reassess the health of our root vdev. */ vdev_reopen(rvd); } /* * Remove a device from the pool - * * Removing a device from the vdev namespace requires several steps * and can take a significant amount of time. As a result we use * the spa_vdev_config_[enter/exit] functions which allow us to * grab and release the spa_config_lock while still holding the namespace * lock. During each step the configuration is synced out. * * Currently, this supports removing only hot spares, slogs, and level 2 ARC * devices. */ int spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare) { vdev_t *vd; sysevent_t *ev = NULL; metaslab_group_t *mg; nvlist_t **spares, **l2cache, *nv; uint64_t txg = 0; uint_t nspares, nl2cache; int error = 0; boolean_t locked = MUTEX_HELD(&spa_namespace_lock); ASSERT(spa_writeable(spa)); if (!locked) txg = spa_vdev_enter(spa); vd = spa_lookup_by_guid(spa, guid, B_FALSE); if (spa->spa_spares.sav_vdevs != NULL && nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0 && (nv = spa_nvlist_lookup_by_guid(spares, nspares, guid)) != NULL) { /* * Only remove the hot spare if it's not currently in use * in this pool. */ if (vd == NULL || unspare) { if (vd == NULL) vd = spa_lookup_by_guid(spa, guid, B_TRUE); ev = spa_event_create(spa, vd, ESC_ZFS_VDEV_REMOVE_AUX); spa_vdev_remove_aux(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares, nv); spa_load_spares(spa); spa->spa_spares.sav_sync = B_TRUE; } else { error = SET_ERROR(EBUSY); } } else if (spa->spa_l2cache.sav_vdevs != NULL && nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0 && (nv = spa_nvlist_lookup_by_guid(l2cache, nl2cache, guid)) != NULL) { /* * Cache devices can always be removed. */ vd = spa_lookup_by_guid(spa, guid, B_TRUE); ev = spa_event_create(spa, vd, ESC_ZFS_VDEV_REMOVE_AUX); spa_vdev_remove_aux(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache, nv); spa_load_l2cache(spa); spa->spa_l2cache.sav_sync = B_TRUE; } else if (vd != NULL && vd->vdev_islog) { ASSERT(!locked); ASSERT(vd == vd->vdev_top); mg = vd->vdev_mg; /* * Stop allocating from this vdev. */ metaslab_group_passivate(mg); /* * Wait for the youngest allocations and frees to sync, * and then wait for the deferral of those frees to finish. */ spa_vdev_config_exit(spa, NULL, txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG); /* * Attempt to evacuate the vdev. */ error = spa_vdev_remove_evacuate(spa, vd); txg = spa_vdev_config_enter(spa); /* * If we couldn't evacuate the vdev, unwind. */ if (error) { metaslab_group_activate(mg); return (spa_vdev_exit(spa, NULL, txg, error)); } /* * Clean up the vdev namespace. */ ev = spa_event_create(spa, vd, ESC_ZFS_VDEV_REMOVE_DEV); spa_vdev_remove_from_namespace(spa, vd); } else if (vd != NULL) { /* * Normal vdevs cannot be removed (yet). */ error = SET_ERROR(ENOTSUP); } else { /* * There is no vdev of any kind with the specified guid. */ error = SET_ERROR(ENOENT); } if (!locked) error = spa_vdev_exit(spa, NULL, txg, error); if (ev) spa_event_post(ev); return (error); } /* * Find any device that's done replacing, or a vdev marked 'unspare' that's * currently spared, so we can detach it. */ static vdev_t * spa_vdev_resilver_done_hunt(vdev_t *vd) { vdev_t *newvd, *oldvd; for (int c = 0; c < vd->vdev_children; c++) { oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); if (oldvd != NULL) return (oldvd); } /* * Check for a completed replacement. We always consider the first * vdev in the list to be the oldest vdev, and the last one to be * the newest (see spa_vdev_attach() for how that works). In * the case where the newest vdev is faulted, we will not automatically * remove it after a resilver completes. This is OK as it will require * user intervention to determine which disk the admin wishes to keep. */ if (vd->vdev_ops == &vdev_replacing_ops) { ASSERT(vd->vdev_children > 1); newvd = vd->vdev_child[vd->vdev_children - 1]; oldvd = vd->vdev_child[0]; if (vdev_dtl_empty(newvd, DTL_MISSING) && vdev_dtl_empty(newvd, DTL_OUTAGE) && !vdev_dtl_required(oldvd)) return (oldvd); } /* * Check for a completed resilver with the 'unspare' flag set. */ if (vd->vdev_ops == &vdev_spare_ops) { vdev_t *first = vd->vdev_child[0]; vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; if (last->vdev_unspare) { oldvd = first; newvd = last; } else if (first->vdev_unspare) { oldvd = last; newvd = first; } else { oldvd = NULL; } if (oldvd != NULL && vdev_dtl_empty(newvd, DTL_MISSING) && vdev_dtl_empty(newvd, DTL_OUTAGE) && !vdev_dtl_required(oldvd)) return (oldvd); /* * If there are more than two spares attached to a disk, * and those spares are not required, then we want to * attempt to free them up now so that they can be used * by other pools. Once we're back down to a single * disk+spare, we stop removing them. */ if (vd->vdev_children > 2) { newvd = vd->vdev_child[1]; if (newvd->vdev_isspare && last->vdev_isspare && vdev_dtl_empty(last, DTL_MISSING) && vdev_dtl_empty(last, DTL_OUTAGE) && !vdev_dtl_required(newvd)) return (newvd); } } return (NULL); } static void spa_vdev_resilver_done(spa_t *spa) { vdev_t *vd, *pvd, *ppvd; uint64_t guid, sguid, pguid, ppguid; spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { pvd = vd->vdev_parent; ppvd = pvd->vdev_parent; guid = vd->vdev_guid; pguid = pvd->vdev_guid; ppguid = ppvd->vdev_guid; sguid = 0; /* * If we have just finished replacing a hot spared device, then * we need to detach the parent's first child (the original hot * spare) as well. */ if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && ppvd->vdev_children == 2) { ASSERT(pvd->vdev_ops == &vdev_replacing_ops); sguid = ppvd->vdev_child[1]->vdev_guid; } ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); spa_config_exit(spa, SCL_ALL, FTAG); if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) return; if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) return; spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); } spa_config_exit(spa, SCL_ALL, FTAG); } /* * Update the stored path or FRU for this vdev. */ int spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, boolean_t ispath) { vdev_t *vd; boolean_t sync = B_FALSE; ASSERT(spa_writeable(spa)); spa_vdev_state_enter(spa, SCL_ALL); if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) return (spa_vdev_state_exit(spa, NULL, ENOENT)); if (!vd->vdev_ops->vdev_op_leaf) return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); if (ispath) { if (strcmp(value, vd->vdev_path) != 0) { spa_strfree(vd->vdev_path); vd->vdev_path = spa_strdup(value); sync = B_TRUE; } } else { if (vd->vdev_fru == NULL) { vd->vdev_fru = spa_strdup(value); sync = B_TRUE; } else if (strcmp(value, vd->vdev_fru) != 0) { spa_strfree(vd->vdev_fru); vd->vdev_fru = spa_strdup(value); sync = B_TRUE; } } return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); } int spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) { return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); } int spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) { return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); } /* * ========================================================================== * SPA Scanning * ========================================================================== */ int spa_scan_stop(spa_t *spa) { ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); if (dsl_scan_resilvering(spa->spa_dsl_pool)) return (SET_ERROR(EBUSY)); return (dsl_scan_cancel(spa->spa_dsl_pool)); } int spa_scan(spa_t *spa, pool_scan_func_t func) { ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) return (SET_ERROR(ENOTSUP)); /* * If a resilver was requested, but there is no DTL on a * writeable leaf device, we have nothing to do. */ if (func == POOL_SCAN_RESILVER && !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); return (0); } return (dsl_scan(spa->spa_dsl_pool, func)); } /* * ========================================================================== * SPA async task processing * ========================================================================== */ static void spa_async_remove(spa_t *spa, vdev_t *vd) { if (vd->vdev_remove_wanted) { vd->vdev_remove_wanted = B_FALSE; vd->vdev_delayed_close = B_FALSE; vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); /* * We want to clear the stats, but we don't want to do a full * vdev_clear() as that will cause us to throw away * degraded/faulted state as well as attempt to reopen the * device, all of which is a waste. */ vd->vdev_stat.vs_read_errors = 0; vd->vdev_stat.vs_write_errors = 0; vd->vdev_stat.vs_checksum_errors = 0; vdev_state_dirty(vd->vdev_top); /* Tell userspace that the vdev is gone. */ zfs_post_remove(spa, vd); } for (int c = 0; c < vd->vdev_children; c++) spa_async_remove(spa, vd->vdev_child[c]); } static void spa_async_probe(spa_t *spa, vdev_t *vd) { if (vd->vdev_probe_wanted) { vd->vdev_probe_wanted = B_FALSE; vdev_reopen(vd); /* vdev_open() does the actual probe */ } for (int c = 0; c < vd->vdev_children; c++) spa_async_probe(spa, vd->vdev_child[c]); } static void spa_async_autoexpand(spa_t *spa, vdev_t *vd) { sysevent_id_t eid; nvlist_t *attr; char *physpath; if (!spa->spa_autoexpand) return; for (int c = 0; c < vd->vdev_children; c++) { vdev_t *cvd = vd->vdev_child[c]; spa_async_autoexpand(spa, cvd); } if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) return; physpath = kmem_zalloc(MAXPATHLEN, KM_SLEEP); (void) snprintf(physpath, MAXPATHLEN, "/devices%s", vd->vdev_physpath); VERIFY(nvlist_alloc(&attr, NV_UNIQUE_NAME, KM_SLEEP) == 0); VERIFY(nvlist_add_string(attr, DEV_PHYS_PATH, physpath) == 0); (void) ddi_log_sysevent(zfs_dip, SUNW_VENDOR, EC_DEV_STATUS, ESC_ZFS_VDEV_AUTOEXPAND, attr, &eid, DDI_SLEEP); nvlist_free(attr); kmem_free(physpath, MAXPATHLEN); } static void spa_async_thread(void *arg) { spa_t *spa = arg; int tasks; ASSERT(spa->spa_sync_on); mutex_enter(&spa->spa_async_lock); tasks = spa->spa_async_tasks; spa->spa_async_tasks &= SPA_ASYNC_REMOVE; mutex_exit(&spa->spa_async_lock); /* * See if the config needs to be updated. */ if (tasks & SPA_ASYNC_CONFIG_UPDATE) { uint64_t old_space, new_space; mutex_enter(&spa_namespace_lock); old_space = metaslab_class_get_space(spa_normal_class(spa)); spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); new_space = metaslab_class_get_space(spa_normal_class(spa)); mutex_exit(&spa_namespace_lock); /* * If the pool grew as a result of the config update, * then log an internal history event. */ if (new_space != old_space) { spa_history_log_internal(spa, "vdev online", NULL, "pool '%s' size: %llu(+%llu)", spa_name(spa), new_space, new_space - old_space); } } if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); spa_async_autoexpand(spa, spa->spa_root_vdev); spa_config_exit(spa, SCL_CONFIG, FTAG); } /* * See if any devices need to be probed. */ if (tasks & SPA_ASYNC_PROBE) { spa_vdev_state_enter(spa, SCL_NONE); spa_async_probe(spa, spa->spa_root_vdev); (void) spa_vdev_state_exit(spa, NULL, 0); } /* * If any devices are done replacing, detach them. */ if (tasks & SPA_ASYNC_RESILVER_DONE) spa_vdev_resilver_done(spa); /* * Kick off a resilver. */ if (tasks & SPA_ASYNC_RESILVER) dsl_resilver_restart(spa->spa_dsl_pool, 0); /* * Let the world know that we're done. */ mutex_enter(&spa->spa_async_lock); spa->spa_async_thread = NULL; cv_broadcast(&spa->spa_async_cv); mutex_exit(&spa->spa_async_lock); thread_exit(); } static void spa_async_thread_vd(void *arg) { spa_t *spa = arg; int tasks; ASSERT(spa->spa_sync_on); mutex_enter(&spa->spa_async_lock); tasks = spa->spa_async_tasks; retry: spa->spa_async_tasks &= ~SPA_ASYNC_REMOVE; mutex_exit(&spa->spa_async_lock); /* * See if any devices need to be marked REMOVED. */ if (tasks & SPA_ASYNC_REMOVE) { spa_vdev_state_enter(spa, SCL_NONE); spa_async_remove(spa, spa->spa_root_vdev); for (int i = 0; i < spa->spa_l2cache.sav_count; i++) spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); for (int i = 0; i < spa->spa_spares.sav_count; i++) spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); (void) spa_vdev_state_exit(spa, NULL, 0); } /* * Let the world know that we're done. */ mutex_enter(&spa->spa_async_lock); tasks = spa->spa_async_tasks; if ((tasks & SPA_ASYNC_REMOVE) != 0) goto retry; spa->spa_async_thread_vd = NULL; cv_broadcast(&spa->spa_async_cv); mutex_exit(&spa->spa_async_lock); thread_exit(); } void spa_async_suspend(spa_t *spa) { mutex_enter(&spa->spa_async_lock); spa->spa_async_suspended++; while (spa->spa_async_thread != NULL && spa->spa_async_thread_vd != NULL) cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); mutex_exit(&spa->spa_async_lock); } void spa_async_resume(spa_t *spa) { mutex_enter(&spa->spa_async_lock); ASSERT(spa->spa_async_suspended != 0); spa->spa_async_suspended--; mutex_exit(&spa->spa_async_lock); } static boolean_t spa_async_tasks_pending(spa_t *spa) { uint_t non_config_tasks; uint_t config_task; boolean_t config_task_suspended; non_config_tasks = spa->spa_async_tasks & ~(SPA_ASYNC_CONFIG_UPDATE | SPA_ASYNC_REMOVE); config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; if (spa->spa_ccw_fail_time == 0) { config_task_suspended = B_FALSE; } else { config_task_suspended = (gethrtime() - spa->spa_ccw_fail_time) < (zfs_ccw_retry_interval * NANOSEC); } return (non_config_tasks || (config_task && !config_task_suspended)); } static void spa_async_dispatch(spa_t *spa) { mutex_enter(&spa->spa_async_lock); if (spa_async_tasks_pending(spa) && !spa->spa_async_suspended && spa->spa_async_thread == NULL && rootdir != NULL) spa->spa_async_thread = thread_create(NULL, 0, spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); mutex_exit(&spa->spa_async_lock); } static void spa_async_dispatch_vd(spa_t *spa) { mutex_enter(&spa->spa_async_lock); if ((spa->spa_async_tasks & SPA_ASYNC_REMOVE) != 0 && !spa->spa_async_suspended && spa->spa_async_thread_vd == NULL && rootdir != NULL) spa->spa_async_thread_vd = thread_create(NULL, 0, spa_async_thread_vd, spa, 0, &p0, TS_RUN, maxclsyspri); mutex_exit(&spa->spa_async_lock); } void spa_async_request(spa_t *spa, int task) { zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); mutex_enter(&spa->spa_async_lock); spa->spa_async_tasks |= task; mutex_exit(&spa->spa_async_lock); spa_async_dispatch_vd(spa); } /* * ========================================================================== * SPA syncing routines * ========================================================================== */ static int bpobj_enqueue_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) { bpobj_t *bpo = arg; bpobj_enqueue(bpo, bp, tx); return (0); } static int spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) { zio_t *zio = arg; zio_nowait(zio_free_sync(zio, zio->io_spa, dmu_tx_get_txg(tx), bp, BP_GET_PSIZE(bp), zio->io_flags)); return (0); } /* * Note: this simple function is not inlined to make it easier to dtrace the * amount of time spent syncing frees. */ static void spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) { zio_t *zio = zio_root(spa, NULL, NULL, 0); bplist_iterate(bpl, spa_free_sync_cb, zio, tx); VERIFY(zio_wait(zio) == 0); } /* * Note: this simple function is not inlined to make it easier to dtrace the * amount of time spent syncing deferred frees. */ static void spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) { zio_t *zio = zio_root(spa, NULL, NULL, 0); VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, spa_free_sync_cb, zio, tx), ==, 0); VERIFY0(zio_wait(zio)); } static void spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) { char *packed = NULL; size_t bufsize; size_t nvsize = 0; dmu_buf_t *db; VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); /* * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration * information. This avoids the dmu_buf_will_dirty() path and * saves us a pre-read to get data we don't actually care about. */ bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); packed = kmem_alloc(bufsize, KM_SLEEP); VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, KM_SLEEP) == 0); bzero(packed + nvsize, bufsize - nvsize); dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); kmem_free(packed, bufsize); VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); dmu_buf_will_dirty(db, tx); *(uint64_t *)db->db_data = nvsize; dmu_buf_rele(db, FTAG); } static void spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, const char *config, const char *entry) { nvlist_t *nvroot; nvlist_t **list; int i; if (!sav->sav_sync) return; /* * Update the MOS nvlist describing the list of available devices. * spa_validate_aux() will have already made sure this nvlist is * valid and the vdevs are labeled appropriately. */ if (sav->sav_object == 0) { sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); VERIFY(zap_update(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, &sav->sav_object, tx) == 0); } VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); if (sav->sav_count == 0) { VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); } else { list = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); for (i = 0; i < sav->sav_count; i++) list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_FALSE, VDEV_CONFIG_L2CACHE); VERIFY(nvlist_add_nvlist_array(nvroot, config, list, sav->sav_count) == 0); for (i = 0; i < sav->sav_count; i++) nvlist_free(list[i]); kmem_free(list, sav->sav_count * sizeof (void *)); } spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); nvlist_free(nvroot); sav->sav_sync = B_FALSE; } /* * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. * The all-vdev ZAP must be empty. */ static void spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) { spa_t *spa = vd->vdev_spa; if (vd->vdev_top_zap != 0) { VERIFY0(zap_add_int(spa->spa_meta_objset, avz, vd->vdev_top_zap, tx)); } if (vd->vdev_leaf_zap != 0) { VERIFY0(zap_add_int(spa->spa_meta_objset, avz, vd->vdev_leaf_zap, tx)); } for (uint64_t i = 0; i < vd->vdev_children; i++) { spa_avz_build(vd->vdev_child[i], avz, tx); } } static void spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) { nvlist_t *config; /* * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, * its config may not be dirty but we still need to build per-vdev ZAPs. * Similarly, if the pool is being assembled (e.g. after a split), we * need to rebuild the AVZ although the config may not be dirty. */ if (list_is_empty(&spa->spa_config_dirty_list) && spa->spa_avz_action == AVZ_ACTION_NONE) return; spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || spa->spa_avz_action == AVZ_ACTION_INITIALIZE || spa->spa_all_vdev_zaps != 0); if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { /* Make and build the new AVZ */ uint64_t new_avz = zap_create(spa->spa_meta_objset, DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); spa_avz_build(spa->spa_root_vdev, new_avz, tx); /* Diff old AVZ with new one */ zap_cursor_t zc; zap_attribute_t za; for (zap_cursor_init(&zc, spa->spa_meta_objset, spa->spa_all_vdev_zaps); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { uint64_t vdzap = za.za_first_integer; if (zap_lookup_int(spa->spa_meta_objset, new_avz, vdzap) == ENOENT) { /* * ZAP is listed in old AVZ but not in new one; * destroy it */ VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, tx)); } } zap_cursor_fini(&zc); /* Destroy the old AVZ */ VERIFY0(zap_destroy(spa->spa_meta_objset, spa->spa_all_vdev_zaps, tx)); /* Replace the old AVZ in the dir obj with the new one */ VERIFY0(zap_update(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, sizeof (new_avz), 1, &new_avz, tx)); spa->spa_all_vdev_zaps = new_avz; } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { zap_cursor_t zc; zap_attribute_t za; /* Walk through the AVZ and destroy all listed ZAPs */ for (zap_cursor_init(&zc, spa->spa_meta_objset, spa->spa_all_vdev_zaps); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { uint64_t zap = za.za_first_integer; VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); } zap_cursor_fini(&zc); /* Destroy and unlink the AVZ itself */ VERIFY0(zap_destroy(spa->spa_meta_objset, spa->spa_all_vdev_zaps, tx)); VERIFY0(zap_remove(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); spa->spa_all_vdev_zaps = 0; } if (spa->spa_all_vdev_zaps == 0) { spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx); } spa->spa_avz_action = AVZ_ACTION_NONE; /* Create ZAPs for vdevs that don't have them. */ vdev_construct_zaps(spa->spa_root_vdev, tx); config = spa_config_generate(spa, spa->spa_root_vdev, dmu_tx_get_txg(tx), B_FALSE); /* * If we're upgrading the spa version then make sure that * the config object gets updated with the correct version. */ if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa->spa_uberblock.ub_version); spa_config_exit(spa, SCL_STATE, FTAG); nvlist_free(spa->spa_config_syncing); spa->spa_config_syncing = config; spa_sync_nvlist(spa, spa->spa_config_object, config, tx); } static void spa_sync_version(void *arg, dmu_tx_t *tx) { uint64_t *versionp = arg; uint64_t version = *versionp; spa_t *spa = dmu_tx_pool(tx)->dp_spa; /* * Setting the version is special cased when first creating the pool. */ ASSERT(tx->tx_txg != TXG_INITIAL); ASSERT(SPA_VERSION_IS_SUPPORTED(version)); ASSERT(version >= spa_version(spa)); spa->spa_uberblock.ub_version = version; vdev_config_dirty(spa->spa_root_vdev); spa_history_log_internal(spa, "set", tx, "version=%lld", version); } /* * Set zpool properties. */ static void spa_sync_props(void *arg, dmu_tx_t *tx) { nvlist_t *nvp = arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; objset_t *mos = spa->spa_meta_objset; nvpair_t *elem = NULL; mutex_enter(&spa->spa_props_lock); while ((elem = nvlist_next_nvpair(nvp, elem))) { uint64_t intval; char *strval, *fname; zpool_prop_t prop; const char *propname; zprop_type_t proptype; spa_feature_t fid; switch (prop = zpool_name_to_prop(nvpair_name(elem))) { case ZPROP_INVAL: /* * We checked this earlier in spa_prop_validate(). */ ASSERT(zpool_prop_feature(nvpair_name(elem))); fname = strchr(nvpair_name(elem), '@') + 1; VERIFY0(zfeature_lookup_name(fname, &fid)); spa_feature_enable(spa, fid, tx); spa_history_log_internal(spa, "set", tx, "%s=enabled", nvpair_name(elem)); break; case ZPOOL_PROP_VERSION: intval = fnvpair_value_uint64(elem); /* * The version is synced seperatly before other * properties and should be correct by now. */ ASSERT3U(spa_version(spa), >=, intval); break; case ZPOOL_PROP_ALTROOT: /* * 'altroot' is a non-persistent property. It should * have been set temporarily at creation or import time. */ ASSERT(spa->spa_root != NULL); break; case ZPOOL_PROP_READONLY: case ZPOOL_PROP_CACHEFILE: /* * 'readonly' and 'cachefile' are also non-persisitent * properties. */ break; case ZPOOL_PROP_COMMENT: strval = fnvpair_value_string(elem); if (spa->spa_comment != NULL) spa_strfree(spa->spa_comment); spa->spa_comment = spa_strdup(strval); /* * We need to dirty the configuration on all the vdevs * so that their labels get updated. It's unnecessary * to do this for pool creation since the vdev's * configuratoin has already been dirtied. */ if (tx->tx_txg != TXG_INITIAL) vdev_config_dirty(spa->spa_root_vdev); spa_history_log_internal(spa, "set", tx, "%s=%s", nvpair_name(elem), strval); break; default: /* * Set pool property values in the poolprops mos object. */ if (spa->spa_pool_props_object == 0) { spa->spa_pool_props_object = zap_create_link(mos, DMU_OT_POOL_PROPS, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, tx); } /* normalize the property name */ propname = zpool_prop_to_name(prop); proptype = zpool_prop_get_type(prop); if (nvpair_type(elem) == DATA_TYPE_STRING) { ASSERT(proptype == PROP_TYPE_STRING); strval = fnvpair_value_string(elem); VERIFY0(zap_update(mos, spa->spa_pool_props_object, propname, 1, strlen(strval) + 1, strval, tx)); spa_history_log_internal(spa, "set", tx, "%s=%s", nvpair_name(elem), strval); } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { intval = fnvpair_value_uint64(elem); if (proptype == PROP_TYPE_INDEX) { const char *unused; VERIFY0(zpool_prop_index_to_string( prop, intval, &unused)); } VERIFY0(zap_update(mos, spa->spa_pool_props_object, propname, 8, 1, &intval, tx)); spa_history_log_internal(spa, "set", tx, "%s=%lld", nvpair_name(elem), intval); } else { ASSERT(0); /* not allowed */ } switch (prop) { case ZPOOL_PROP_DELEGATION: spa->spa_delegation = intval; break; case ZPOOL_PROP_BOOTFS: spa->spa_bootfs = intval; break; case ZPOOL_PROP_FAILUREMODE: spa->spa_failmode = intval; break; case ZPOOL_PROP_AUTOEXPAND: spa->spa_autoexpand = intval; if (tx->tx_txg != TXG_INITIAL) spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); break; case ZPOOL_PROP_DEDUPDITTO: spa->spa_dedup_ditto = intval; break; default: break; } } } mutex_exit(&spa->spa_props_lock); } /* * Perform one-time upgrade on-disk changes. spa_version() does not * reflect the new version this txg, so there must be no changes this * txg to anything that the upgrade code depends on after it executes. * Therefore this must be called after dsl_pool_sync() does the sync * tasks. */ static void spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) { dsl_pool_t *dp = spa->spa_dsl_pool; ASSERT(spa->spa_sync_pass == 1); rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { dsl_pool_create_origin(dp, tx); /* Keeping the origin open increases spa_minref */ spa->spa_minref += 3; } if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { dsl_pool_upgrade_clones(dp, tx); } if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { dsl_pool_upgrade_dir_clones(dp, tx); /* Keeping the freedir open increases spa_minref */ spa->spa_minref += 3; } if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { spa_feature_create_zap_objects(spa, tx); } /* * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable * when possibility to use lz4 compression for metadata was added * Old pools that have this feature enabled must be upgraded to have * this feature active */ if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { boolean_t lz4_en = spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS); boolean_t lz4_ac = spa_feature_is_active(spa, SPA_FEATURE_LZ4_COMPRESS); if (lz4_en && !lz4_ac) spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); } /* * If we haven't written the salt, do so now. Note that the * feature may not be activated yet, but that's fine since * the presence of this ZAP entry is backwards compatible. */ if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT) == ENOENT) { VERIFY0(zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, sizeof (spa->spa_cksum_salt.zcs_bytes), spa->spa_cksum_salt.zcs_bytes, tx)); } rrw_exit(&dp->dp_config_rwlock, FTAG); } /* * Sync the specified transaction group. New blocks may be dirtied as * part of the process, so we iterate until it converges. */ void spa_sync(spa_t *spa, uint64_t txg) { dsl_pool_t *dp = spa->spa_dsl_pool; objset_t *mos = spa->spa_meta_objset; bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; vdev_t *rvd = spa->spa_root_vdev; vdev_t *vd; dmu_tx_t *tx; int error; uint32_t max_queue_depth = zfs_vdev_async_write_max_active * zfs_vdev_queue_depth_pct / 100; VERIFY(spa_writeable(spa)); /* * Lock out configuration changes. */ spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); spa->spa_syncing_txg = txg; spa->spa_sync_pass = 0; mutex_enter(&spa->spa_alloc_lock); VERIFY0(avl_numnodes(&spa->spa_alloc_tree)); mutex_exit(&spa->spa_alloc_lock); /* * If there are any pending vdev state changes, convert them * into config changes that go out with this transaction group. */ spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); while (list_head(&spa->spa_state_dirty_list) != NULL) { /* * We need the write lock here because, for aux vdevs, * calling vdev_config_dirty() modifies sav_config. * This is ugly and will become unnecessary when we * eliminate the aux vdev wart by integrating all vdevs * into the root vdev tree. */ spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { vdev_state_clean(vd); vdev_config_dirty(vd); } spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); } spa_config_exit(spa, SCL_STATE, FTAG); tx = dmu_tx_create_assigned(dp, txg); spa->spa_sync_starttime = gethrtime(); #ifdef illumos VERIFY(cyclic_reprogram(spa->spa_deadman_cycid, spa->spa_sync_starttime + spa->spa_deadman_synctime)); #else /* !illumos */ #ifdef _KERNEL callout_schedule(&spa->spa_deadman_cycid, hz * spa->spa_deadman_synctime / NANOSEC); #endif #endif /* illumos */ /* * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, * set spa_deflate if we have no raid-z vdevs. */ if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { int i; for (i = 0; i < rvd->vdev_children; i++) { vd = rvd->vdev_child[i]; if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) break; } if (i == rvd->vdev_children) { spa->spa_deflate = TRUE; VERIFY(0 == zap_add(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, sizeof (uint64_t), 1, &spa->spa_deflate, tx)); } } /* * Set the top-level vdev's max queue depth. Evaluate each * top-level's async write queue depth in case it changed. * The max queue depth will not change in the middle of syncing * out this txg. */ uint64_t queue_depth_total = 0; for (int c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; metaslab_group_t *mg = tvd->vdev_mg; if (mg == NULL || mg->mg_class != spa_normal_class(spa) || !metaslab_group_initialized(mg)) continue; /* * It is safe to do a lock-free check here because only async * allocations look at mg_max_alloc_queue_depth, and async * allocations all happen from spa_sync(). */ ASSERT0(refcount_count(&mg->mg_alloc_queue_depth)); mg->mg_max_alloc_queue_depth = max_queue_depth; queue_depth_total += mg->mg_max_alloc_queue_depth; } metaslab_class_t *mc = spa_normal_class(spa); ASSERT0(refcount_count(&mc->mc_alloc_slots)); mc->mc_alloc_max_slots = queue_depth_total; mc->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; ASSERT3U(mc->mc_alloc_max_slots, <=, max_queue_depth * rvd->vdev_children); /* * Iterate to convergence. */ do { int pass = ++spa->spa_sync_pass; spa_sync_config_object(spa, tx); spa_sync_aux_dev(spa, &spa->spa_spares, tx, ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); spa_errlog_sync(spa, txg); dsl_pool_sync(dp, txg); if (pass < zfs_sync_pass_deferred_free) { spa_sync_frees(spa, free_bpl, tx); } else { /* * We can not defer frees in pass 1, because * we sync the deferred frees later in pass 1. */ ASSERT3U(pass, >, 1); bplist_iterate(free_bpl, bpobj_enqueue_cb, &spa->spa_deferred_bpobj, tx); } ddt_sync(spa, txg); dsl_scan_sync(dp, tx); while (vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) vdev_sync(vd, txg); if (pass == 1) { spa_sync_upgrades(spa, tx); ASSERT3U(txg, >=, spa->spa_uberblock.ub_rootbp.blk_birth); /* * Note: We need to check if the MOS is dirty * because we could have marked the MOS dirty * without updating the uberblock (e.g. if we * have sync tasks but no dirty user data). We * need to check the uberblock's rootbp because * it is updated if we have synced out dirty * data (though in this case the MOS will most * likely also be dirty due to second order * effects, we don't want to rely on that here). */ if (spa->spa_uberblock.ub_rootbp.blk_birth < txg && !dmu_objset_is_dirty(mos, txg)) { /* * Nothing changed on the first pass, * therefore this TXG is a no-op. Avoid * syncing deferred frees, so that we * can keep this TXG as a no-op. */ ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); break; } spa_sync_deferred_frees(spa, tx); } } while (dmu_objset_is_dirty(mos, txg)); if (!list_is_empty(&spa->spa_config_dirty_list)) { /* * Make sure that the number of ZAPs for all the vdevs matches * the number of ZAPs in the per-vdev ZAP list. This only gets * called if the config is dirty; otherwise there may be * outstanding AVZ operations that weren't completed in * spa_sync_config_object. */ uint64_t all_vdev_zap_entry_count; ASSERT0(zap_count(spa->spa_meta_objset, spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, all_vdev_zap_entry_count); } /* * Rewrite the vdev configuration (which includes the uberblock) * to commit the transaction group. * * If there are no dirty vdevs, we sync the uberblock to a few * random top-level vdevs that are known to be visible in the * config cache (see spa_vdev_add() for a complete description). * If there *are* dirty vdevs, sync the uberblock to all vdevs. */ for (;;) { /* * We hold SCL_STATE to prevent vdev open/close/etc. * while we're attempting to write the vdev labels. */ spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); if (list_is_empty(&spa->spa_config_dirty_list)) { vdev_t *svd[SPA_DVAS_PER_BP]; int svdcount = 0; int children = rvd->vdev_children; int c0 = spa_get_random(children); for (int c = 0; c < children; c++) { vd = rvd->vdev_child[(c0 + c) % children]; if (vd->vdev_ms_array == 0 || vd->vdev_islog) continue; svd[svdcount++] = vd; if (svdcount == SPA_DVAS_PER_BP) break; } error = vdev_config_sync(svd, svdcount, txg); } else { error = vdev_config_sync(rvd->vdev_child, rvd->vdev_children, txg); } if (error == 0) spa->spa_last_synced_guid = rvd->vdev_guid; spa_config_exit(spa, SCL_STATE, FTAG); if (error == 0) break; zio_suspend(spa, NULL); zio_resume_wait(spa); } dmu_tx_commit(tx); #ifdef illumos VERIFY(cyclic_reprogram(spa->spa_deadman_cycid, CY_INFINITY)); #else /* !illumos */ #ifdef _KERNEL callout_drain(&spa->spa_deadman_cycid); #endif #endif /* illumos */ /* * Clear the dirty config list. */ while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) vdev_config_clean(vd); /* * Now that the new config has synced transactionally, * let it become visible to the config cache. */ if (spa->spa_config_syncing != NULL) { spa_config_set(spa, spa->spa_config_syncing); spa->spa_config_txg = txg; spa->spa_config_syncing = NULL; } dsl_pool_sync_done(dp, txg); mutex_enter(&spa->spa_alloc_lock); VERIFY0(avl_numnodes(&spa->spa_alloc_tree)); mutex_exit(&spa->spa_alloc_lock); /* * Update usable space statistics. */ while (vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) vdev_sync_done(vd, txg); spa_update_dspace(spa); /* * It had better be the case that we didn't dirty anything * since vdev_config_sync(). */ ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); spa->spa_sync_pass = 0; /* * Update the last synced uberblock here. We want to do this at * the end of spa_sync() so that consumers of spa_last_synced_txg() * will be guaranteed that all the processing associated with * that txg has been completed. */ spa->spa_ubsync = spa->spa_uberblock; spa_config_exit(spa, SCL_CONFIG, FTAG); spa_handle_ignored_writes(spa); /* * If any async tasks have been requested, kick them off. */ spa_async_dispatch(spa); spa_async_dispatch_vd(spa); } /* * Sync all pools. We don't want to hold the namespace lock across these * operations, so we take a reference on the spa_t and drop the lock during the * sync. */ void spa_sync_allpools(void) { spa_t *spa = NULL; mutex_enter(&spa_namespace_lock); while ((spa = spa_next(spa)) != NULL) { if (spa_state(spa) != POOL_STATE_ACTIVE || !spa_writeable(spa) || spa_suspended(spa)) continue; spa_open_ref(spa, FTAG); mutex_exit(&spa_namespace_lock); txg_wait_synced(spa_get_dsl(spa), 0); mutex_enter(&spa_namespace_lock); spa_close(spa, FTAG); } mutex_exit(&spa_namespace_lock); } /* * ========================================================================== * Miscellaneous routines * ========================================================================== */ /* * Remove all pools in the system. */ void spa_evict_all(void) { spa_t *spa; /* * Remove all cached state. All pools should be closed now, * so every spa in the AVL tree should be unreferenced. */ mutex_enter(&spa_namespace_lock); while ((spa = spa_next(NULL)) != NULL) { /* * Stop async tasks. The async thread may need to detach * a device that's been replaced, which requires grabbing * spa_namespace_lock, so we must drop it here. */ spa_open_ref(spa, FTAG); mutex_exit(&spa_namespace_lock); spa_async_suspend(spa); mutex_enter(&spa_namespace_lock); spa_close(spa, FTAG); if (spa->spa_state != POOL_STATE_UNINITIALIZED) { spa_unload(spa); spa_deactivate(spa); } spa_remove(spa); } mutex_exit(&spa_namespace_lock); } vdev_t * spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) { vdev_t *vd; int i; if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) return (vd); if (aux) { for (i = 0; i < spa->spa_l2cache.sav_count; i++) { vd = spa->spa_l2cache.sav_vdevs[i]; if (vd->vdev_guid == guid) return (vd); } for (i = 0; i < spa->spa_spares.sav_count; i++) { vd = spa->spa_spares.sav_vdevs[i]; if (vd->vdev_guid == guid) return (vd); } } return (NULL); } void spa_upgrade(spa_t *spa, uint64_t version) { ASSERT(spa_writeable(spa)); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); /* * This should only be called for a non-faulted pool, and since a * future version would result in an unopenable pool, this shouldn't be * possible. */ ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); ASSERT3U(version, >=, spa->spa_uberblock.ub_version); spa->spa_uberblock.ub_version = version; vdev_config_dirty(spa->spa_root_vdev); spa_config_exit(spa, SCL_ALL, FTAG); txg_wait_synced(spa_get_dsl(spa), 0); } boolean_t spa_has_spare(spa_t *spa, uint64_t guid) { int i; uint64_t spareguid; spa_aux_vdev_t *sav = &spa->spa_spares; for (i = 0; i < sav->sav_count; i++) if (sav->sav_vdevs[i]->vdev_guid == guid) return (B_TRUE); for (i = 0; i < sav->sav_npending; i++) { if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, &spareguid) == 0 && spareguid == guid) return (B_TRUE); } return (B_FALSE); } /* * Check if a pool has an active shared spare device. * Note: reference count of an active spare is 2, as a spare and as a replace */ static boolean_t spa_has_active_shared_spare(spa_t *spa) { int i, refcnt; uint64_t pool; spa_aux_vdev_t *sav = &spa->spa_spares; for (i = 0; i < sav->sav_count; i++) { if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, &refcnt) && pool != 0ULL && pool == spa_guid(spa) && refcnt > 2) return (B_TRUE); } return (B_FALSE); } static sysevent_t * spa_event_create(spa_t *spa, vdev_t *vd, const char *name) { sysevent_t *ev = NULL; #ifdef _KERNEL sysevent_attr_list_t *attr = NULL; sysevent_value_t value; ev = sysevent_alloc(EC_ZFS, (char *)name, SUNW_KERN_PUB "zfs", SE_SLEEP); ASSERT(ev != NULL); value.value_type = SE_DATA_TYPE_STRING; value.value.sv_string = spa_name(spa); if (sysevent_add_attr(&attr, ZFS_EV_POOL_NAME, &value, SE_SLEEP) != 0) goto done; value.value_type = SE_DATA_TYPE_UINT64; value.value.sv_uint64 = spa_guid(spa); if (sysevent_add_attr(&attr, ZFS_EV_POOL_GUID, &value, SE_SLEEP) != 0) goto done; if (vd) { value.value_type = SE_DATA_TYPE_UINT64; value.value.sv_uint64 = vd->vdev_guid; if (sysevent_add_attr(&attr, ZFS_EV_VDEV_GUID, &value, SE_SLEEP) != 0) goto done; if (vd->vdev_path) { value.value_type = SE_DATA_TYPE_STRING; value.value.sv_string = vd->vdev_path; if (sysevent_add_attr(&attr, ZFS_EV_VDEV_PATH, &value, SE_SLEEP) != 0) goto done; } } if (sysevent_attach_attributes(ev, attr) != 0) goto done; attr = NULL; done: if (attr) sysevent_free_attr(attr); #endif return (ev); } static void spa_event_post(sysevent_t *ev) { #ifdef _KERNEL sysevent_id_t eid; (void) log_sysevent(ev, SE_SLEEP, &eid); sysevent_free(ev); #endif } /* * Post a sysevent corresponding to the given event. The 'name' must be one of * the event definitions in sys/sysevent/eventdefs.h. The payload will be * filled in from the spa and (optionally) the vdev. This doesn't do anything * in the userland libzpool, as we don't want consumers to misinterpret ztest * or zdb as real changes. */ void spa_event_notify(spa_t *spa, vdev_t *vd, const char *name) { spa_event_post(spa_event_create(spa, vd, name)); } Index: projects/runtime-coverage/sys/cddl/contrib/opensolaris =================================================================== --- projects/runtime-coverage/sys/cddl/contrib/opensolaris (revision 321584) +++ projects/runtime-coverage/sys/cddl/contrib/opensolaris (revision 321585) Property changes on: projects/runtime-coverage/sys/cddl/contrib/opensolaris ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/cddl/contrib/opensolaris:r321426-321584 Index: projects/runtime-coverage/sys/compat/cloudabi/cloudabi_fd.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi/cloudabi_fd.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi/cloudabi_fd.c (revision 321585) @@ -1,537 +1,529 @@ /*- * Copyright (c) 2015 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Translation between CloudABI and Capsicum rights. */ #define RIGHTS_MAPPINGS \ MAPPING(CLOUDABI_RIGHT_FD_DATASYNC, CAP_FSYNC) \ MAPPING(CLOUDABI_RIGHT_FD_READ, CAP_READ) \ MAPPING(CLOUDABI_RIGHT_FD_SEEK, CAP_SEEK) \ MAPPING(CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS, CAP_FCNTL) \ MAPPING(CLOUDABI_RIGHT_FD_SYNC, CAP_FSYNC) \ MAPPING(CLOUDABI_RIGHT_FD_TELL, CAP_SEEK_TELL) \ MAPPING(CLOUDABI_RIGHT_FD_WRITE, CAP_WRITE) \ MAPPING(CLOUDABI_RIGHT_FILE_ADVISE) \ MAPPING(CLOUDABI_RIGHT_FILE_ALLOCATE, CAP_WRITE) \ MAPPING(CLOUDABI_RIGHT_FILE_CREATE_DIRECTORY, CAP_MKDIRAT) \ MAPPING(CLOUDABI_RIGHT_FILE_CREATE_FILE, CAP_CREATE) \ MAPPING(CLOUDABI_RIGHT_FILE_CREATE_FIFO, CAP_MKFIFOAT) \ MAPPING(CLOUDABI_RIGHT_FILE_LINK_SOURCE, CAP_LINKAT_SOURCE) \ MAPPING(CLOUDABI_RIGHT_FILE_LINK_TARGET, CAP_LINKAT_TARGET) \ MAPPING(CLOUDABI_RIGHT_FILE_OPEN, CAP_LOOKUP) \ MAPPING(CLOUDABI_RIGHT_FILE_READDIR, CAP_READ) \ MAPPING(CLOUDABI_RIGHT_FILE_READLINK, CAP_LOOKUP) \ MAPPING(CLOUDABI_RIGHT_FILE_RENAME_SOURCE, CAP_RENAMEAT_SOURCE) \ MAPPING(CLOUDABI_RIGHT_FILE_RENAME_TARGET, CAP_RENAMEAT_TARGET) \ MAPPING(CLOUDABI_RIGHT_FILE_STAT_FGET, CAP_FSTAT) \ MAPPING(CLOUDABI_RIGHT_FILE_STAT_FPUT_SIZE, CAP_FTRUNCATE) \ MAPPING(CLOUDABI_RIGHT_FILE_STAT_FPUT_TIMES, CAP_FUTIMES) \ MAPPING(CLOUDABI_RIGHT_FILE_STAT_GET, CAP_FSTATAT) \ MAPPING(CLOUDABI_RIGHT_FILE_STAT_PUT_TIMES, CAP_FUTIMESAT) \ MAPPING(CLOUDABI_RIGHT_FILE_SYMLINK, CAP_SYMLINKAT) \ MAPPING(CLOUDABI_RIGHT_FILE_UNLINK, CAP_UNLINKAT) \ MAPPING(CLOUDABI_RIGHT_MEM_MAP, CAP_MMAP) \ MAPPING(CLOUDABI_RIGHT_MEM_MAP_EXEC, CAP_MMAP_X) \ MAPPING(CLOUDABI_RIGHT_POLL_FD_READWRITE, CAP_EVENT) \ MAPPING(CLOUDABI_RIGHT_POLL_MODIFY, CAP_KQUEUE_CHANGE) \ MAPPING(CLOUDABI_RIGHT_POLL_PROC_TERMINATE, CAP_EVENT) \ MAPPING(CLOUDABI_RIGHT_POLL_WAIT, CAP_KQUEUE_EVENT) \ MAPPING(CLOUDABI_RIGHT_PROC_EXEC, CAP_FEXECVE) \ MAPPING(CLOUDABI_RIGHT_SOCK_ACCEPT, CAP_ACCEPT) \ MAPPING(CLOUDABI_RIGHT_SOCK_BIND_DIRECTORY, CAP_BINDAT) \ MAPPING(CLOUDABI_RIGHT_SOCK_BIND_SOCKET, CAP_BIND) \ MAPPING(CLOUDABI_RIGHT_SOCK_CONNECT_DIRECTORY, CAP_CONNECTAT) \ MAPPING(CLOUDABI_RIGHT_SOCK_CONNECT_SOCKET, CAP_CONNECT) \ MAPPING(CLOUDABI_RIGHT_SOCK_LISTEN, CAP_LISTEN) \ MAPPING(CLOUDABI_RIGHT_SOCK_SHUTDOWN, CAP_SHUTDOWN) \ MAPPING(CLOUDABI_RIGHT_SOCK_STAT_GET, CAP_GETPEERNAME, \ CAP_GETSOCKNAME, CAP_GETSOCKOPT) int cloudabi_sys_fd_close(struct thread *td, struct cloudabi_sys_fd_close_args *uap) { return (kern_close(td, uap->fd)); } int cloudabi_sys_fd_create1(struct thread *td, struct cloudabi_sys_fd_create1_args *uap) { struct filecaps fcaps = {}; switch (uap->type) { case CLOUDABI_FILETYPE_POLL: cap_rights_init(&fcaps.fc_rights, CAP_FSTAT, CAP_KQUEUE); return (kern_kqueue(td, 0, &fcaps)); case CLOUDABI_FILETYPE_SHARED_MEMORY: cap_rights_init(&fcaps.fc_rights, CAP_FSTAT, CAP_FTRUNCATE, CAP_MMAP_RWX); return (kern_shm_open(td, SHM_ANON, O_RDWR, 0, &fcaps)); case CLOUDABI_FILETYPE_SOCKET_DGRAM: return (kern_socket(td, AF_UNIX, SOCK_DGRAM, 0)); - case CLOUDABI_FILETYPE_SOCKET_SEQPACKET: - return (kern_socket(td, AF_UNIX, SOCK_SEQPACKET, 0)); case CLOUDABI_FILETYPE_SOCKET_STREAM: return (kern_socket(td, AF_UNIX, SOCK_STREAM, 0)); default: return (EINVAL); } } int cloudabi_sys_fd_create2(struct thread *td, struct cloudabi_sys_fd_create2_args *uap) { struct filecaps fcaps1 = {}, fcaps2 = {}; int fds[2]; int error; switch (uap->type) { case CLOUDABI_FILETYPE_FIFO: /* * CloudABI pipes are unidirectional. Restrict rights on * the pipe to simulate this. */ cap_rights_init(&fcaps1.fc_rights, CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_READ); fcaps1.fc_fcntls = CAP_FCNTL_SETFL; cap_rights_init(&fcaps2.fc_rights, CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_WRITE); fcaps2.fc_fcntls = CAP_FCNTL_SETFL; error = kern_pipe(td, fds, 0, &fcaps1, &fcaps2); break; case CLOUDABI_FILETYPE_SOCKET_DGRAM: error = kern_socketpair(td, AF_UNIX, SOCK_DGRAM, 0, fds); break; - case CLOUDABI_FILETYPE_SOCKET_SEQPACKET: - error = kern_socketpair(td, AF_UNIX, SOCK_SEQPACKET, 0, fds); - break; case CLOUDABI_FILETYPE_SOCKET_STREAM: error = kern_socketpair(td, AF_UNIX, SOCK_STREAM, 0, fds); break; default: return (EINVAL); } if (error == 0) { td->td_retval[0] = fds[0]; td->td_retval[1] = fds[1]; } return (0); } int cloudabi_sys_fd_datasync(struct thread *td, struct cloudabi_sys_fd_datasync_args *uap) { return (kern_fsync(td, uap->fd, false)); } int cloudabi_sys_fd_dup(struct thread *td, struct cloudabi_sys_fd_dup_args *uap) { return (kern_dup(td, FDDUP_NORMAL, 0, uap->from, 0)); } int cloudabi_sys_fd_replace(struct thread *td, struct cloudabi_sys_fd_replace_args *uap) { int error; /* * CloudABI's equivalent to dup2(). CloudABI processes should * not depend on hardcoded file descriptor layouts, but simply * use the file descriptor numbers that are allocated by the * kernel. Duplicating file descriptors to arbitrary numbers * should not be done. * * Invoke kern_dup() with FDDUP_MUSTREPLACE, so that we return * EBADF when duplicating to a nonexistent file descriptor. Also * clear the return value, as this system call yields no return * value. */ error = kern_dup(td, FDDUP_MUSTREPLACE, 0, uap->from, uap->to); td->td_retval[0] = 0; return (error); } int cloudabi_sys_fd_seek(struct thread *td, struct cloudabi_sys_fd_seek_args *uap) { int whence; switch (uap->whence) { case CLOUDABI_WHENCE_CUR: whence = SEEK_CUR; break; case CLOUDABI_WHENCE_END: whence = SEEK_END; break; case CLOUDABI_WHENCE_SET: whence = SEEK_SET; break; default: return (EINVAL); } return (kern_lseek(td, uap->fd, uap->offset, whence)); } /* Converts a file descriptor to a CloudABI file descriptor type. */ cloudabi_filetype_t cloudabi_convert_filetype(const struct file *fp) { struct socket *so; struct vnode *vp; switch (fp->f_type) { case DTYPE_FIFO: return (CLOUDABI_FILETYPE_FIFO); case DTYPE_KQUEUE: return (CLOUDABI_FILETYPE_POLL); case DTYPE_PIPE: return (CLOUDABI_FILETYPE_FIFO); case DTYPE_PROCDESC: return (CLOUDABI_FILETYPE_PROCESS); case DTYPE_SHM: return (CLOUDABI_FILETYPE_SHARED_MEMORY); case DTYPE_SOCKET: so = fp->f_data; switch (so->so_type) { case SOCK_DGRAM: return (CLOUDABI_FILETYPE_SOCKET_DGRAM); - case SOCK_SEQPACKET: - return (CLOUDABI_FILETYPE_SOCKET_SEQPACKET); case SOCK_STREAM: return (CLOUDABI_FILETYPE_SOCKET_STREAM); default: return (CLOUDABI_FILETYPE_UNKNOWN); } case DTYPE_VNODE: vp = fp->f_vnode; switch (vp->v_type) { case VBLK: return (CLOUDABI_FILETYPE_BLOCK_DEVICE); case VCHR: return (CLOUDABI_FILETYPE_CHARACTER_DEVICE); case VDIR: return (CLOUDABI_FILETYPE_DIRECTORY); case VFIFO: return (CLOUDABI_FILETYPE_FIFO); case VLNK: return (CLOUDABI_FILETYPE_SYMBOLIC_LINK); case VREG: return (CLOUDABI_FILETYPE_REGULAR_FILE); case VSOCK: return (CLOUDABI_FILETYPE_SOCKET_STREAM); default: return (CLOUDABI_FILETYPE_UNKNOWN); } default: return (CLOUDABI_FILETYPE_UNKNOWN); } } /* Removes rights that conflict with the file descriptor type. */ void cloudabi_remove_conflicting_rights(cloudabi_filetype_t filetype, cloudabi_rights_t *base, cloudabi_rights_t *inheriting) { /* * CloudABI has a small number of additional rights bits to * disambiguate between multiple purposes. Remove the bits that * don't apply to the type of the file descriptor. * * As file descriptor access modes (O_ACCMODE) has been fully * replaced by rights bits, CloudABI distinguishes between * rights that apply to the file descriptor itself (base) versus * rights of new file descriptors derived from them * (inheriting). The code below approximates the pair by * decomposing depending on the file descriptor type. * * We need to be somewhat accurate about which actions can * actually be performed on the file descriptor, as functions * like fcntl(fd, F_GETFL) are emulated on top of this. */ switch (filetype) { case CLOUDABI_FILETYPE_DIRECTORY: *base &= CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS | CLOUDABI_RIGHT_FD_SYNC | CLOUDABI_RIGHT_FILE_ADVISE | CLOUDABI_RIGHT_FILE_CREATE_DIRECTORY | CLOUDABI_RIGHT_FILE_CREATE_FILE | CLOUDABI_RIGHT_FILE_CREATE_FIFO | CLOUDABI_RIGHT_FILE_LINK_SOURCE | CLOUDABI_RIGHT_FILE_LINK_TARGET | CLOUDABI_RIGHT_FILE_OPEN | CLOUDABI_RIGHT_FILE_READDIR | CLOUDABI_RIGHT_FILE_READLINK | CLOUDABI_RIGHT_FILE_RENAME_SOURCE | CLOUDABI_RIGHT_FILE_RENAME_TARGET | CLOUDABI_RIGHT_FILE_STAT_FGET | CLOUDABI_RIGHT_FILE_STAT_FPUT_TIMES | CLOUDABI_RIGHT_FILE_STAT_GET | CLOUDABI_RIGHT_FILE_STAT_PUT_TIMES | CLOUDABI_RIGHT_FILE_SYMLINK | CLOUDABI_RIGHT_FILE_UNLINK | CLOUDABI_RIGHT_POLL_FD_READWRITE | CLOUDABI_RIGHT_SOCK_BIND_DIRECTORY | CLOUDABI_RIGHT_SOCK_CONNECT_DIRECTORY; *inheriting &= CLOUDABI_RIGHT_FD_DATASYNC | CLOUDABI_RIGHT_FD_READ | CLOUDABI_RIGHT_FD_SEEK | CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS | CLOUDABI_RIGHT_FD_SYNC | CLOUDABI_RIGHT_FD_TELL | CLOUDABI_RIGHT_FD_WRITE | CLOUDABI_RIGHT_FILE_ADVISE | CLOUDABI_RIGHT_FILE_ALLOCATE | CLOUDABI_RIGHT_FILE_CREATE_DIRECTORY | CLOUDABI_RIGHT_FILE_CREATE_FILE | CLOUDABI_RIGHT_FILE_CREATE_FIFO | CLOUDABI_RIGHT_FILE_LINK_SOURCE | CLOUDABI_RIGHT_FILE_LINK_TARGET | CLOUDABI_RIGHT_FILE_OPEN | CLOUDABI_RIGHT_FILE_READDIR | CLOUDABI_RIGHT_FILE_READLINK | CLOUDABI_RIGHT_FILE_RENAME_SOURCE | CLOUDABI_RIGHT_FILE_RENAME_TARGET | CLOUDABI_RIGHT_FILE_STAT_FGET | CLOUDABI_RIGHT_FILE_STAT_FPUT_SIZE | CLOUDABI_RIGHT_FILE_STAT_FPUT_TIMES | CLOUDABI_RIGHT_FILE_STAT_GET | CLOUDABI_RIGHT_FILE_STAT_PUT_TIMES | CLOUDABI_RIGHT_FILE_SYMLINK | CLOUDABI_RIGHT_FILE_UNLINK | CLOUDABI_RIGHT_MEM_MAP | CLOUDABI_RIGHT_MEM_MAP_EXEC | CLOUDABI_RIGHT_POLL_FD_READWRITE | CLOUDABI_RIGHT_PROC_EXEC | CLOUDABI_RIGHT_SOCK_BIND_DIRECTORY | CLOUDABI_RIGHT_SOCK_CONNECT_DIRECTORY; break; case CLOUDABI_FILETYPE_FIFO: *base &= CLOUDABI_RIGHT_FD_READ | CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS | CLOUDABI_RIGHT_FD_WRITE | CLOUDABI_RIGHT_FILE_STAT_FGET | CLOUDABI_RIGHT_POLL_FD_READWRITE; *inheriting = 0; break; case CLOUDABI_FILETYPE_POLL: *base &= ~CLOUDABI_RIGHT_FILE_ADVISE; *inheriting = 0; break; case CLOUDABI_FILETYPE_PROCESS: *base &= ~(CLOUDABI_RIGHT_FILE_ADVISE | CLOUDABI_RIGHT_POLL_FD_READWRITE); *inheriting = 0; break; case CLOUDABI_FILETYPE_REGULAR_FILE: *base &= CLOUDABI_RIGHT_FD_DATASYNC | CLOUDABI_RIGHT_FD_READ | CLOUDABI_RIGHT_FD_SEEK | CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS | CLOUDABI_RIGHT_FD_SYNC | CLOUDABI_RIGHT_FD_TELL | CLOUDABI_RIGHT_FD_WRITE | CLOUDABI_RIGHT_FILE_ADVISE | CLOUDABI_RIGHT_FILE_ALLOCATE | CLOUDABI_RIGHT_FILE_STAT_FGET | CLOUDABI_RIGHT_FILE_STAT_FPUT_SIZE | CLOUDABI_RIGHT_FILE_STAT_FPUT_TIMES | CLOUDABI_RIGHT_MEM_MAP | CLOUDABI_RIGHT_MEM_MAP_EXEC | CLOUDABI_RIGHT_POLL_FD_READWRITE | CLOUDABI_RIGHT_PROC_EXEC; *inheriting = 0; break; case CLOUDABI_FILETYPE_SHARED_MEMORY: *base &= ~(CLOUDABI_RIGHT_FD_SEEK | CLOUDABI_RIGHT_FD_TELL | CLOUDABI_RIGHT_FILE_ADVISE | CLOUDABI_RIGHT_FILE_ALLOCATE | CLOUDABI_RIGHT_FILE_READDIR); *inheriting = 0; break; case CLOUDABI_FILETYPE_SOCKET_DGRAM: - case CLOUDABI_FILETYPE_SOCKET_SEQPACKET: case CLOUDABI_FILETYPE_SOCKET_STREAM: *base &= CLOUDABI_RIGHT_FD_READ | CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS | CLOUDABI_RIGHT_FD_WRITE | CLOUDABI_RIGHT_FILE_STAT_FGET | CLOUDABI_RIGHT_POLL_FD_READWRITE | CLOUDABI_RIGHT_SOCK_ACCEPT | CLOUDABI_RIGHT_SOCK_BIND_SOCKET | CLOUDABI_RIGHT_SOCK_CONNECT_SOCKET | CLOUDABI_RIGHT_SOCK_LISTEN | CLOUDABI_RIGHT_SOCK_SHUTDOWN | CLOUDABI_RIGHT_SOCK_STAT_GET; break; default: *inheriting = 0; break; } } /* Converts FreeBSD's Capsicum rights to CloudABI's set of rights. */ static void convert_capabilities(const cap_rights_t *capabilities, cloudabi_filetype_t filetype, cloudabi_rights_t *base, cloudabi_rights_t *inheriting) { cloudabi_rights_t rights; /* Convert FreeBSD bits to CloudABI bits. */ rights = 0; #define MAPPING(cloudabi, ...) do { \ if (cap_rights_is_set(capabilities, ##__VA_ARGS__)) \ rights |= (cloudabi); \ } while (0); RIGHTS_MAPPINGS #undef MAPPING *base = rights; *inheriting = rights; cloudabi_remove_conflicting_rights(filetype, base, inheriting); } int cloudabi_sys_fd_stat_get(struct thread *td, struct cloudabi_sys_fd_stat_get_args *uap) { cloudabi_fdstat_t fsb = {}; struct file *fp; cap_rights_t rights; struct filecaps fcaps; int error, oflags; /* Obtain file descriptor properties. */ error = fget_cap(td, uap->fd, cap_rights_init(&rights), &fp, &fcaps); if (error != 0) return (error); oflags = OFLAGS(fp->f_flag); fsb.fs_filetype = cloudabi_convert_filetype(fp); fdrop(fp, td); /* Convert file descriptor flags. */ if (oflags & O_APPEND) fsb.fs_flags |= CLOUDABI_FDFLAG_APPEND; if (oflags & O_NONBLOCK) fsb.fs_flags |= CLOUDABI_FDFLAG_NONBLOCK; if (oflags & O_SYNC) fsb.fs_flags |= CLOUDABI_FDFLAG_SYNC; /* Convert capabilities to CloudABI rights. */ convert_capabilities(&fcaps.fc_rights, fsb.fs_filetype, &fsb.fs_rights_base, &fsb.fs_rights_inheriting); filecaps_free(&fcaps); return (copyout(&fsb, (void *)uap->buf, sizeof(fsb))); } /* Converts CloudABI rights to a set of Capsicum capabilities. */ int cloudabi_convert_rights(cloudabi_rights_t in, cap_rights_t *out) { cap_rights_init(out); #define MAPPING(cloudabi, ...) do { \ if (in & (cloudabi)) { \ cap_rights_set(out, ##__VA_ARGS__); \ in &= ~(cloudabi); \ } \ } while (0); RIGHTS_MAPPINGS #undef MAPPING if (in != 0) return (ENOTCAPABLE); return (0); } int cloudabi_sys_fd_stat_put(struct thread *td, struct cloudabi_sys_fd_stat_put_args *uap) { cloudabi_fdstat_t fsb; cap_rights_t rights; int error, oflags; error = copyin(uap->buf, &fsb, sizeof(fsb)); if (error != 0) return (error); if (uap->flags == CLOUDABI_FDSTAT_FLAGS) { /* Convert flags. */ oflags = 0; if (fsb.fs_flags & CLOUDABI_FDFLAG_APPEND) oflags |= O_APPEND; if (fsb.fs_flags & CLOUDABI_FDFLAG_NONBLOCK) oflags |= O_NONBLOCK; if (fsb.fs_flags & (CLOUDABI_FDFLAG_SYNC | CLOUDABI_FDFLAG_DSYNC | CLOUDABI_FDFLAG_RSYNC)) oflags |= O_SYNC; return (kern_fcntl(td, uap->fd, F_SETFL, oflags)); } else if (uap->flags == CLOUDABI_FDSTAT_RIGHTS) { /* Convert rights. */ error = cloudabi_convert_rights( fsb.fs_rights_base | fsb.fs_rights_inheriting, &rights); if (error != 0) return (error); return (kern_cap_rights_limit(td, uap->fd, &rights)); } return (EINVAL); } int cloudabi_sys_fd_sync(struct thread *td, struct cloudabi_sys_fd_sync_args *uap) { return (kern_fsync(td, uap->fd, true)); } Index: projects/runtime-coverage/sys/compat/cloudabi/cloudabi_mem.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi/cloudabi_mem.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi/cloudabi_mem.c (revision 321585) @@ -1,178 +1,161 @@ /*- * Copyright (c) 2015 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include /* Converts CloudABI's memory protection flags to FreeBSD's. */ static int convert_mprot(cloudabi_mprot_t in, int *out) { /* Unknown protection flags. */ if ((in & ~(CLOUDABI_PROT_EXEC | CLOUDABI_PROT_WRITE | CLOUDABI_PROT_READ)) != 0) return (ENOTSUP); /* W^X: Write and exec cannot be enabled at the same time. */ if ((in & (CLOUDABI_PROT_EXEC | CLOUDABI_PROT_WRITE)) == (CLOUDABI_PROT_EXEC | CLOUDABI_PROT_WRITE)) return (ENOTSUP); *out = 0; if (in & CLOUDABI_PROT_EXEC) *out |= PROT_EXEC; if (in & CLOUDABI_PROT_WRITE) *out |= PROT_WRITE; if (in & CLOUDABI_PROT_READ) *out |= PROT_READ; return (0); } int cloudabi_sys_mem_advise(struct thread *td, struct cloudabi_sys_mem_advise_args *uap) { int behav; switch (uap->advice) { case CLOUDABI_ADVICE_DONTNEED: behav = MADV_DONTNEED; break; case CLOUDABI_ADVICE_NORMAL: behav = MADV_NORMAL; break; case CLOUDABI_ADVICE_RANDOM: behav = MADV_RANDOM; break; case CLOUDABI_ADVICE_SEQUENTIAL: behav = MADV_SEQUENTIAL; break; case CLOUDABI_ADVICE_WILLNEED: behav = MADV_WILLNEED; break; default: return (EINVAL); } return (kern_madvise(td, (uintptr_t)uap->mapping, uap->mapping_len, behav)); } int -cloudabi_sys_mem_lock(struct thread *td, struct cloudabi_sys_mem_lock_args *uap) -{ - - return (kern_mlock(td->td_proc, td->td_ucred, - __DECONST(uintptr_t, uap->mapping), uap->mapping_len)); -} - -int cloudabi_sys_mem_map(struct thread *td, struct cloudabi_sys_mem_map_args *uap) { int error, flags, prot; /* Translate flags. */ flags = 0; if (uap->flags & CLOUDABI_MAP_ANON) flags |= MAP_ANON; if (uap->flags & CLOUDABI_MAP_FIXED) flags |= MAP_FIXED; if (uap->flags & CLOUDABI_MAP_PRIVATE) flags |= MAP_PRIVATE; if (uap->flags & CLOUDABI_MAP_SHARED) flags |= MAP_SHARED; /* Translate protection. */ error = convert_mprot(uap->prot, &prot); if (error != 0) return (error); return (kern_mmap(td, (uintptr_t)uap->addr, uap->len, prot, flags, uap->fd, uap->off)); } int cloudabi_sys_mem_protect(struct thread *td, struct cloudabi_sys_mem_protect_args *uap) { int error, prot; /* Translate protection. */ error = convert_mprot(uap->prot, &prot); if (error != 0) return (error); return (kern_mprotect(td, (uintptr_t)uap->mapping, uap->mapping_len, prot)); } int cloudabi_sys_mem_sync(struct thread *td, struct cloudabi_sys_mem_sync_args *uap) { int flags; /* Convert flags. */ switch (uap->flags & (CLOUDABI_MS_ASYNC | CLOUDABI_MS_SYNC)) { case CLOUDABI_MS_ASYNC: flags = MS_ASYNC; break; case CLOUDABI_MS_SYNC: flags = MS_SYNC; break; default: return (EINVAL); } if ((uap->flags & CLOUDABI_MS_INVALIDATE) != 0) flags |= MS_INVALIDATE; return (kern_msync(td, (uintptr_t)uap->mapping, uap->mapping_len, flags)); -} - -int -cloudabi_sys_mem_unlock(struct thread *td, - struct cloudabi_sys_mem_unlock_args *uap) -{ - - return (kern_munlock(td, __DECONST(uintptr_t, uap->mapping), - uap->mapping_len)); } int cloudabi_sys_mem_unmap(struct thread *td, struct cloudabi_sys_mem_unmap_args *uap) { return (kern_munmap(td, (uintptr_t)uap->mapping, uap->mapping_len)); } Index: projects/runtime-coverage/sys/compat/cloudabi/cloudabi_sock.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi/cloudabi_sock.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi/cloudabi_sock.c (revision 321585) @@ -1,370 +1,283 @@ /*- * Copyright (c) 2015-2017 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -/* Converts FreeBSD's struct sockaddr to CloudABI's cloudabi_sockaddr_t. */ -static void -cloudabi_convert_sockaddr(const struct sockaddr *sa, socklen_t sal, - cloudabi_sockaddr_t *rsa) -{ - const struct sockaddr_in *sin; - const struct sockaddr_in6 *sin6; - - /* Zero-sized socket address. */ - if (sal < offsetof(struct sockaddr, sa_family) + sizeof(sa->sa_family)) - return; - - switch (sa->sa_family) { - case AF_INET: - if (sal < sizeof(struct sockaddr_in)) - return; - sin = (const struct sockaddr_in *)sa; - rsa->sa_family = CLOUDABI_AF_INET; - memcpy(&rsa->sa_inet.addr, &sin->sin_addr, - sizeof(rsa->sa_inet.addr)); - rsa->sa_inet.port = ntohs(sin->sin_port); - return; - case AF_INET6: - if (sal < sizeof(struct sockaddr_in6)) - return; - sin6 = (const struct sockaddr_in6 *)sa; - rsa->sa_family = CLOUDABI_AF_INET6; - memcpy(&rsa->sa_inet6.addr, &sin6->sin6_addr, - sizeof(rsa->sa_inet6.addr)); - rsa->sa_inet6.port = ntohs(sin6->sin6_port); - return; - case AF_UNIX: - rsa->sa_family = CLOUDABI_AF_UNIX; - return; - } -} - /* Copies a pathname into a UNIX socket address structure. */ static int copyin_sockaddr_un(const char *path, size_t pathlen, struct sockaddr_un *sun) { int error; /* Copy in pathname string if there's enough space. */ if (pathlen >= sizeof(sun->sun_path)) return (ENAMETOOLONG); error = copyin(path, &sun->sun_path, pathlen); if (error != 0) return (error); if (memchr(sun->sun_path, '\0', pathlen) != NULL) return (EINVAL); /* Initialize the rest of the socket address. */ sun->sun_path[pathlen] = '\0'; sun->sun_family = AF_UNIX; sun->sun_len = sizeof(*sun); return (0); } int cloudabi_sys_sock_accept(struct thread *td, struct cloudabi_sys_sock_accept_args *uap) { - struct sockaddr *sa; - cloudabi_sockstat_t ss = {}; - socklen_t sal; - int error; - if (uap->buf == NULL) { - /* Only return the new file descriptor number. */ - return (kern_accept(td, uap->sock, NULL, NULL, NULL)); - } else { - /* Also return properties of the new socket descriptor. */ - sal = MAX(sizeof(struct sockaddr_in), - sizeof(struct sockaddr_in6)); - error = kern_accept(td, uap->sock, (void *)&sa, &sal, NULL); - if (error != 0) - return (error); - - /* TODO(ed): Fill the other members of cloudabi_sockstat_t. */ - cloudabi_convert_sockaddr(sa, sal, &ss.ss_peername); - free(sa, M_SONAME); - return (copyout(&ss, uap->buf, sizeof(ss))); - } + return (kern_accept(td, uap->sock, NULL, NULL, NULL)); } int cloudabi_sys_sock_bind(struct thread *td, struct cloudabi_sys_sock_bind_args *uap) { struct sockaddr_un sun; int error; error = copyin_sockaddr_un(uap->path, uap->path_len, &sun); if (error != 0) return (error); return (kern_bindat(td, uap->fd, uap->sock, (struct sockaddr *)&sun)); } int cloudabi_sys_sock_connect(struct thread *td, struct cloudabi_sys_sock_connect_args *uap) { struct sockaddr_un sun; int error; error = copyin_sockaddr_un(uap->path, uap->path_len, &sun); if (error != 0) return (error); return (kern_connectat(td, uap->fd, uap->sock, (struct sockaddr *)&sun)); } int cloudabi_sys_sock_listen(struct thread *td, struct cloudabi_sys_sock_listen_args *uap) { return (kern_listen(td, uap->sock, uap->backlog)); } int cloudabi_sys_sock_shutdown(struct thread *td, struct cloudabi_sys_sock_shutdown_args *uap) { int how; switch (uap->how) { case CLOUDABI_SHUT_RD: how = SHUT_RD; break; case CLOUDABI_SHUT_WR: how = SHUT_WR; break; case CLOUDABI_SHUT_RD | CLOUDABI_SHUT_WR: how = SHUT_RDWR; break; default: return (EINVAL); } return (kern_shutdown(td, uap->sock, how)); } int cloudabi_sys_sock_stat_get(struct thread *td, struct cloudabi_sys_sock_stat_get_args *uap) { cloudabi_sockstat_t ss = {}; cap_rights_t rights; struct file *fp; - struct sockaddr *sa; struct socket *so; int error; error = getsock_cap(td, uap->sock, cap_rights_init(&rights, CAP_GETSOCKOPT, CAP_GETPEERNAME, CAP_GETSOCKNAME), &fp, NULL, NULL); if (error != 0) return (error); so = fp->f_data; - CURVNET_SET(so->so_vnet); - - /* Set ss_sockname. */ - error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa); - if (error == 0) { - cloudabi_convert_sockaddr(sa, sa->sa_len, &ss.ss_sockname); - free(sa, M_SONAME); - } - - /* Set ss_peername. */ - if ((so->so_state & (SS_ISCONNECTED | SS_ISCONFIRMING)) != 0) { - error = so->so_proto->pr_usrreqs->pru_peeraddr(so, &sa); - if (error == 0) { - cloudabi_convert_sockaddr(sa, sa->sa_len, - &ss.ss_peername); - free(sa, M_SONAME); - } - } - - CURVNET_RESTORE(); - /* Set ss_error. */ SOCK_LOCK(so); ss.ss_error = cloudabi_convert_errno(so->so_error); if ((uap->flags & CLOUDABI_SOCKSTAT_CLEAR_ERROR) != 0) so->so_error = 0; SOCK_UNLOCK(so); /* Set ss_state. */ if ((so->so_options & SO_ACCEPTCONN) != 0) ss.ss_state |= CLOUDABI_SOCKSTATE_ACCEPTCONN; fdrop(fp, td); return (copyout(&ss, uap->buf, sizeof(ss))); } int cloudabi_sock_recv(struct thread *td, cloudabi_fd_t fd, struct iovec *data, size_t datalen, cloudabi_fd_t *fds, size_t fdslen, - cloudabi_msgflags_t flags, size_t *rdatalen, size_t *rfdslen, - cloudabi_sockaddr_t *peername, cloudabi_msgflags_t *rflags) + cloudabi_riflags_t flags, size_t *rdatalen, size_t *rfdslen, + cloudabi_roflags_t *rflags) { struct sockaddr_storage ss; struct msghdr hdr = { .msg_name = &ss, .msg_namelen = sizeof(ss), .msg_iov = data, .msg_iovlen = datalen, }; struct mbuf *control; int error; /* Convert flags. */ - if (flags & CLOUDABI_MSG_PEEK) + if (flags & CLOUDABI_SOCK_RECV_PEEK) hdr.msg_flags |= MSG_PEEK; - if (flags & CLOUDABI_MSG_WAITALL) + if (flags & CLOUDABI_SOCK_RECV_WAITALL) hdr.msg_flags |= MSG_WAITALL; control = NULL; error = kern_recvit(td, fd, &hdr, UIO_SYSSPACE, fdslen > 0 ? &control : NULL); if (error != 0) return (error); /* Convert return values. */ *rdatalen = td->td_retval[0]; td->td_retval[0] = 0; *rfdslen = 0; - cloudabi_convert_sockaddr((struct sockaddr *)&ss, - MIN(hdr.msg_namelen, sizeof(ss)), peername); *rflags = 0; - if (hdr.msg_flags & MSG_EOR) - *rflags |= CLOUDABI_MSG_EOR; if (hdr.msg_flags & MSG_TRUNC) - *rflags |= CLOUDABI_MSG_TRUNC; + *rflags |= CLOUDABI_SOCK_RECV_DATA_TRUNCATED; /* Extract file descriptors from SCM_RIGHTS messages. */ if (control != NULL) { struct cmsghdr *chdr; hdr.msg_control = mtod(control, void *); hdr.msg_controllen = control->m_len; for (chdr = CMSG_FIRSTHDR(&hdr); chdr != NULL; chdr = CMSG_NXTHDR(&hdr, chdr)) { if (chdr->cmsg_level == SOL_SOCKET && chdr->cmsg_type == SCM_RIGHTS) { size_t nfds; nfds = (chdr->cmsg_len - CMSG_LEN(0)) / sizeof(int); if (nfds > fdslen) { /* Unable to store file descriptors. */ nfds = fdslen; - *rflags |= CLOUDABI_MSG_CTRUNC; + *rflags |= + CLOUDABI_SOCK_RECV_FDS_TRUNCATED; } error = copyout(CMSG_DATA(chdr), fds, nfds * sizeof(int)); if (error != 0) { m_free(control); return (error); } fds += nfds; fdslen -= nfds; *rfdslen += nfds; } } m_free(control); } return (0); } int cloudabi_sock_send(struct thread *td, cloudabi_fd_t fd, struct iovec *data, - size_t datalen, const cloudabi_fd_t *fds, size_t fdslen, - cloudabi_msgflags_t flags, size_t *rdatalen) + size_t datalen, const cloudabi_fd_t *fds, size_t fdslen, size_t *rdatalen) { struct msghdr hdr = { .msg_iov = data, .msg_iovlen = datalen, }; struct mbuf *control; - int error, mflags; + int error; - /* Convert flags. */ - mflags = MSG_NOSIGNAL; - if (flags & CLOUDABI_MSG_EOR) - mflags |= MSG_EOR; - /* Convert file descriptor array to an SCM_RIGHTS message. */ if (fdslen > MCLBYTES || CMSG_SPACE(fdslen * sizeof(int)) > MCLBYTES) { return (EINVAL); } else if (fdslen > 0) { struct cmsghdr *chdr; control = m_get2(CMSG_SPACE(fdslen * sizeof(int)), M_WAITOK, MT_CONTROL, 0); control->m_len = CMSG_SPACE(fdslen * sizeof(int)); chdr = mtod(control, struct cmsghdr *); chdr->cmsg_len = CMSG_LEN(fdslen * sizeof(int)); chdr->cmsg_level = SOL_SOCKET; chdr->cmsg_type = SCM_RIGHTS; error = copyin(fds, CMSG_DATA(chdr), fdslen * sizeof(int)); if (error != 0) { m_free(control); return (error); } } else { control = NULL; } - error = kern_sendit(td, fd, &hdr, mflags, control, UIO_USERSPACE); + error = kern_sendit(td, fd, &hdr, MSG_NOSIGNAL, control, UIO_USERSPACE); if (error != 0) return (error); *rdatalen = td->td_retval[0]; td->td_retval[0] = 0; return (0); } Index: projects/runtime-coverage/sys/compat/cloudabi/cloudabi_util.h =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi/cloudabi_util.h (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi/cloudabi_util.h (revision 321585) @@ -1,87 +1,87 @@ /*- * Copyright (c) 2015-2017 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _CLOUDABI_UTIL_H_ #define _CLOUDABI_UTIL_H_ #include #include struct file; struct sysentvec; struct thread; struct timespec; /* Fetches the time value of a clock. */ int cloudabi_clock_time_get(struct thread *, cloudabi_clockid_t, cloudabi_timestamp_t *); /* Converts a FreeBSD errno to a CloudABI errno. */ cloudabi_errno_t cloudabi_convert_errno(int); /* Converts a file descriptor to a CloudABI file descriptor type. */ cloudabi_filetype_t cloudabi_convert_filetype(const struct file *); /* Converts CloudABI rights to a set of Capsicum capabilities. */ int cloudabi_convert_rights(cloudabi_rights_t, cap_rights_t *); /* Removes rights that conflict with the file descriptor type. */ void cloudabi_remove_conflicting_rights(cloudabi_filetype_t, cloudabi_rights_t *, cloudabi_rights_t *); /* Converts a struct timespec to a CloudABI timestamp. */ int cloudabi_convert_timespec(const struct timespec *, cloudabi_timestamp_t *); /* * Blocking futex functions. * * These functions are called by CloudABI's polling system calls to * sleep on a lock or condition variable. */ int cloudabi_futex_condvar_wait(struct thread *, cloudabi_condvar_t *, cloudabi_scope_t, cloudabi_lock_t *, cloudabi_scope_t, cloudabi_clockid_t, cloudabi_timestamp_t, cloudabi_timestamp_t); int cloudabi_futex_lock_rdlock(struct thread *, cloudabi_lock_t *, cloudabi_scope_t, cloudabi_clockid_t, cloudabi_timestamp_t, cloudabi_timestamp_t); int cloudabi_futex_lock_wrlock(struct thread *, cloudabi_lock_t *, cloudabi_scope_t, cloudabi_clockid_t, cloudabi_timestamp_t, cloudabi_timestamp_t); /* Socket operations. */ int cloudabi_sock_recv(struct thread *, cloudabi_fd_t, struct iovec *, size_t, - cloudabi_fd_t *, size_t, cloudabi_msgflags_t, size_t *, size_t *, - cloudabi_sockaddr_t *, cloudabi_msgflags_t *); + cloudabi_fd_t *, size_t, cloudabi_riflags_t, size_t *, size_t *, + cloudabi_roflags_t *); int cloudabi_sock_send(struct thread *, cloudabi_fd_t, struct iovec *, size_t, - const cloudabi_fd_t *, size_t, cloudabi_msgflags_t, size_t *); + const cloudabi_fd_t *, size_t, size_t *); /* vDSO setup and teardown. */ void cloudabi_vdso_init(struct sysentvec *, char *, char *); void cloudabi_vdso_destroy(struct sysentvec *); #endif Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_proto.h =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_proto.h (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_proto.h (revision 321585) @@ -1,464 +1,458 @@ /* * System call prototypes. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #ifndef _CLOUDABI32_SYSPROTO_H_ #define _CLOUDABI32_SYSPROTO_H_ #include #include #include #include #include #include #include #include struct proc; struct thread; #define PAD_(t) (sizeof(register_t) <= sizeof(t) ? \ 0 : sizeof(register_t) - sizeof(t)) #if BYTE_ORDER == LITTLE_ENDIAN #define PADL_(t) 0 #define PADR_(t) PAD_(t) #else #define PADL_(t) PAD_(t) #define PADR_(t) 0 #endif struct cloudabi_sys_clock_res_get_args { char clock_id_l_[PADL_(cloudabi_clockid_t)]; cloudabi_clockid_t clock_id; char clock_id_r_[PADR_(cloudabi_clockid_t)]; }; struct cloudabi_sys_clock_time_get_args { char clock_id_l_[PADL_(cloudabi_clockid_t)]; cloudabi_clockid_t clock_id; char clock_id_r_[PADR_(cloudabi_clockid_t)]; char precision_l_[PADL_(cloudabi_timestamp_t)]; cloudabi_timestamp_t precision; char precision_r_[PADR_(cloudabi_timestamp_t)]; }; struct cloudabi_sys_condvar_signal_args { char condvar_l_[PADL_(cloudabi_condvar_t *)]; cloudabi_condvar_t * condvar; char condvar_r_[PADR_(cloudabi_condvar_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; char nwaiters_l_[PADL_(cloudabi_nthreads_t)]; cloudabi_nthreads_t nwaiters; char nwaiters_r_[PADR_(cloudabi_nthreads_t)]; }; struct cloudabi_sys_fd_close_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_create1_args { char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_fd_create2_args { char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_fd_datasync_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_dup_args { char from_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t from; char from_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi32_sys_fd_pread_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi32_iovec_t *)]; const cloudabi32_iovec_t * iovs; char iovs_r_[PADR_(const cloudabi32_iovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi32_sys_fd_pwrite_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi32_ciovec_t *)]; const cloudabi32_ciovec_t * iovs; char iovs_r_[PADR_(const cloudabi32_ciovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi32_sys_fd_read_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi32_iovec_t *)]; const cloudabi32_iovec_t * iovs; char iovs_r_[PADR_(const cloudabi32_iovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_fd_replace_args { char from_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t from; char from_r_[PADR_(cloudabi_fd_t)]; char to_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t to; char to_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_seek_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filedelta_t)]; cloudabi_filedelta_t offset; char offset_r_[PADR_(cloudabi_filedelta_t)]; char whence_l_[PADL_(cloudabi_whence_t)]; cloudabi_whence_t whence; char whence_r_[PADR_(cloudabi_whence_t)]; }; struct cloudabi_sys_fd_stat_get_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_fdstat_t *)]; cloudabi_fdstat_t * buf; char buf_r_[PADR_(cloudabi_fdstat_t *)]; }; struct cloudabi_sys_fd_stat_put_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(const cloudabi_fdstat_t *)]; const cloudabi_fdstat_t * buf; char buf_r_[PADR_(const cloudabi_fdstat_t *)]; char flags_l_[PADL_(cloudabi_fdsflags_t)]; cloudabi_fdsflags_t flags; char flags_r_[PADR_(cloudabi_fdsflags_t)]; }; struct cloudabi_sys_fd_sync_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi32_sys_fd_write_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi32_ciovec_t *)]; const cloudabi32_ciovec_t * iovs; char iovs_r_[PADR_(const cloudabi32_ciovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_advise_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; char len_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t len; char len_r_[PADR_(cloudabi_filesize_t)]; char advice_l_[PADL_(cloudabi_advice_t)]; cloudabi_advice_t advice; char advice_r_[PADR_(cloudabi_advice_t)]; }; struct cloudabi_sys_file_allocate_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; char len_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t len; char len_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi_sys_file_create_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_file_link_args { char fd1_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd1; char fd1_r_[PADR_(cloudabi_lookup_t)]; char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd2_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd2; char fd2_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_open_args { char dirfd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t dirfd; char dirfd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char oflags_l_[PADL_(cloudabi_oflags_t)]; cloudabi_oflags_t oflags; char oflags_r_[PADR_(cloudabi_oflags_t)]; char fds_l_[PADL_(const cloudabi_fdstat_t *)]; const cloudabi_fdstat_t * fds; char fds_r_[PADR_(const cloudabi_fdstat_t *)]; }; struct cloudabi_sys_file_readdir_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(void *)]; void * buf; char buf_r_[PADR_(void *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; char cookie_l_[PADL_(cloudabi_dircookie_t)]; cloudabi_dircookie_t cookie; char cookie_r_[PADR_(cloudabi_dircookie_t)]; }; struct cloudabi_sys_file_readlink_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(char *)]; char * buf; char buf_r_[PADR_(char *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_rename_args { char fd1_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd1; char fd1_r_[PADR_(cloudabi_fd_t)]; char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd2_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd2; char fd2_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_stat_fget_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_filestat_t *)]; cloudabi_filestat_t * buf; char buf_r_[PADR_(cloudabi_filestat_t *)]; }; struct cloudabi_sys_file_stat_fput_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(const cloudabi_filestat_t *)]; const cloudabi_filestat_t * buf; char buf_r_[PADR_(const cloudabi_filestat_t *)]; char flags_l_[PADL_(cloudabi_fsflags_t)]; cloudabi_fsflags_t flags; char flags_r_[PADR_(cloudabi_fsflags_t)]; }; struct cloudabi_sys_file_stat_get_args { char fd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd; char fd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(cloudabi_filestat_t *)]; cloudabi_filestat_t * buf; char buf_r_[PADR_(cloudabi_filestat_t *)]; }; struct cloudabi_sys_file_stat_put_args { char fd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd; char fd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(const cloudabi_filestat_t *)]; const cloudabi_filestat_t * buf; char buf_r_[PADR_(const cloudabi_filestat_t *)]; char flags_l_[PADL_(cloudabi_fsflags_t)]; cloudabi_fsflags_t flags; char flags_r_[PADR_(cloudabi_fsflags_t)]; }; struct cloudabi_sys_file_symlink_args { char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_unlink_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char flags_l_[PADL_(cloudabi_ulflags_t)]; cloudabi_ulflags_t flags; char flags_r_[PADR_(cloudabi_ulflags_t)]; }; struct cloudabi_sys_lock_unlock_args { char lock_l_[PADL_(cloudabi_lock_t *)]; cloudabi_lock_t * lock; char lock_r_[PADR_(cloudabi_lock_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; }; struct cloudabi_sys_mem_advise_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char advice_l_[PADL_(cloudabi_advice_t)]; cloudabi_advice_t advice; char advice_r_[PADR_(cloudabi_advice_t)]; }; -struct cloudabi_sys_mem_lock_args { - char mapping_l_[PADL_(const void *)]; const void * mapping; char mapping_r_[PADR_(const void *)]; - char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; -}; struct cloudabi_sys_mem_map_args { char addr_l_[PADL_(void *)]; void * addr; char addr_r_[PADR_(void *)]; char len_l_[PADL_(size_t)]; size_t len; char len_r_[PADR_(size_t)]; char prot_l_[PADL_(cloudabi_mprot_t)]; cloudabi_mprot_t prot; char prot_r_[PADR_(cloudabi_mprot_t)]; char flags_l_[PADL_(cloudabi_mflags_t)]; cloudabi_mflags_t flags; char flags_r_[PADR_(cloudabi_mflags_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char off_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t off; char off_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi_sys_mem_protect_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char prot_l_[PADL_(cloudabi_mprot_t)]; cloudabi_mprot_t prot; char prot_r_[PADR_(cloudabi_mprot_t)]; }; struct cloudabi_sys_mem_sync_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char flags_l_[PADL_(cloudabi_msflags_t)]; cloudabi_msflags_t flags; char flags_r_[PADR_(cloudabi_msflags_t)]; }; -struct cloudabi_sys_mem_unlock_args { - char mapping_l_[PADL_(const void *)]; const void * mapping; char mapping_r_[PADR_(const void *)]; - char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; -}; struct cloudabi_sys_mem_unmap_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; }; struct cloudabi32_sys_poll_args { char in_l_[PADL_(const cloudabi32_subscription_t *)]; const cloudabi32_subscription_t * in; char in_r_[PADR_(const cloudabi32_subscription_t *)]; char out_l_[PADL_(cloudabi32_event_t *)]; cloudabi32_event_t * out; char out_r_[PADR_(cloudabi32_event_t *)]; char nsubscriptions_l_[PADL_(size_t)]; size_t nsubscriptions; char nsubscriptions_r_[PADR_(size_t)]; }; struct cloudabi32_sys_poll_fd_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi32_subscription_t *)]; const cloudabi32_subscription_t * in; char in_r_[PADR_(const cloudabi32_subscription_t *)]; char in_len_l_[PADL_(size_t)]; size_t in_len; char in_len_r_[PADR_(size_t)]; char out_l_[PADL_(cloudabi32_event_t *)]; cloudabi32_event_t * out; char out_r_[PADR_(cloudabi32_event_t *)]; char out_len_l_[PADL_(size_t)]; size_t out_len; char out_len_r_[PADR_(size_t)]; char timeout_l_[PADL_(const cloudabi32_subscription_t *)]; const cloudabi32_subscription_t * timeout; char timeout_r_[PADR_(const cloudabi32_subscription_t *)]; }; struct cloudabi_sys_proc_exec_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char data_l_[PADL_(const void *)]; const void * data; char data_r_[PADR_(const void *)]; char data_len_l_[PADL_(size_t)]; size_t data_len; char data_len_r_[PADR_(size_t)]; char fds_l_[PADL_(const cloudabi_fd_t *)]; const cloudabi_fd_t * fds; char fds_r_[PADR_(const cloudabi_fd_t *)]; char fds_len_l_[PADL_(size_t)]; size_t fds_len; char fds_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_proc_exit_args { char rval_l_[PADL_(cloudabi_exitcode_t)]; cloudabi_exitcode_t rval; char rval_r_[PADR_(cloudabi_exitcode_t)]; }; struct cloudabi_sys_proc_fork_args { register_t dummy; }; struct cloudabi_sys_proc_raise_args { char sig_l_[PADL_(cloudabi_signal_t)]; cloudabi_signal_t sig; char sig_r_[PADR_(cloudabi_signal_t)]; }; struct cloudabi_sys_random_get_args { char buf_l_[PADL_(void *)]; void * buf; char buf_r_[PADR_(void *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_accept_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; - char buf_l_[PADL_(cloudabi_sockstat_t *)]; cloudabi_sockstat_t * buf; char buf_r_[PADR_(cloudabi_sockstat_t *)]; + char unused_l_[PADL_(void *)]; void * unused; char unused_r_[PADR_(void *)]; }; struct cloudabi_sys_sock_bind_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_connect_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_listen_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char backlog_l_[PADL_(cloudabi_backlog_t)]; cloudabi_backlog_t backlog; char backlog_r_[PADR_(cloudabi_backlog_t)]; }; struct cloudabi32_sys_sock_recv_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi32_recv_in_t *)]; const cloudabi32_recv_in_t * in; char in_r_[PADR_(const cloudabi32_recv_in_t *)]; char out_l_[PADL_(cloudabi32_recv_out_t *)]; cloudabi32_recv_out_t * out; char out_r_[PADR_(cloudabi32_recv_out_t *)]; }; struct cloudabi32_sys_sock_send_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi32_send_in_t *)]; const cloudabi32_send_in_t * in; char in_r_[PADR_(const cloudabi32_send_in_t *)]; char out_l_[PADL_(cloudabi32_send_out_t *)]; cloudabi32_send_out_t * out; char out_r_[PADR_(cloudabi32_send_out_t *)]; }; struct cloudabi_sys_sock_shutdown_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char how_l_[PADL_(cloudabi_sdflags_t)]; cloudabi_sdflags_t how; char how_r_[PADR_(cloudabi_sdflags_t)]; }; struct cloudabi_sys_sock_stat_get_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_sockstat_t *)]; cloudabi_sockstat_t * buf; char buf_r_[PADR_(cloudabi_sockstat_t *)]; char flags_l_[PADL_(cloudabi_ssflags_t)]; cloudabi_ssflags_t flags; char flags_r_[PADR_(cloudabi_ssflags_t)]; }; struct cloudabi32_sys_thread_create_args { char attr_l_[PADL_(cloudabi32_threadattr_t *)]; cloudabi32_threadattr_t * attr; char attr_r_[PADR_(cloudabi32_threadattr_t *)]; }; struct cloudabi_sys_thread_exit_args { char lock_l_[PADL_(cloudabi_lock_t *)]; cloudabi_lock_t * lock; char lock_r_[PADR_(cloudabi_lock_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; }; struct cloudabi_sys_thread_yield_args { register_t dummy; }; int cloudabi_sys_clock_res_get(struct thread *, struct cloudabi_sys_clock_res_get_args *); int cloudabi_sys_clock_time_get(struct thread *, struct cloudabi_sys_clock_time_get_args *); int cloudabi_sys_condvar_signal(struct thread *, struct cloudabi_sys_condvar_signal_args *); int cloudabi_sys_fd_close(struct thread *, struct cloudabi_sys_fd_close_args *); int cloudabi_sys_fd_create1(struct thread *, struct cloudabi_sys_fd_create1_args *); int cloudabi_sys_fd_create2(struct thread *, struct cloudabi_sys_fd_create2_args *); int cloudabi_sys_fd_datasync(struct thread *, struct cloudabi_sys_fd_datasync_args *); int cloudabi_sys_fd_dup(struct thread *, struct cloudabi_sys_fd_dup_args *); int cloudabi32_sys_fd_pread(struct thread *, struct cloudabi32_sys_fd_pread_args *); int cloudabi32_sys_fd_pwrite(struct thread *, struct cloudabi32_sys_fd_pwrite_args *); int cloudabi32_sys_fd_read(struct thread *, struct cloudabi32_sys_fd_read_args *); int cloudabi_sys_fd_replace(struct thread *, struct cloudabi_sys_fd_replace_args *); int cloudabi_sys_fd_seek(struct thread *, struct cloudabi_sys_fd_seek_args *); int cloudabi_sys_fd_stat_get(struct thread *, struct cloudabi_sys_fd_stat_get_args *); int cloudabi_sys_fd_stat_put(struct thread *, struct cloudabi_sys_fd_stat_put_args *); int cloudabi_sys_fd_sync(struct thread *, struct cloudabi_sys_fd_sync_args *); int cloudabi32_sys_fd_write(struct thread *, struct cloudabi32_sys_fd_write_args *); int cloudabi_sys_file_advise(struct thread *, struct cloudabi_sys_file_advise_args *); int cloudabi_sys_file_allocate(struct thread *, struct cloudabi_sys_file_allocate_args *); int cloudabi_sys_file_create(struct thread *, struct cloudabi_sys_file_create_args *); int cloudabi_sys_file_link(struct thread *, struct cloudabi_sys_file_link_args *); int cloudabi_sys_file_open(struct thread *, struct cloudabi_sys_file_open_args *); int cloudabi_sys_file_readdir(struct thread *, struct cloudabi_sys_file_readdir_args *); int cloudabi_sys_file_readlink(struct thread *, struct cloudabi_sys_file_readlink_args *); int cloudabi_sys_file_rename(struct thread *, struct cloudabi_sys_file_rename_args *); int cloudabi_sys_file_stat_fget(struct thread *, struct cloudabi_sys_file_stat_fget_args *); int cloudabi_sys_file_stat_fput(struct thread *, struct cloudabi_sys_file_stat_fput_args *); int cloudabi_sys_file_stat_get(struct thread *, struct cloudabi_sys_file_stat_get_args *); int cloudabi_sys_file_stat_put(struct thread *, struct cloudabi_sys_file_stat_put_args *); int cloudabi_sys_file_symlink(struct thread *, struct cloudabi_sys_file_symlink_args *); int cloudabi_sys_file_unlink(struct thread *, struct cloudabi_sys_file_unlink_args *); int cloudabi_sys_lock_unlock(struct thread *, struct cloudabi_sys_lock_unlock_args *); int cloudabi_sys_mem_advise(struct thread *, struct cloudabi_sys_mem_advise_args *); -int cloudabi_sys_mem_lock(struct thread *, struct cloudabi_sys_mem_lock_args *); int cloudabi_sys_mem_map(struct thread *, struct cloudabi_sys_mem_map_args *); int cloudabi_sys_mem_protect(struct thread *, struct cloudabi_sys_mem_protect_args *); int cloudabi_sys_mem_sync(struct thread *, struct cloudabi_sys_mem_sync_args *); -int cloudabi_sys_mem_unlock(struct thread *, struct cloudabi_sys_mem_unlock_args *); int cloudabi_sys_mem_unmap(struct thread *, struct cloudabi_sys_mem_unmap_args *); int cloudabi32_sys_poll(struct thread *, struct cloudabi32_sys_poll_args *); int cloudabi32_sys_poll_fd(struct thread *, struct cloudabi32_sys_poll_fd_args *); int cloudabi_sys_proc_exec(struct thread *, struct cloudabi_sys_proc_exec_args *); int cloudabi_sys_proc_exit(struct thread *, struct cloudabi_sys_proc_exit_args *); int cloudabi_sys_proc_fork(struct thread *, struct cloudabi_sys_proc_fork_args *); int cloudabi_sys_proc_raise(struct thread *, struct cloudabi_sys_proc_raise_args *); int cloudabi_sys_random_get(struct thread *, struct cloudabi_sys_random_get_args *); int cloudabi_sys_sock_accept(struct thread *, struct cloudabi_sys_sock_accept_args *); int cloudabi_sys_sock_bind(struct thread *, struct cloudabi_sys_sock_bind_args *); int cloudabi_sys_sock_connect(struct thread *, struct cloudabi_sys_sock_connect_args *); int cloudabi_sys_sock_listen(struct thread *, struct cloudabi_sys_sock_listen_args *); int cloudabi32_sys_sock_recv(struct thread *, struct cloudabi32_sys_sock_recv_args *); int cloudabi32_sys_sock_send(struct thread *, struct cloudabi32_sys_sock_send_args *); int cloudabi_sys_sock_shutdown(struct thread *, struct cloudabi_sys_sock_shutdown_args *); int cloudabi_sys_sock_stat_get(struct thread *, struct cloudabi_sys_sock_stat_get_args *); int cloudabi32_sys_thread_create(struct thread *, struct cloudabi32_sys_thread_create_args *); int cloudabi_sys_thread_exit(struct thread *, struct cloudabi_sys_thread_exit_args *); int cloudabi_sys_thread_yield(struct thread *, struct cloudabi_sys_thread_yield_args *); #ifdef COMPAT_43 #endif /* COMPAT_43 */ #ifdef COMPAT_FREEBSD4 #endif /* COMPAT_FREEBSD4 */ #ifdef COMPAT_FREEBSD6 #endif /* COMPAT_FREEBSD6 */ #ifdef COMPAT_FREEBSD7 #endif /* COMPAT_FREEBSD7 */ #ifdef COMPAT_FREEBSD10 #endif /* COMPAT_FREEBSD10 */ + +#ifdef COMPAT_FREEBSD11 + + +#endif /* COMPAT_FREEBSD11 */ + #define CLOUDABI32_SYS_AUE_cloudabi_sys_clock_res_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_clock_time_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_condvar_signal AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_close AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_create1 AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_create2 AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_datasync AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_dup AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_fd_pread AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_fd_pwrite AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_fd_read AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_replace AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_seek AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_stat_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_stat_put AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_fd_sync AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_fd_write AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_advise AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_allocate AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_create AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_link AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_open AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_readdir AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_readlink AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_rename AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_stat_fget AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_stat_fput AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_stat_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_stat_put AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_symlink AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_file_unlink AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_lock_unlock AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_advise AUE_NULL -#define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_lock AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_map AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_protect AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_sync AUE_NULL -#define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_unlock AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_mem_unmap AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_poll AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_poll_fd AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_proc_exec AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_proc_exit AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_proc_fork AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_proc_raise AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_random_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_accept AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_bind AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_connect AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_listen AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_sock_recv AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_sock_send AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_shutdown AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_sock_stat_get AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi32_sys_thread_create AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_thread_exit AUE_NULL #define CLOUDABI32_SYS_AUE_cloudabi_sys_thread_yield AUE_NULL #undef PAD_ #undef PADL_ #undef PADR_ #endif /* !_CLOUDABI32_SYSPROTO_H_ */ Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sock.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sock.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sock.c (revision 321585) @@ -1,128 +1,128 @@ /*- * Copyright (c) 2015-2017 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_SOCKET, "socket", "CloudABI socket"); int cloudabi32_sys_sock_recv(struct thread *td, struct cloudabi32_sys_sock_recv_args *uap) { cloudabi32_recv_in_t ri; cloudabi32_recv_out_t ro = {}; cloudabi32_iovec_t iovobj; struct iovec *iov; const cloudabi32_iovec_t *user_iov; size_t i, rdatalen, rfdslen; int error; error = copyin(uap->in, &ri, sizeof(ri)); if (error != 0) return (error); /* Convert iovecs to native format. */ if (ri.ri_data_len > UIO_MAXIOV) return (EINVAL); iov = malloc(ri.ri_data_len * sizeof(struct iovec), M_SOCKET, M_WAITOK); user_iov = TO_PTR(ri.ri_data); for (i = 0; i < ri.ri_data_len; i++) { error = copyin(&user_iov[i], &iovobj, sizeof(iovobj)); if (error != 0) { free(iov, M_SOCKET); return (error); } iov[i].iov_base = TO_PTR(iovobj.buf); iov[i].iov_len = iovobj.buf_len; } error = cloudabi_sock_recv(td, uap->sock, iov, ri.ri_data_len, TO_PTR(ri.ri_fds), ri.ri_fds_len, ri.ri_flags, &rdatalen, - &rfdslen, &ro.ro_peername, &ro.ro_flags); + &rfdslen, &ro.ro_flags); free(iov, M_SOCKET); if (error != 0) return (error); ro.ro_datalen = rdatalen; ro.ro_fdslen = rfdslen; return (copyout(&ro, uap->out, sizeof(ro))); } int cloudabi32_sys_sock_send(struct thread *td, struct cloudabi32_sys_sock_send_args *uap) { cloudabi32_send_in_t si; cloudabi32_send_out_t so = {}; cloudabi32_ciovec_t iovobj; struct iovec *iov; const cloudabi32_ciovec_t *user_iov; size_t datalen, i; int error; error = copyin(uap->in, &si, sizeof(si)); if (error != 0) return (error); /* Convert iovecs to native format. */ if (si.si_data_len > UIO_MAXIOV) return (EINVAL); iov = malloc(si.si_data_len * sizeof(struct iovec), M_SOCKET, M_WAITOK); user_iov = TO_PTR(si.si_data); for (i = 0; i < si.si_data_len; i++) { error = copyin(&user_iov[i], &iovobj, sizeof(iovobj)); if (error != 0) { free(iov, M_SOCKET); return (error); } iov[i].iov_base = TO_PTR(iovobj.buf); iov[i].iov_len = iovobj.buf_len; } error = cloudabi_sock_send(td, uap->sock, iov, si.si_data_len, - TO_PTR(si.si_fds), si.si_fds_len, si.si_flags, &datalen); + TO_PTR(si.si_fds), si.si_fds_len, &datalen); free(iov, M_SOCKET); if (error != 0) return (error); so.so_datalen = datalen; return (copyout(&so, uap->out, sizeof(so))); } Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscall.h =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscall.h (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscall.h (revision 321585) @@ -1,65 +1,63 @@ /* * System call numbers. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #define CLOUDABI32_SYS_cloudabi_sys_clock_res_get 0 #define CLOUDABI32_SYS_cloudabi_sys_clock_time_get 1 #define CLOUDABI32_SYS_cloudabi_sys_condvar_signal 2 #define CLOUDABI32_SYS_cloudabi_sys_fd_close 3 #define CLOUDABI32_SYS_cloudabi_sys_fd_create1 4 #define CLOUDABI32_SYS_cloudabi_sys_fd_create2 5 #define CLOUDABI32_SYS_cloudabi_sys_fd_datasync 6 #define CLOUDABI32_SYS_cloudabi_sys_fd_dup 7 #define CLOUDABI32_SYS_cloudabi32_sys_fd_pread 8 #define CLOUDABI32_SYS_cloudabi32_sys_fd_pwrite 9 #define CLOUDABI32_SYS_cloudabi32_sys_fd_read 10 #define CLOUDABI32_SYS_cloudabi_sys_fd_replace 11 #define CLOUDABI32_SYS_cloudabi_sys_fd_seek 12 #define CLOUDABI32_SYS_cloudabi_sys_fd_stat_get 13 #define CLOUDABI32_SYS_cloudabi_sys_fd_stat_put 14 #define CLOUDABI32_SYS_cloudabi_sys_fd_sync 15 #define CLOUDABI32_SYS_cloudabi32_sys_fd_write 16 #define CLOUDABI32_SYS_cloudabi_sys_file_advise 17 #define CLOUDABI32_SYS_cloudabi_sys_file_allocate 18 #define CLOUDABI32_SYS_cloudabi_sys_file_create 19 #define CLOUDABI32_SYS_cloudabi_sys_file_link 20 #define CLOUDABI32_SYS_cloudabi_sys_file_open 21 #define CLOUDABI32_SYS_cloudabi_sys_file_readdir 22 #define CLOUDABI32_SYS_cloudabi_sys_file_readlink 23 #define CLOUDABI32_SYS_cloudabi_sys_file_rename 24 #define CLOUDABI32_SYS_cloudabi_sys_file_stat_fget 25 #define CLOUDABI32_SYS_cloudabi_sys_file_stat_fput 26 #define CLOUDABI32_SYS_cloudabi_sys_file_stat_get 27 #define CLOUDABI32_SYS_cloudabi_sys_file_stat_put 28 #define CLOUDABI32_SYS_cloudabi_sys_file_symlink 29 #define CLOUDABI32_SYS_cloudabi_sys_file_unlink 30 #define CLOUDABI32_SYS_cloudabi_sys_lock_unlock 31 #define CLOUDABI32_SYS_cloudabi_sys_mem_advise 32 -#define CLOUDABI32_SYS_cloudabi_sys_mem_lock 33 -#define CLOUDABI32_SYS_cloudabi_sys_mem_map 34 -#define CLOUDABI32_SYS_cloudabi_sys_mem_protect 35 -#define CLOUDABI32_SYS_cloudabi_sys_mem_sync 36 -#define CLOUDABI32_SYS_cloudabi_sys_mem_unlock 37 -#define CLOUDABI32_SYS_cloudabi_sys_mem_unmap 38 -#define CLOUDABI32_SYS_cloudabi32_sys_poll 39 -#define CLOUDABI32_SYS_cloudabi32_sys_poll_fd 40 -#define CLOUDABI32_SYS_cloudabi_sys_proc_exec 41 -#define CLOUDABI32_SYS_cloudabi_sys_proc_exit 42 -#define CLOUDABI32_SYS_cloudabi_sys_proc_fork 43 -#define CLOUDABI32_SYS_cloudabi_sys_proc_raise 44 -#define CLOUDABI32_SYS_cloudabi_sys_random_get 45 -#define CLOUDABI32_SYS_cloudabi_sys_sock_accept 46 -#define CLOUDABI32_SYS_cloudabi_sys_sock_bind 47 -#define CLOUDABI32_SYS_cloudabi_sys_sock_connect 48 -#define CLOUDABI32_SYS_cloudabi_sys_sock_listen 49 -#define CLOUDABI32_SYS_cloudabi32_sys_sock_recv 50 -#define CLOUDABI32_SYS_cloudabi32_sys_sock_send 51 -#define CLOUDABI32_SYS_cloudabi_sys_sock_shutdown 52 -#define CLOUDABI32_SYS_cloudabi_sys_sock_stat_get 53 -#define CLOUDABI32_SYS_cloudabi32_sys_thread_create 54 -#define CLOUDABI32_SYS_cloudabi_sys_thread_exit 55 -#define CLOUDABI32_SYS_cloudabi_sys_thread_yield 56 -#define CLOUDABI32_SYS_MAXSYSCALL 57 +#define CLOUDABI32_SYS_cloudabi_sys_mem_map 33 +#define CLOUDABI32_SYS_cloudabi_sys_mem_protect 34 +#define CLOUDABI32_SYS_cloudabi_sys_mem_sync 35 +#define CLOUDABI32_SYS_cloudabi_sys_mem_unmap 36 +#define CLOUDABI32_SYS_cloudabi32_sys_poll 37 +#define CLOUDABI32_SYS_cloudabi32_sys_poll_fd 38 +#define CLOUDABI32_SYS_cloudabi_sys_proc_exec 39 +#define CLOUDABI32_SYS_cloudabi_sys_proc_exit 40 +#define CLOUDABI32_SYS_cloudabi_sys_proc_fork 41 +#define CLOUDABI32_SYS_cloudabi_sys_proc_raise 42 +#define CLOUDABI32_SYS_cloudabi_sys_random_get 43 +#define CLOUDABI32_SYS_cloudabi_sys_sock_accept 44 +#define CLOUDABI32_SYS_cloudabi_sys_sock_bind 45 +#define CLOUDABI32_SYS_cloudabi_sys_sock_connect 46 +#define CLOUDABI32_SYS_cloudabi_sys_sock_listen 47 +#define CLOUDABI32_SYS_cloudabi32_sys_sock_recv 48 +#define CLOUDABI32_SYS_cloudabi32_sys_sock_send 49 +#define CLOUDABI32_SYS_cloudabi_sys_sock_shutdown 50 +#define CLOUDABI32_SYS_cloudabi_sys_sock_stat_get 51 +#define CLOUDABI32_SYS_cloudabi32_sys_thread_create 52 +#define CLOUDABI32_SYS_cloudabi_sys_thread_exit 53 +#define CLOUDABI32_SYS_cloudabi_sys_thread_yield 54 +#define CLOUDABI32_SYS_MAXSYSCALL 55 Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscalls.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscalls.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_syscalls.c (revision 321585) @@ -1,66 +1,64 @@ /* * System call names. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ const char *cloudabi32_syscallnames[] = { "cloudabi_sys_clock_res_get", /* 0 = cloudabi_sys_clock_res_get */ "cloudabi_sys_clock_time_get", /* 1 = cloudabi_sys_clock_time_get */ "cloudabi_sys_condvar_signal", /* 2 = cloudabi_sys_condvar_signal */ "cloudabi_sys_fd_close", /* 3 = cloudabi_sys_fd_close */ "cloudabi_sys_fd_create1", /* 4 = cloudabi_sys_fd_create1 */ "cloudabi_sys_fd_create2", /* 5 = cloudabi_sys_fd_create2 */ "cloudabi_sys_fd_datasync", /* 6 = cloudabi_sys_fd_datasync */ "cloudabi_sys_fd_dup", /* 7 = cloudabi_sys_fd_dup */ "cloudabi32_sys_fd_pread", /* 8 = cloudabi32_sys_fd_pread */ "cloudabi32_sys_fd_pwrite", /* 9 = cloudabi32_sys_fd_pwrite */ "cloudabi32_sys_fd_read", /* 10 = cloudabi32_sys_fd_read */ "cloudabi_sys_fd_replace", /* 11 = cloudabi_sys_fd_replace */ "cloudabi_sys_fd_seek", /* 12 = cloudabi_sys_fd_seek */ "cloudabi_sys_fd_stat_get", /* 13 = cloudabi_sys_fd_stat_get */ "cloudabi_sys_fd_stat_put", /* 14 = cloudabi_sys_fd_stat_put */ "cloudabi_sys_fd_sync", /* 15 = cloudabi_sys_fd_sync */ "cloudabi32_sys_fd_write", /* 16 = cloudabi32_sys_fd_write */ "cloudabi_sys_file_advise", /* 17 = cloudabi_sys_file_advise */ "cloudabi_sys_file_allocate", /* 18 = cloudabi_sys_file_allocate */ "cloudabi_sys_file_create", /* 19 = cloudabi_sys_file_create */ "cloudabi_sys_file_link", /* 20 = cloudabi_sys_file_link */ "cloudabi_sys_file_open", /* 21 = cloudabi_sys_file_open */ "cloudabi_sys_file_readdir", /* 22 = cloudabi_sys_file_readdir */ "cloudabi_sys_file_readlink", /* 23 = cloudabi_sys_file_readlink */ "cloudabi_sys_file_rename", /* 24 = cloudabi_sys_file_rename */ "cloudabi_sys_file_stat_fget", /* 25 = cloudabi_sys_file_stat_fget */ "cloudabi_sys_file_stat_fput", /* 26 = cloudabi_sys_file_stat_fput */ "cloudabi_sys_file_stat_get", /* 27 = cloudabi_sys_file_stat_get */ "cloudabi_sys_file_stat_put", /* 28 = cloudabi_sys_file_stat_put */ "cloudabi_sys_file_symlink", /* 29 = cloudabi_sys_file_symlink */ "cloudabi_sys_file_unlink", /* 30 = cloudabi_sys_file_unlink */ "cloudabi_sys_lock_unlock", /* 31 = cloudabi_sys_lock_unlock */ "cloudabi_sys_mem_advise", /* 32 = cloudabi_sys_mem_advise */ - "cloudabi_sys_mem_lock", /* 33 = cloudabi_sys_mem_lock */ - "cloudabi_sys_mem_map", /* 34 = cloudabi_sys_mem_map */ - "cloudabi_sys_mem_protect", /* 35 = cloudabi_sys_mem_protect */ - "cloudabi_sys_mem_sync", /* 36 = cloudabi_sys_mem_sync */ - "cloudabi_sys_mem_unlock", /* 37 = cloudabi_sys_mem_unlock */ - "cloudabi_sys_mem_unmap", /* 38 = cloudabi_sys_mem_unmap */ - "cloudabi32_sys_poll", /* 39 = cloudabi32_sys_poll */ - "cloudabi32_sys_poll_fd", /* 40 = cloudabi32_sys_poll_fd */ - "cloudabi_sys_proc_exec", /* 41 = cloudabi_sys_proc_exec */ - "cloudabi_sys_proc_exit", /* 42 = cloudabi_sys_proc_exit */ - "cloudabi_sys_proc_fork", /* 43 = cloudabi_sys_proc_fork */ - "cloudabi_sys_proc_raise", /* 44 = cloudabi_sys_proc_raise */ - "cloudabi_sys_random_get", /* 45 = cloudabi_sys_random_get */ - "cloudabi_sys_sock_accept", /* 46 = cloudabi_sys_sock_accept */ - "cloudabi_sys_sock_bind", /* 47 = cloudabi_sys_sock_bind */ - "cloudabi_sys_sock_connect", /* 48 = cloudabi_sys_sock_connect */ - "cloudabi_sys_sock_listen", /* 49 = cloudabi_sys_sock_listen */ - "cloudabi32_sys_sock_recv", /* 50 = cloudabi32_sys_sock_recv */ - "cloudabi32_sys_sock_send", /* 51 = cloudabi32_sys_sock_send */ - "cloudabi_sys_sock_shutdown", /* 52 = cloudabi_sys_sock_shutdown */ - "cloudabi_sys_sock_stat_get", /* 53 = cloudabi_sys_sock_stat_get */ - "cloudabi32_sys_thread_create", /* 54 = cloudabi32_sys_thread_create */ - "cloudabi_sys_thread_exit", /* 55 = cloudabi_sys_thread_exit */ - "cloudabi_sys_thread_yield", /* 56 = cloudabi_sys_thread_yield */ + "cloudabi_sys_mem_map", /* 33 = cloudabi_sys_mem_map */ + "cloudabi_sys_mem_protect", /* 34 = cloudabi_sys_mem_protect */ + "cloudabi_sys_mem_sync", /* 35 = cloudabi_sys_mem_sync */ + "cloudabi_sys_mem_unmap", /* 36 = cloudabi_sys_mem_unmap */ + "cloudabi32_sys_poll", /* 37 = cloudabi32_sys_poll */ + "cloudabi32_sys_poll_fd", /* 38 = cloudabi32_sys_poll_fd */ + "cloudabi_sys_proc_exec", /* 39 = cloudabi_sys_proc_exec */ + "cloudabi_sys_proc_exit", /* 40 = cloudabi_sys_proc_exit */ + "cloudabi_sys_proc_fork", /* 41 = cloudabi_sys_proc_fork */ + "cloudabi_sys_proc_raise", /* 42 = cloudabi_sys_proc_raise */ + "cloudabi_sys_random_get", /* 43 = cloudabi_sys_random_get */ + "cloudabi_sys_sock_accept", /* 44 = cloudabi_sys_sock_accept */ + "cloudabi_sys_sock_bind", /* 45 = cloudabi_sys_sock_bind */ + "cloudabi_sys_sock_connect", /* 46 = cloudabi_sys_sock_connect */ + "cloudabi_sys_sock_listen", /* 47 = cloudabi_sys_sock_listen */ + "cloudabi32_sys_sock_recv", /* 48 = cloudabi32_sys_sock_recv */ + "cloudabi32_sys_sock_send", /* 49 = cloudabi32_sys_sock_send */ + "cloudabi_sys_sock_shutdown", /* 50 = cloudabi_sys_sock_shutdown */ + "cloudabi_sys_sock_stat_get", /* 51 = cloudabi_sys_sock_stat_get */ + "cloudabi32_sys_thread_create", /* 52 = cloudabi32_sys_thread_create */ + "cloudabi_sys_thread_exit", /* 53 = cloudabi_sys_thread_exit */ + "cloudabi_sys_thread_yield", /* 54 = cloudabi_sys_thread_yield */ }; Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sysent.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sysent.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_sysent.c (revision 321585) @@ -1,74 +1,72 @@ /* * System call switch table. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #include #include #include #include #define AS(name) (sizeof(struct name) / sizeof(register_t)) /* The casts are bogus but will do for now. */ struct sysent cloudabi32_sysent[] = { { AS(cloudabi_sys_clock_res_get_args), (sy_call_t *)cloudabi_sys_clock_res_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 0 = cloudabi_sys_clock_res_get */ { AS(cloudabi_sys_clock_time_get_args), (sy_call_t *)cloudabi_sys_clock_time_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 1 = cloudabi_sys_clock_time_get */ { AS(cloudabi_sys_condvar_signal_args), (sy_call_t *)cloudabi_sys_condvar_signal, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 2 = cloudabi_sys_condvar_signal */ { AS(cloudabi_sys_fd_close_args), (sy_call_t *)cloudabi_sys_fd_close, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 3 = cloudabi_sys_fd_close */ { AS(cloudabi_sys_fd_create1_args), (sy_call_t *)cloudabi_sys_fd_create1, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 4 = cloudabi_sys_fd_create1 */ { AS(cloudabi_sys_fd_create2_args), (sy_call_t *)cloudabi_sys_fd_create2, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 5 = cloudabi_sys_fd_create2 */ { AS(cloudabi_sys_fd_datasync_args), (sy_call_t *)cloudabi_sys_fd_datasync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 6 = cloudabi_sys_fd_datasync */ { AS(cloudabi_sys_fd_dup_args), (sy_call_t *)cloudabi_sys_fd_dup, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 7 = cloudabi_sys_fd_dup */ { AS(cloudabi32_sys_fd_pread_args), (sy_call_t *)cloudabi32_sys_fd_pread, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 8 = cloudabi32_sys_fd_pread */ { AS(cloudabi32_sys_fd_pwrite_args), (sy_call_t *)cloudabi32_sys_fd_pwrite, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 9 = cloudabi32_sys_fd_pwrite */ { AS(cloudabi32_sys_fd_read_args), (sy_call_t *)cloudabi32_sys_fd_read, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 10 = cloudabi32_sys_fd_read */ { AS(cloudabi_sys_fd_replace_args), (sy_call_t *)cloudabi_sys_fd_replace, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 11 = cloudabi_sys_fd_replace */ { AS(cloudabi_sys_fd_seek_args), (sy_call_t *)cloudabi_sys_fd_seek, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 12 = cloudabi_sys_fd_seek */ { AS(cloudabi_sys_fd_stat_get_args), (sy_call_t *)cloudabi_sys_fd_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 13 = cloudabi_sys_fd_stat_get */ { AS(cloudabi_sys_fd_stat_put_args), (sy_call_t *)cloudabi_sys_fd_stat_put, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 14 = cloudabi_sys_fd_stat_put */ { AS(cloudabi_sys_fd_sync_args), (sy_call_t *)cloudabi_sys_fd_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 15 = cloudabi_sys_fd_sync */ { AS(cloudabi32_sys_fd_write_args), (sy_call_t *)cloudabi32_sys_fd_write, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 16 = cloudabi32_sys_fd_write */ { AS(cloudabi_sys_file_advise_args), (sy_call_t *)cloudabi_sys_file_advise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 17 = cloudabi_sys_file_advise */ { AS(cloudabi_sys_file_allocate_args), (sy_call_t *)cloudabi_sys_file_allocate, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 18 = cloudabi_sys_file_allocate */ { AS(cloudabi_sys_file_create_args), (sy_call_t *)cloudabi_sys_file_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 19 = cloudabi_sys_file_create */ { AS(cloudabi_sys_file_link_args), (sy_call_t *)cloudabi_sys_file_link, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 20 = cloudabi_sys_file_link */ { AS(cloudabi_sys_file_open_args), (sy_call_t *)cloudabi_sys_file_open, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 21 = cloudabi_sys_file_open */ { AS(cloudabi_sys_file_readdir_args), (sy_call_t *)cloudabi_sys_file_readdir, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 22 = cloudabi_sys_file_readdir */ { AS(cloudabi_sys_file_readlink_args), (sy_call_t *)cloudabi_sys_file_readlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 23 = cloudabi_sys_file_readlink */ { AS(cloudabi_sys_file_rename_args), (sy_call_t *)cloudabi_sys_file_rename, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 24 = cloudabi_sys_file_rename */ { AS(cloudabi_sys_file_stat_fget_args), (sy_call_t *)cloudabi_sys_file_stat_fget, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 25 = cloudabi_sys_file_stat_fget */ { AS(cloudabi_sys_file_stat_fput_args), (sy_call_t *)cloudabi_sys_file_stat_fput, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 26 = cloudabi_sys_file_stat_fput */ { AS(cloudabi_sys_file_stat_get_args), (sy_call_t *)cloudabi_sys_file_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 27 = cloudabi_sys_file_stat_get */ { AS(cloudabi_sys_file_stat_put_args), (sy_call_t *)cloudabi_sys_file_stat_put, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 28 = cloudabi_sys_file_stat_put */ { AS(cloudabi_sys_file_symlink_args), (sy_call_t *)cloudabi_sys_file_symlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 29 = cloudabi_sys_file_symlink */ { AS(cloudabi_sys_file_unlink_args), (sy_call_t *)cloudabi_sys_file_unlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 30 = cloudabi_sys_file_unlink */ { AS(cloudabi_sys_lock_unlock_args), (sy_call_t *)cloudabi_sys_lock_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 31 = cloudabi_sys_lock_unlock */ { AS(cloudabi_sys_mem_advise_args), (sy_call_t *)cloudabi_sys_mem_advise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 32 = cloudabi_sys_mem_advise */ - { AS(cloudabi_sys_mem_lock_args), (sy_call_t *)cloudabi_sys_mem_lock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 33 = cloudabi_sys_mem_lock */ - { AS(cloudabi_sys_mem_map_args), (sy_call_t *)cloudabi_sys_mem_map, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 34 = cloudabi_sys_mem_map */ - { AS(cloudabi_sys_mem_protect_args), (sy_call_t *)cloudabi_sys_mem_protect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 35 = cloudabi_sys_mem_protect */ - { AS(cloudabi_sys_mem_sync_args), (sy_call_t *)cloudabi_sys_mem_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 36 = cloudabi_sys_mem_sync */ - { AS(cloudabi_sys_mem_unlock_args), (sy_call_t *)cloudabi_sys_mem_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 37 = cloudabi_sys_mem_unlock */ - { AS(cloudabi_sys_mem_unmap_args), (sy_call_t *)cloudabi_sys_mem_unmap, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 38 = cloudabi_sys_mem_unmap */ - { AS(cloudabi32_sys_poll_args), (sy_call_t *)cloudabi32_sys_poll, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 39 = cloudabi32_sys_poll */ - { AS(cloudabi32_sys_poll_fd_args), (sy_call_t *)cloudabi32_sys_poll_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 40 = cloudabi32_sys_poll_fd */ - { AS(cloudabi_sys_proc_exec_args), (sy_call_t *)cloudabi_sys_proc_exec, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 41 = cloudabi_sys_proc_exec */ - { AS(cloudabi_sys_proc_exit_args), (sy_call_t *)cloudabi_sys_proc_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 42 = cloudabi_sys_proc_exit */ - { 0, (sy_call_t *)cloudabi_sys_proc_fork, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 43 = cloudabi_sys_proc_fork */ - { AS(cloudabi_sys_proc_raise_args), (sy_call_t *)cloudabi_sys_proc_raise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 44 = cloudabi_sys_proc_raise */ - { AS(cloudabi_sys_random_get_args), (sy_call_t *)cloudabi_sys_random_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 45 = cloudabi_sys_random_get */ - { AS(cloudabi_sys_sock_accept_args), (sy_call_t *)cloudabi_sys_sock_accept, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 46 = cloudabi_sys_sock_accept */ - { AS(cloudabi_sys_sock_bind_args), (sy_call_t *)cloudabi_sys_sock_bind, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 47 = cloudabi_sys_sock_bind */ - { AS(cloudabi_sys_sock_connect_args), (sy_call_t *)cloudabi_sys_sock_connect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 48 = cloudabi_sys_sock_connect */ - { AS(cloudabi_sys_sock_listen_args), (sy_call_t *)cloudabi_sys_sock_listen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 49 = cloudabi_sys_sock_listen */ - { AS(cloudabi32_sys_sock_recv_args), (sy_call_t *)cloudabi32_sys_sock_recv, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 50 = cloudabi32_sys_sock_recv */ - { AS(cloudabi32_sys_sock_send_args), (sy_call_t *)cloudabi32_sys_sock_send, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 51 = cloudabi32_sys_sock_send */ - { AS(cloudabi_sys_sock_shutdown_args), (sy_call_t *)cloudabi_sys_sock_shutdown, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 52 = cloudabi_sys_sock_shutdown */ - { AS(cloudabi_sys_sock_stat_get_args), (sy_call_t *)cloudabi_sys_sock_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 53 = cloudabi_sys_sock_stat_get */ - { AS(cloudabi32_sys_thread_create_args), (sy_call_t *)cloudabi32_sys_thread_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 54 = cloudabi32_sys_thread_create */ - { AS(cloudabi_sys_thread_exit_args), (sy_call_t *)cloudabi_sys_thread_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 55 = cloudabi_sys_thread_exit */ - { 0, (sy_call_t *)cloudabi_sys_thread_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 56 = cloudabi_sys_thread_yield */ + { AS(cloudabi_sys_mem_map_args), (sy_call_t *)cloudabi_sys_mem_map, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 33 = cloudabi_sys_mem_map */ + { AS(cloudabi_sys_mem_protect_args), (sy_call_t *)cloudabi_sys_mem_protect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 34 = cloudabi_sys_mem_protect */ + { AS(cloudabi_sys_mem_sync_args), (sy_call_t *)cloudabi_sys_mem_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 35 = cloudabi_sys_mem_sync */ + { AS(cloudabi_sys_mem_unmap_args), (sy_call_t *)cloudabi_sys_mem_unmap, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 36 = cloudabi_sys_mem_unmap */ + { AS(cloudabi32_sys_poll_args), (sy_call_t *)cloudabi32_sys_poll, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 37 = cloudabi32_sys_poll */ + { AS(cloudabi32_sys_poll_fd_args), (sy_call_t *)cloudabi32_sys_poll_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 38 = cloudabi32_sys_poll_fd */ + { AS(cloudabi_sys_proc_exec_args), (sy_call_t *)cloudabi_sys_proc_exec, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 39 = cloudabi_sys_proc_exec */ + { AS(cloudabi_sys_proc_exit_args), (sy_call_t *)cloudabi_sys_proc_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 40 = cloudabi_sys_proc_exit */ + { 0, (sy_call_t *)cloudabi_sys_proc_fork, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 41 = cloudabi_sys_proc_fork */ + { AS(cloudabi_sys_proc_raise_args), (sy_call_t *)cloudabi_sys_proc_raise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 42 = cloudabi_sys_proc_raise */ + { AS(cloudabi_sys_random_get_args), (sy_call_t *)cloudabi_sys_random_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 43 = cloudabi_sys_random_get */ + { AS(cloudabi_sys_sock_accept_args), (sy_call_t *)cloudabi_sys_sock_accept, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 44 = cloudabi_sys_sock_accept */ + { AS(cloudabi_sys_sock_bind_args), (sy_call_t *)cloudabi_sys_sock_bind, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 45 = cloudabi_sys_sock_bind */ + { AS(cloudabi_sys_sock_connect_args), (sy_call_t *)cloudabi_sys_sock_connect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 46 = cloudabi_sys_sock_connect */ + { AS(cloudabi_sys_sock_listen_args), (sy_call_t *)cloudabi_sys_sock_listen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 47 = cloudabi_sys_sock_listen */ + { AS(cloudabi32_sys_sock_recv_args), (sy_call_t *)cloudabi32_sys_sock_recv, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 48 = cloudabi32_sys_sock_recv */ + { AS(cloudabi32_sys_sock_send_args), (sy_call_t *)cloudabi32_sys_sock_send, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 49 = cloudabi32_sys_sock_send */ + { AS(cloudabi_sys_sock_shutdown_args), (sy_call_t *)cloudabi_sys_sock_shutdown, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 50 = cloudabi_sys_sock_shutdown */ + { AS(cloudabi_sys_sock_stat_get_args), (sy_call_t *)cloudabi_sys_sock_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 51 = cloudabi_sys_sock_stat_get */ + { AS(cloudabi32_sys_thread_create_args), (sy_call_t *)cloudabi32_sys_thread_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 52 = cloudabi32_sys_thread_create */ + { AS(cloudabi_sys_thread_exit_args), (sy_call_t *)cloudabi_sys_thread_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 53 = cloudabi_sys_thread_exit */ + { 0, (sy_call_t *)cloudabi_sys_thread_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 54 = cloudabi_sys_thread_yield */ }; Index: projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_systrace_args.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_systrace_args.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi32/cloudabi32_systrace_args.c (revision 321585) @@ -1,1702 +1,1650 @@ /* * System call argument to DTrace register array converstion. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ * This file is part of the DTrace syscall provider. */ static void systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) { int64_t *iarg = (int64_t *) uarg; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: { struct cloudabi_sys_clock_res_get_args *p = params; iarg[0] = p->clock_id; /* cloudabi_clockid_t */ *n_args = 1; break; } /* cloudabi_sys_clock_time_get */ case 1: { struct cloudabi_sys_clock_time_get_args *p = params; iarg[0] = p->clock_id; /* cloudabi_clockid_t */ iarg[1] = p->precision; /* cloudabi_timestamp_t */ *n_args = 2; break; } /* cloudabi_sys_condvar_signal */ case 2: { struct cloudabi_sys_condvar_signal_args *p = params; uarg[0] = (intptr_t) p->condvar; /* cloudabi_condvar_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ iarg[2] = p->nwaiters; /* cloudabi_nthreads_t */ *n_args = 3; break; } /* cloudabi_sys_fd_close */ case 3: { struct cloudabi_sys_fd_close_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi_sys_fd_create1 */ case 4: { struct cloudabi_sys_fd_create1_args *p = params; iarg[0] = p->type; /* cloudabi_filetype_t */ *n_args = 1; break; } /* cloudabi_sys_fd_create2 */ case 5: { struct cloudabi_sys_fd_create2_args *p = params; iarg[0] = p->type; /* cloudabi_filetype_t */ *n_args = 1; break; } /* cloudabi_sys_fd_datasync */ case 6: { struct cloudabi_sys_fd_datasync_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi_sys_fd_dup */ case 7: { struct cloudabi_sys_fd_dup_args *p = params; iarg[0] = p->from; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi32_sys_fd_pread */ case 8: { struct cloudabi32_sys_fd_pread_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi32_iovec_t * */ uarg[2] = p->iovs_len; /* size_t */ iarg[3] = p->offset; /* cloudabi_filesize_t */ *n_args = 4; break; } /* cloudabi32_sys_fd_pwrite */ case 9: { struct cloudabi32_sys_fd_pwrite_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi32_ciovec_t * */ uarg[2] = p->iovs_len; /* size_t */ iarg[3] = p->offset; /* cloudabi_filesize_t */ *n_args = 4; break; } /* cloudabi32_sys_fd_read */ case 10: { struct cloudabi32_sys_fd_read_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi32_iovec_t * */ uarg[2] = p->iovs_len; /* size_t */ *n_args = 3; break; } /* cloudabi_sys_fd_replace */ case 11: { struct cloudabi_sys_fd_replace_args *p = params; iarg[0] = p->from; /* cloudabi_fd_t */ iarg[1] = p->to; /* cloudabi_fd_t */ *n_args = 2; break; } /* cloudabi_sys_fd_seek */ case 12: { struct cloudabi_sys_fd_seek_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filedelta_t */ iarg[2] = p->whence; /* cloudabi_whence_t */ *n_args = 3; break; } /* cloudabi_sys_fd_stat_get */ case 13: { struct cloudabi_sys_fd_stat_get_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_fdstat_t * */ *n_args = 2; break; } /* cloudabi_sys_fd_stat_put */ case 14: { struct cloudabi_sys_fd_stat_put_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* const cloudabi_fdstat_t * */ iarg[2] = p->flags; /* cloudabi_fdsflags_t */ *n_args = 3; break; } /* cloudabi_sys_fd_sync */ case 15: { struct cloudabi_sys_fd_sync_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi32_sys_fd_write */ case 16: { struct cloudabi32_sys_fd_write_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi32_ciovec_t * */ uarg[2] = p->iovs_len; /* size_t */ *n_args = 3; break; } /* cloudabi_sys_file_advise */ case 17: { struct cloudabi_sys_file_advise_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filesize_t */ iarg[2] = p->len; /* cloudabi_filesize_t */ iarg[3] = p->advice; /* cloudabi_advice_t */ *n_args = 4; break; } /* cloudabi_sys_file_allocate */ case 18: { struct cloudabi_sys_file_allocate_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filesize_t */ iarg[2] = p->len; /* cloudabi_filesize_t */ *n_args = 3; break; } /* cloudabi_sys_file_create */ case 19: { struct cloudabi_sys_file_create_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->type; /* cloudabi_filetype_t */ *n_args = 4; break; } /* cloudabi_sys_file_link */ case 20: { struct cloudabi_sys_file_link_args *p = params; iarg[0] = p->fd1; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path1; /* const char * */ uarg[2] = p->path1_len; /* size_t */ iarg[3] = p->fd2; /* cloudabi_fd_t */ uarg[4] = (intptr_t) p->path2; /* const char * */ uarg[5] = p->path2_len; /* size_t */ *n_args = 6; break; } /* cloudabi_sys_file_open */ case 21: { struct cloudabi_sys_file_open_args *p = params; iarg[0] = p->dirfd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->oflags; /* cloudabi_oflags_t */ uarg[4] = (intptr_t) p->fds; /* const cloudabi_fdstat_t * */ *n_args = 5; break; } /* cloudabi_sys_file_readdir */ case 22: { struct cloudabi_sys_file_readdir_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* void * */ uarg[2] = p->buf_len; /* size_t */ iarg[3] = p->cookie; /* cloudabi_dircookie_t */ *n_args = 4; break; } /* cloudabi_sys_file_readlink */ case 23: { struct cloudabi_sys_file_readlink_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* char * */ uarg[4] = p->buf_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_file_rename */ case 24: { struct cloudabi_sys_file_rename_args *p = params; iarg[0] = p->fd1; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path1; /* const char * */ uarg[2] = p->path1_len; /* size_t */ iarg[3] = p->fd2; /* cloudabi_fd_t */ uarg[4] = (intptr_t) p->path2; /* const char * */ uarg[5] = p->path2_len; /* size_t */ *n_args = 6; break; } /* cloudabi_sys_file_stat_fget */ case 25: { struct cloudabi_sys_file_stat_fget_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_filestat_t * */ *n_args = 2; break; } /* cloudabi_sys_file_stat_fput */ case 26: { struct cloudabi_sys_file_stat_fput_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* const cloudabi_filestat_t * */ iarg[2] = p->flags; /* cloudabi_fsflags_t */ *n_args = 3; break; } /* cloudabi_sys_file_stat_get */ case 27: { struct cloudabi_sys_file_stat_get_args *p = params; iarg[0] = p->fd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* cloudabi_filestat_t * */ *n_args = 4; break; } /* cloudabi_sys_file_stat_put */ case 28: { struct cloudabi_sys_file_stat_put_args *p = params; iarg[0] = p->fd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* const cloudabi_filestat_t * */ iarg[4] = p->flags; /* cloudabi_fsflags_t */ *n_args = 5; break; } /* cloudabi_sys_file_symlink */ case 29: { struct cloudabi_sys_file_symlink_args *p = params; uarg[0] = (intptr_t) p->path1; /* const char * */ uarg[1] = p->path1_len; /* size_t */ iarg[2] = p->fd; /* cloudabi_fd_t */ uarg[3] = (intptr_t) p->path2; /* const char * */ uarg[4] = p->path2_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_file_unlink */ case 30: { struct cloudabi_sys_file_unlink_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->flags; /* cloudabi_ulflags_t */ *n_args = 4; break; } /* cloudabi_sys_lock_unlock */ case 31: { struct cloudabi_sys_lock_unlock_args *p = params; uarg[0] = (intptr_t) p->lock; /* cloudabi_lock_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ *n_args = 2; break; } /* cloudabi_sys_mem_advise */ case 32: { struct cloudabi_sys_mem_advise_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->advice; /* cloudabi_advice_t */ *n_args = 3; break; } - /* cloudabi_sys_mem_lock */ - case 33: { - struct cloudabi_sys_mem_lock_args *p = params; - uarg[0] = (intptr_t) p->mapping; /* const void * */ - uarg[1] = p->mapping_len; /* size_t */ - *n_args = 2; - break; - } /* cloudabi_sys_mem_map */ - case 34: { + case 33: { struct cloudabi_sys_mem_map_args *p = params; uarg[0] = (intptr_t) p->addr; /* void * */ uarg[1] = p->len; /* size_t */ iarg[2] = p->prot; /* cloudabi_mprot_t */ iarg[3] = p->flags; /* cloudabi_mflags_t */ iarg[4] = p->fd; /* cloudabi_fd_t */ iarg[5] = p->off; /* cloudabi_filesize_t */ *n_args = 6; break; } /* cloudabi_sys_mem_protect */ - case 35: { + case 34: { struct cloudabi_sys_mem_protect_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->prot; /* cloudabi_mprot_t */ *n_args = 3; break; } /* cloudabi_sys_mem_sync */ - case 36: { + case 35: { struct cloudabi_sys_mem_sync_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->flags; /* cloudabi_msflags_t */ *n_args = 3; break; } - /* cloudabi_sys_mem_unlock */ - case 37: { - struct cloudabi_sys_mem_unlock_args *p = params; - uarg[0] = (intptr_t) p->mapping; /* const void * */ - uarg[1] = p->mapping_len; /* size_t */ - *n_args = 2; - break; - } /* cloudabi_sys_mem_unmap */ - case 38: { + case 36: { struct cloudabi_sys_mem_unmap_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ *n_args = 2; break; } /* cloudabi32_sys_poll */ - case 39: { + case 37: { struct cloudabi32_sys_poll_args *p = params; uarg[0] = (intptr_t) p->in; /* const cloudabi32_subscription_t * */ uarg[1] = (intptr_t) p->out; /* cloudabi32_event_t * */ uarg[2] = p->nsubscriptions; /* size_t */ *n_args = 3; break; } /* cloudabi32_sys_poll_fd */ - case 40: { + case 38: { struct cloudabi32_sys_poll_fd_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi32_subscription_t * */ uarg[2] = p->in_len; /* size_t */ uarg[3] = (intptr_t) p->out; /* cloudabi32_event_t * */ uarg[4] = p->out_len; /* size_t */ uarg[5] = (intptr_t) p->timeout; /* const cloudabi32_subscription_t * */ *n_args = 6; break; } /* cloudabi_sys_proc_exec */ - case 41: { + case 39: { struct cloudabi_sys_proc_exec_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->data; /* const void * */ uarg[2] = p->data_len; /* size_t */ uarg[3] = (intptr_t) p->fds; /* const cloudabi_fd_t * */ uarg[4] = p->fds_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_proc_exit */ - case 42: { + case 40: { struct cloudabi_sys_proc_exit_args *p = params; iarg[0] = p->rval; /* cloudabi_exitcode_t */ *n_args = 1; break; } /* cloudabi_sys_proc_fork */ - case 43: { + case 41: { *n_args = 0; break; } /* cloudabi_sys_proc_raise */ - case 44: { + case 42: { struct cloudabi_sys_proc_raise_args *p = params; iarg[0] = p->sig; /* cloudabi_signal_t */ *n_args = 1; break; } /* cloudabi_sys_random_get */ - case 45: { + case 43: { struct cloudabi_sys_random_get_args *p = params; uarg[0] = (intptr_t) p->buf; /* void * */ uarg[1] = p->buf_len; /* size_t */ *n_args = 2; break; } /* cloudabi_sys_sock_accept */ - case 46: { + case 44: { struct cloudabi_sys_sock_accept_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ - uarg[1] = (intptr_t) p->buf; /* cloudabi_sockstat_t * */ + uarg[1] = (intptr_t) p->unused; /* void * */ *n_args = 2; break; } /* cloudabi_sys_sock_bind */ - case 47: { + case 45: { struct cloudabi_sys_sock_bind_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->fd; /* cloudabi_fd_t */ uarg[2] = (intptr_t) p->path; /* const char * */ uarg[3] = p->path_len; /* size_t */ *n_args = 4; break; } /* cloudabi_sys_sock_connect */ - case 48: { + case 46: { struct cloudabi_sys_sock_connect_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->fd; /* cloudabi_fd_t */ uarg[2] = (intptr_t) p->path; /* const char * */ uarg[3] = p->path_len; /* size_t */ *n_args = 4; break; } /* cloudabi_sys_sock_listen */ - case 49: { + case 47: { struct cloudabi_sys_sock_listen_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->backlog; /* cloudabi_backlog_t */ *n_args = 2; break; } /* cloudabi32_sys_sock_recv */ - case 50: { + case 48: { struct cloudabi32_sys_sock_recv_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi32_recv_in_t * */ uarg[2] = (intptr_t) p->out; /* cloudabi32_recv_out_t * */ *n_args = 3; break; } /* cloudabi32_sys_sock_send */ - case 51: { + case 49: { struct cloudabi32_sys_sock_send_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi32_send_in_t * */ uarg[2] = (intptr_t) p->out; /* cloudabi32_send_out_t * */ *n_args = 3; break; } /* cloudabi_sys_sock_shutdown */ - case 52: { + case 50: { struct cloudabi_sys_sock_shutdown_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->how; /* cloudabi_sdflags_t */ *n_args = 2; break; } /* cloudabi_sys_sock_stat_get */ - case 53: { + case 51: { struct cloudabi_sys_sock_stat_get_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_sockstat_t * */ iarg[2] = p->flags; /* cloudabi_ssflags_t */ *n_args = 3; break; } /* cloudabi32_sys_thread_create */ - case 54: { + case 52: { struct cloudabi32_sys_thread_create_args *p = params; uarg[0] = (intptr_t) p->attr; /* cloudabi32_threadattr_t * */ *n_args = 1; break; } /* cloudabi_sys_thread_exit */ - case 55: { + case 53: { struct cloudabi_sys_thread_exit_args *p = params; uarg[0] = (intptr_t) p->lock; /* cloudabi_lock_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ *n_args = 2; break; } /* cloudabi_sys_thread_yield */ - case 56: { + case 54: { *n_args = 0; break; } default: *n_args = 0; break; }; } static void systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) { const char *p = NULL; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: switch(ndx) { case 0: p = "cloudabi_clockid_t"; break; default: break; }; break; /* cloudabi_sys_clock_time_get */ case 1: switch(ndx) { case 0: p = "cloudabi_clockid_t"; break; case 1: p = "cloudabi_timestamp_t"; break; default: break; }; break; /* cloudabi_sys_condvar_signal */ case 2: switch(ndx) { case 0: p = "userland cloudabi_condvar_t *"; break; case 1: p = "cloudabi_scope_t"; break; case 2: p = "cloudabi_nthreads_t"; break; default: break; }; break; /* cloudabi_sys_fd_close */ case 3: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_create1 */ case 4: switch(ndx) { case 0: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_fd_create2 */ case 5: switch(ndx) { case 0: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_fd_datasync */ case 6: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_dup */ case 7: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi32_sys_fd_pread */ case 8: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_iovec_t *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi32_sys_fd_pwrite */ case 9: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_ciovec_t *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi32_sys_fd_read */ case 10: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_iovec_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_fd_replace */ case 11: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_seek */ case 12: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filedelta_t"; break; case 2: p = "cloudabi_whence_t"; break; default: break; }; break; /* cloudabi_sys_fd_stat_get */ case 13: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_fdstat_t *"; break; default: break; }; break; /* cloudabi_sys_fd_stat_put */ case 14: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi_fdstat_t *"; break; case 2: p = "cloudabi_fdsflags_t"; break; default: break; }; break; /* cloudabi_sys_fd_sync */ case 15: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi32_sys_fd_write */ case 16: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_ciovec_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_advise */ case 17: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filesize_t"; break; case 2: p = "cloudabi_filesize_t"; break; case 3: p = "cloudabi_advice_t"; break; default: break; }; break; /* cloudabi_sys_file_allocate */ case 18: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filesize_t"; break; case 2: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi_sys_file_create */ case 19: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_file_link */ case 20: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_fd_t"; break; case 4: p = "userland const char *"; break; case 5: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_open */ case 21: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_oflags_t"; break; case 4: p = "userland const cloudabi_fdstat_t *"; break; default: break; }; break; /* cloudabi_sys_file_readdir */ case 22: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland void *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_dircookie_t"; break; default: break; }; break; /* cloudabi_sys_file_readlink */ case 23: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland char *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_rename */ case 24: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_fd_t"; break; case 4: p = "userland const char *"; break; case 5: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_stat_fget */ case 25: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_filestat_t *"; break; default: break; }; break; /* cloudabi_sys_file_stat_fput */ case 26: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi_filestat_t *"; break; case 2: p = "cloudabi_fsflags_t"; break; default: break; }; break; /* cloudabi_sys_file_stat_get */ case 27: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland cloudabi_filestat_t *"; break; default: break; }; break; /* cloudabi_sys_file_stat_put */ case 28: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland const cloudabi_filestat_t *"; break; case 4: p = "cloudabi_fsflags_t"; break; default: break; }; break; /* cloudabi_sys_file_symlink */ case 29: switch(ndx) { case 0: p = "userland const char *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_fd_t"; break; case 3: p = "userland const char *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_unlink */ case 30: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_ulflags_t"; break; default: break; }; break; /* cloudabi_sys_lock_unlock */ case 31: switch(ndx) { case 0: p = "userland cloudabi_lock_t *"; break; case 1: p = "cloudabi_scope_t"; break; default: break; }; break; /* cloudabi_sys_mem_advise */ case 32: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_advice_t"; break; default: break; }; break; - /* cloudabi_sys_mem_lock */ + /* cloudabi_sys_mem_map */ case 33: switch(ndx) { case 0: - p = "userland const void *"; - break; - case 1: - p = "size_t"; - break; - default: - break; - }; - break; - /* cloudabi_sys_mem_map */ - case 34: - switch(ndx) { - case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_mprot_t"; break; case 3: p = "cloudabi_mflags_t"; break; case 4: p = "cloudabi_fd_t"; break; case 5: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi_sys_mem_protect */ - case 35: + case 34: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_mprot_t"; break; default: break; }; break; /* cloudabi_sys_mem_sync */ - case 36: + case 35: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_msflags_t"; break; default: break; }; break; - /* cloudabi_sys_mem_unlock */ - case 37: - switch(ndx) { - case 0: - p = "userland const void *"; - break; - case 1: - p = "size_t"; - break; - default: - break; - }; - break; /* cloudabi_sys_mem_unmap */ - case 38: + case 36: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; default: break; }; break; /* cloudabi32_sys_poll */ - case 39: + case 37: switch(ndx) { case 0: p = "userland const cloudabi32_subscription_t *"; break; case 1: p = "userland cloudabi32_event_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi32_sys_poll_fd */ - case 40: + case 38: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_subscription_t *"; break; case 2: p = "size_t"; break; case 3: p = "userland cloudabi32_event_t *"; break; case 4: p = "size_t"; break; case 5: p = "userland const cloudabi32_subscription_t *"; break; default: break; }; break; /* cloudabi_sys_proc_exec */ - case 41: + case 39: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const void *"; break; case 2: p = "size_t"; break; case 3: p = "userland const cloudabi_fd_t *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_proc_exit */ - case 42: + case 40: switch(ndx) { case 0: p = "cloudabi_exitcode_t"; break; default: break; }; break; /* cloudabi_sys_proc_fork */ - case 43: + case 41: break; /* cloudabi_sys_proc_raise */ - case 44: + case 42: switch(ndx) { case 0: p = "cloudabi_signal_t"; break; default: break; }; break; /* cloudabi_sys_random_get */ - case 45: + case 43: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_accept */ - case 46: + case 44: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: - p = "userland cloudabi_sockstat_t *"; + p = "userland void *"; break; default: break; }; break; /* cloudabi_sys_sock_bind */ - case 47: + case 45: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; case 2: p = "userland const char *"; break; case 3: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_connect */ - case 48: + case 46: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; case 2: p = "userland const char *"; break; case 3: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_listen */ - case 49: + case 47: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_backlog_t"; break; default: break; }; break; /* cloudabi32_sys_sock_recv */ - case 50: + case 48: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_recv_in_t *"; break; case 2: p = "userland cloudabi32_recv_out_t *"; break; default: break; }; break; /* cloudabi32_sys_sock_send */ - case 51: + case 49: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi32_send_in_t *"; break; case 2: p = "userland cloudabi32_send_out_t *"; break; default: break; }; break; /* cloudabi_sys_sock_shutdown */ - case 52: + case 50: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_sdflags_t"; break; default: break; }; break; /* cloudabi_sys_sock_stat_get */ - case 53: + case 51: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_sockstat_t *"; break; case 2: p = "cloudabi_ssflags_t"; break; default: break; }; break; /* cloudabi32_sys_thread_create */ - case 54: + case 52: switch(ndx) { case 0: p = "userland cloudabi32_threadattr_t *"; break; default: break; }; break; /* cloudabi_sys_thread_exit */ - case 55: + case 53: switch(ndx) { case 0: p = "userland cloudabi_lock_t *"; break; case 1: p = "cloudabi_scope_t"; break; default: break; }; break; /* cloudabi_sys_thread_yield */ - case 56: + case 54: break; default: break; }; if (p != NULL) strlcpy(desc, p, descsz); } static void systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) { const char *p = NULL; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: if (ndx == 0 || ndx == 1) p = "cloudabi_timestamp_t"; break; /* cloudabi_sys_clock_time_get */ case 1: if (ndx == 0 || ndx == 1) p = "cloudabi_timestamp_t"; break; /* cloudabi_sys_condvar_signal */ case 2: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_close */ case 3: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_create1 */ case 4: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_fd_create2 */ case 5: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_datasync */ case 6: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_dup */ case 7: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi32_sys_fd_pread */ case 8: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi32_sys_fd_pwrite */ case 9: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi32_sys_fd_read */ case 10: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_fd_replace */ case 11: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_seek */ case 12: if (ndx == 0 || ndx == 1) p = "cloudabi_filesize_t"; break; /* cloudabi_sys_fd_stat_get */ case 13: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_stat_put */ case 14: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_sync */ case 15: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi32_sys_fd_write */ case 16: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_advise */ case 17: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_allocate */ case 18: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_create */ case 19: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_link */ case 20: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_open */ case 21: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_file_readdir */ case 22: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_readlink */ case 23: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_rename */ case 24: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_fget */ case 25: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_fput */ case 26: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_get */ case 27: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_put */ case 28: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_symlink */ case 29: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_unlink */ case 30: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_lock_unlock */ case 31: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_mem_advise */ case 32: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_lock */ + /* cloudabi_sys_mem_map */ case 33: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_map */ + /* cloudabi_sys_mem_protect */ case 34: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_protect */ + /* cloudabi_sys_mem_sync */ case 35: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_sync */ + /* cloudabi_sys_mem_unmap */ case 36: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_unlock */ + /* cloudabi32_sys_poll */ case 37: if (ndx == 0 || ndx == 1) - p = "void"; - break; - /* cloudabi_sys_mem_unmap */ - case 38: - if (ndx == 0 || ndx == 1) - p = "void"; - break; - /* cloudabi32_sys_poll */ - case 39: - if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi32_sys_poll_fd */ - case 40: + case 38: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_proc_exec */ - case 41: + case 39: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_proc_exit */ - case 42: + case 40: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_proc_fork */ - case 43: + case 41: /* cloudabi_sys_proc_raise */ - case 44: + case 42: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_random_get */ - case 45: + case 43: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_accept */ - case 46: + case 44: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_sock_bind */ - case 47: + case 45: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_connect */ - case 48: + case 46: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_listen */ - case 49: + case 47: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi32_sys_sock_recv */ - case 50: + case 48: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi32_sys_sock_send */ - case 51: + case 49: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_shutdown */ - case 52: + case 50: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_stat_get */ - case 53: + case 51: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi32_sys_thread_create */ - case 54: + case 52: if (ndx == 0 || ndx == 1) p = "cloudabi_tid_t"; break; /* cloudabi_sys_thread_exit */ - case 55: + case 53: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_thread_yield */ - case 56: + case 54: default: break; }; if (p != NULL) strlcpy(desc, p, descsz); } Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_proto.h =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_proto.h (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_proto.h (revision 321585) @@ -1,464 +1,458 @@ /* * System call prototypes. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #ifndef _CLOUDABI64_SYSPROTO_H_ #define _CLOUDABI64_SYSPROTO_H_ #include #include #include #include #include #include #include #include struct proc; struct thread; #define PAD_(t) (sizeof(register_t) <= sizeof(t) ? \ 0 : sizeof(register_t) - sizeof(t)) #if BYTE_ORDER == LITTLE_ENDIAN #define PADL_(t) 0 #define PADR_(t) PAD_(t) #else #define PADL_(t) PAD_(t) #define PADR_(t) 0 #endif struct cloudabi_sys_clock_res_get_args { char clock_id_l_[PADL_(cloudabi_clockid_t)]; cloudabi_clockid_t clock_id; char clock_id_r_[PADR_(cloudabi_clockid_t)]; }; struct cloudabi_sys_clock_time_get_args { char clock_id_l_[PADL_(cloudabi_clockid_t)]; cloudabi_clockid_t clock_id; char clock_id_r_[PADR_(cloudabi_clockid_t)]; char precision_l_[PADL_(cloudabi_timestamp_t)]; cloudabi_timestamp_t precision; char precision_r_[PADR_(cloudabi_timestamp_t)]; }; struct cloudabi_sys_condvar_signal_args { char condvar_l_[PADL_(cloudabi_condvar_t *)]; cloudabi_condvar_t * condvar; char condvar_r_[PADR_(cloudabi_condvar_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; char nwaiters_l_[PADL_(cloudabi_nthreads_t)]; cloudabi_nthreads_t nwaiters; char nwaiters_r_[PADR_(cloudabi_nthreads_t)]; }; struct cloudabi_sys_fd_close_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_create1_args { char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_fd_create2_args { char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_fd_datasync_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_dup_args { char from_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t from; char from_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi64_sys_fd_pread_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi64_iovec_t *)]; const cloudabi64_iovec_t * iovs; char iovs_r_[PADR_(const cloudabi64_iovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi64_sys_fd_pwrite_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi64_ciovec_t *)]; const cloudabi64_ciovec_t * iovs; char iovs_r_[PADR_(const cloudabi64_ciovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi64_sys_fd_read_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi64_iovec_t *)]; const cloudabi64_iovec_t * iovs; char iovs_r_[PADR_(const cloudabi64_iovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_fd_replace_args { char from_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t from; char from_r_[PADR_(cloudabi_fd_t)]; char to_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t to; char to_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi_sys_fd_seek_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filedelta_t)]; cloudabi_filedelta_t offset; char offset_r_[PADR_(cloudabi_filedelta_t)]; char whence_l_[PADL_(cloudabi_whence_t)]; cloudabi_whence_t whence; char whence_r_[PADR_(cloudabi_whence_t)]; }; struct cloudabi_sys_fd_stat_get_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_fdstat_t *)]; cloudabi_fdstat_t * buf; char buf_r_[PADR_(cloudabi_fdstat_t *)]; }; struct cloudabi_sys_fd_stat_put_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(const cloudabi_fdstat_t *)]; const cloudabi_fdstat_t * buf; char buf_r_[PADR_(const cloudabi_fdstat_t *)]; char flags_l_[PADL_(cloudabi_fdsflags_t)]; cloudabi_fdsflags_t flags; char flags_r_[PADR_(cloudabi_fdsflags_t)]; }; struct cloudabi_sys_fd_sync_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; }; struct cloudabi64_sys_fd_write_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char iovs_l_[PADL_(const cloudabi64_ciovec_t *)]; const cloudabi64_ciovec_t * iovs; char iovs_r_[PADR_(const cloudabi64_ciovec_t *)]; char iovs_len_l_[PADL_(size_t)]; size_t iovs_len; char iovs_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_advise_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; char len_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t len; char len_r_[PADR_(cloudabi_filesize_t)]; char advice_l_[PADL_(cloudabi_advice_t)]; cloudabi_advice_t advice; char advice_r_[PADR_(cloudabi_advice_t)]; }; struct cloudabi_sys_file_allocate_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char offset_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t offset; char offset_r_[PADR_(cloudabi_filesize_t)]; char len_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t len; char len_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi_sys_file_create_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char type_l_[PADL_(cloudabi_filetype_t)]; cloudabi_filetype_t type; char type_r_[PADR_(cloudabi_filetype_t)]; }; struct cloudabi_sys_file_link_args { char fd1_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd1; char fd1_r_[PADR_(cloudabi_lookup_t)]; char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd2_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd2; char fd2_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_open_args { char dirfd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t dirfd; char dirfd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char oflags_l_[PADL_(cloudabi_oflags_t)]; cloudabi_oflags_t oflags; char oflags_r_[PADR_(cloudabi_oflags_t)]; char fds_l_[PADL_(const cloudabi_fdstat_t *)]; const cloudabi_fdstat_t * fds; char fds_r_[PADR_(const cloudabi_fdstat_t *)]; }; struct cloudabi_sys_file_readdir_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(void *)]; void * buf; char buf_r_[PADR_(void *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; char cookie_l_[PADL_(cloudabi_dircookie_t)]; cloudabi_dircookie_t cookie; char cookie_r_[PADR_(cloudabi_dircookie_t)]; }; struct cloudabi_sys_file_readlink_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(char *)]; char * buf; char buf_r_[PADR_(char *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_rename_args { char fd1_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd1; char fd1_r_[PADR_(cloudabi_fd_t)]; char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd2_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd2; char fd2_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_stat_fget_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_filestat_t *)]; cloudabi_filestat_t * buf; char buf_r_[PADR_(cloudabi_filestat_t *)]; }; struct cloudabi_sys_file_stat_fput_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(const cloudabi_filestat_t *)]; const cloudabi_filestat_t * buf; char buf_r_[PADR_(const cloudabi_filestat_t *)]; char flags_l_[PADL_(cloudabi_fsflags_t)]; cloudabi_fsflags_t flags; char flags_r_[PADR_(cloudabi_fsflags_t)]; }; struct cloudabi_sys_file_stat_get_args { char fd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd; char fd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(cloudabi_filestat_t *)]; cloudabi_filestat_t * buf; char buf_r_[PADR_(cloudabi_filestat_t *)]; }; struct cloudabi_sys_file_stat_put_args { char fd_l_[PADL_(cloudabi_lookup_t)]; cloudabi_lookup_t fd; char fd_r_[PADR_(cloudabi_lookup_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char buf_l_[PADL_(const cloudabi_filestat_t *)]; const cloudabi_filestat_t * buf; char buf_r_[PADR_(const cloudabi_filestat_t *)]; char flags_l_[PADL_(cloudabi_fsflags_t)]; cloudabi_fsflags_t flags; char flags_r_[PADR_(cloudabi_fsflags_t)]; }; struct cloudabi_sys_file_symlink_args { char path1_l_[PADL_(const char *)]; const char * path1; char path1_r_[PADR_(const char *)]; char path1_len_l_[PADL_(size_t)]; size_t path1_len; char path1_len_r_[PADR_(size_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path2_l_[PADL_(const char *)]; const char * path2; char path2_r_[PADR_(const char *)]; char path2_len_l_[PADL_(size_t)]; size_t path2_len; char path2_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_file_unlink_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; char flags_l_[PADL_(cloudabi_ulflags_t)]; cloudabi_ulflags_t flags; char flags_r_[PADR_(cloudabi_ulflags_t)]; }; struct cloudabi_sys_lock_unlock_args { char lock_l_[PADL_(cloudabi_lock_t *)]; cloudabi_lock_t * lock; char lock_r_[PADR_(cloudabi_lock_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; }; struct cloudabi_sys_mem_advise_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char advice_l_[PADL_(cloudabi_advice_t)]; cloudabi_advice_t advice; char advice_r_[PADR_(cloudabi_advice_t)]; }; -struct cloudabi_sys_mem_lock_args { - char mapping_l_[PADL_(const void *)]; const void * mapping; char mapping_r_[PADR_(const void *)]; - char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; -}; struct cloudabi_sys_mem_map_args { char addr_l_[PADL_(void *)]; void * addr; char addr_r_[PADR_(void *)]; char len_l_[PADL_(size_t)]; size_t len; char len_r_[PADR_(size_t)]; char prot_l_[PADL_(cloudabi_mprot_t)]; cloudabi_mprot_t prot; char prot_r_[PADR_(cloudabi_mprot_t)]; char flags_l_[PADL_(cloudabi_mflags_t)]; cloudabi_mflags_t flags; char flags_r_[PADR_(cloudabi_mflags_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char off_l_[PADL_(cloudabi_filesize_t)]; cloudabi_filesize_t off; char off_r_[PADR_(cloudabi_filesize_t)]; }; struct cloudabi_sys_mem_protect_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char prot_l_[PADL_(cloudabi_mprot_t)]; cloudabi_mprot_t prot; char prot_r_[PADR_(cloudabi_mprot_t)]; }; struct cloudabi_sys_mem_sync_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; char flags_l_[PADL_(cloudabi_msflags_t)]; cloudabi_msflags_t flags; char flags_r_[PADR_(cloudabi_msflags_t)]; }; -struct cloudabi_sys_mem_unlock_args { - char mapping_l_[PADL_(const void *)]; const void * mapping; char mapping_r_[PADR_(const void *)]; - char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; -}; struct cloudabi_sys_mem_unmap_args { char mapping_l_[PADL_(void *)]; void * mapping; char mapping_r_[PADR_(void *)]; char mapping_len_l_[PADL_(size_t)]; size_t mapping_len; char mapping_len_r_[PADR_(size_t)]; }; struct cloudabi64_sys_poll_args { char in_l_[PADL_(const cloudabi64_subscription_t *)]; const cloudabi64_subscription_t * in; char in_r_[PADR_(const cloudabi64_subscription_t *)]; char out_l_[PADL_(cloudabi64_event_t *)]; cloudabi64_event_t * out; char out_r_[PADR_(cloudabi64_event_t *)]; char nsubscriptions_l_[PADL_(size_t)]; size_t nsubscriptions; char nsubscriptions_r_[PADR_(size_t)]; }; struct cloudabi64_sys_poll_fd_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi64_subscription_t *)]; const cloudabi64_subscription_t * in; char in_r_[PADR_(const cloudabi64_subscription_t *)]; char in_len_l_[PADL_(size_t)]; size_t in_len; char in_len_r_[PADR_(size_t)]; char out_l_[PADL_(cloudabi64_event_t *)]; cloudabi64_event_t * out; char out_r_[PADR_(cloudabi64_event_t *)]; char out_len_l_[PADL_(size_t)]; size_t out_len; char out_len_r_[PADR_(size_t)]; char timeout_l_[PADL_(const cloudabi64_subscription_t *)]; const cloudabi64_subscription_t * timeout; char timeout_r_[PADR_(const cloudabi64_subscription_t *)]; }; struct cloudabi_sys_proc_exec_args { char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char data_l_[PADL_(const void *)]; const void * data; char data_r_[PADR_(const void *)]; char data_len_l_[PADL_(size_t)]; size_t data_len; char data_len_r_[PADR_(size_t)]; char fds_l_[PADL_(const cloudabi_fd_t *)]; const cloudabi_fd_t * fds; char fds_r_[PADR_(const cloudabi_fd_t *)]; char fds_len_l_[PADL_(size_t)]; size_t fds_len; char fds_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_proc_exit_args { char rval_l_[PADL_(cloudabi_exitcode_t)]; cloudabi_exitcode_t rval; char rval_r_[PADR_(cloudabi_exitcode_t)]; }; struct cloudabi_sys_proc_fork_args { register_t dummy; }; struct cloudabi_sys_proc_raise_args { char sig_l_[PADL_(cloudabi_signal_t)]; cloudabi_signal_t sig; char sig_r_[PADR_(cloudabi_signal_t)]; }; struct cloudabi_sys_random_get_args { char buf_l_[PADL_(void *)]; void * buf; char buf_r_[PADR_(void *)]; char buf_len_l_[PADL_(size_t)]; size_t buf_len; char buf_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_accept_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; - char buf_l_[PADL_(cloudabi_sockstat_t *)]; cloudabi_sockstat_t * buf; char buf_r_[PADR_(cloudabi_sockstat_t *)]; + char unused_l_[PADL_(void *)]; void * unused; char unused_r_[PADR_(void *)]; }; struct cloudabi_sys_sock_bind_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_connect_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char fd_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t fd; char fd_r_[PADR_(cloudabi_fd_t)]; char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char path_len_l_[PADL_(size_t)]; size_t path_len; char path_len_r_[PADR_(size_t)]; }; struct cloudabi_sys_sock_listen_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char backlog_l_[PADL_(cloudabi_backlog_t)]; cloudabi_backlog_t backlog; char backlog_r_[PADR_(cloudabi_backlog_t)]; }; struct cloudabi64_sys_sock_recv_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi64_recv_in_t *)]; const cloudabi64_recv_in_t * in; char in_r_[PADR_(const cloudabi64_recv_in_t *)]; char out_l_[PADL_(cloudabi64_recv_out_t *)]; cloudabi64_recv_out_t * out; char out_r_[PADR_(cloudabi64_recv_out_t *)]; }; struct cloudabi64_sys_sock_send_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char in_l_[PADL_(const cloudabi64_send_in_t *)]; const cloudabi64_send_in_t * in; char in_r_[PADR_(const cloudabi64_send_in_t *)]; char out_l_[PADL_(cloudabi64_send_out_t *)]; cloudabi64_send_out_t * out; char out_r_[PADR_(cloudabi64_send_out_t *)]; }; struct cloudabi_sys_sock_shutdown_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char how_l_[PADL_(cloudabi_sdflags_t)]; cloudabi_sdflags_t how; char how_r_[PADR_(cloudabi_sdflags_t)]; }; struct cloudabi_sys_sock_stat_get_args { char sock_l_[PADL_(cloudabi_fd_t)]; cloudabi_fd_t sock; char sock_r_[PADR_(cloudabi_fd_t)]; char buf_l_[PADL_(cloudabi_sockstat_t *)]; cloudabi_sockstat_t * buf; char buf_r_[PADR_(cloudabi_sockstat_t *)]; char flags_l_[PADL_(cloudabi_ssflags_t)]; cloudabi_ssflags_t flags; char flags_r_[PADR_(cloudabi_ssflags_t)]; }; struct cloudabi64_sys_thread_create_args { char attr_l_[PADL_(cloudabi64_threadattr_t *)]; cloudabi64_threadattr_t * attr; char attr_r_[PADR_(cloudabi64_threadattr_t *)]; }; struct cloudabi_sys_thread_exit_args { char lock_l_[PADL_(cloudabi_lock_t *)]; cloudabi_lock_t * lock; char lock_r_[PADR_(cloudabi_lock_t *)]; char scope_l_[PADL_(cloudabi_scope_t)]; cloudabi_scope_t scope; char scope_r_[PADR_(cloudabi_scope_t)]; }; struct cloudabi_sys_thread_yield_args { register_t dummy; }; int cloudabi_sys_clock_res_get(struct thread *, struct cloudabi_sys_clock_res_get_args *); int cloudabi_sys_clock_time_get(struct thread *, struct cloudabi_sys_clock_time_get_args *); int cloudabi_sys_condvar_signal(struct thread *, struct cloudabi_sys_condvar_signal_args *); int cloudabi_sys_fd_close(struct thread *, struct cloudabi_sys_fd_close_args *); int cloudabi_sys_fd_create1(struct thread *, struct cloudabi_sys_fd_create1_args *); int cloudabi_sys_fd_create2(struct thread *, struct cloudabi_sys_fd_create2_args *); int cloudabi_sys_fd_datasync(struct thread *, struct cloudabi_sys_fd_datasync_args *); int cloudabi_sys_fd_dup(struct thread *, struct cloudabi_sys_fd_dup_args *); int cloudabi64_sys_fd_pread(struct thread *, struct cloudabi64_sys_fd_pread_args *); int cloudabi64_sys_fd_pwrite(struct thread *, struct cloudabi64_sys_fd_pwrite_args *); int cloudabi64_sys_fd_read(struct thread *, struct cloudabi64_sys_fd_read_args *); int cloudabi_sys_fd_replace(struct thread *, struct cloudabi_sys_fd_replace_args *); int cloudabi_sys_fd_seek(struct thread *, struct cloudabi_sys_fd_seek_args *); int cloudabi_sys_fd_stat_get(struct thread *, struct cloudabi_sys_fd_stat_get_args *); int cloudabi_sys_fd_stat_put(struct thread *, struct cloudabi_sys_fd_stat_put_args *); int cloudabi_sys_fd_sync(struct thread *, struct cloudabi_sys_fd_sync_args *); int cloudabi64_sys_fd_write(struct thread *, struct cloudabi64_sys_fd_write_args *); int cloudabi_sys_file_advise(struct thread *, struct cloudabi_sys_file_advise_args *); int cloudabi_sys_file_allocate(struct thread *, struct cloudabi_sys_file_allocate_args *); int cloudabi_sys_file_create(struct thread *, struct cloudabi_sys_file_create_args *); int cloudabi_sys_file_link(struct thread *, struct cloudabi_sys_file_link_args *); int cloudabi_sys_file_open(struct thread *, struct cloudabi_sys_file_open_args *); int cloudabi_sys_file_readdir(struct thread *, struct cloudabi_sys_file_readdir_args *); int cloudabi_sys_file_readlink(struct thread *, struct cloudabi_sys_file_readlink_args *); int cloudabi_sys_file_rename(struct thread *, struct cloudabi_sys_file_rename_args *); int cloudabi_sys_file_stat_fget(struct thread *, struct cloudabi_sys_file_stat_fget_args *); int cloudabi_sys_file_stat_fput(struct thread *, struct cloudabi_sys_file_stat_fput_args *); int cloudabi_sys_file_stat_get(struct thread *, struct cloudabi_sys_file_stat_get_args *); int cloudabi_sys_file_stat_put(struct thread *, struct cloudabi_sys_file_stat_put_args *); int cloudabi_sys_file_symlink(struct thread *, struct cloudabi_sys_file_symlink_args *); int cloudabi_sys_file_unlink(struct thread *, struct cloudabi_sys_file_unlink_args *); int cloudabi_sys_lock_unlock(struct thread *, struct cloudabi_sys_lock_unlock_args *); int cloudabi_sys_mem_advise(struct thread *, struct cloudabi_sys_mem_advise_args *); -int cloudabi_sys_mem_lock(struct thread *, struct cloudabi_sys_mem_lock_args *); int cloudabi_sys_mem_map(struct thread *, struct cloudabi_sys_mem_map_args *); int cloudabi_sys_mem_protect(struct thread *, struct cloudabi_sys_mem_protect_args *); int cloudabi_sys_mem_sync(struct thread *, struct cloudabi_sys_mem_sync_args *); -int cloudabi_sys_mem_unlock(struct thread *, struct cloudabi_sys_mem_unlock_args *); int cloudabi_sys_mem_unmap(struct thread *, struct cloudabi_sys_mem_unmap_args *); int cloudabi64_sys_poll(struct thread *, struct cloudabi64_sys_poll_args *); int cloudabi64_sys_poll_fd(struct thread *, struct cloudabi64_sys_poll_fd_args *); int cloudabi_sys_proc_exec(struct thread *, struct cloudabi_sys_proc_exec_args *); int cloudabi_sys_proc_exit(struct thread *, struct cloudabi_sys_proc_exit_args *); int cloudabi_sys_proc_fork(struct thread *, struct cloudabi_sys_proc_fork_args *); int cloudabi_sys_proc_raise(struct thread *, struct cloudabi_sys_proc_raise_args *); int cloudabi_sys_random_get(struct thread *, struct cloudabi_sys_random_get_args *); int cloudabi_sys_sock_accept(struct thread *, struct cloudabi_sys_sock_accept_args *); int cloudabi_sys_sock_bind(struct thread *, struct cloudabi_sys_sock_bind_args *); int cloudabi_sys_sock_connect(struct thread *, struct cloudabi_sys_sock_connect_args *); int cloudabi_sys_sock_listen(struct thread *, struct cloudabi_sys_sock_listen_args *); int cloudabi64_sys_sock_recv(struct thread *, struct cloudabi64_sys_sock_recv_args *); int cloudabi64_sys_sock_send(struct thread *, struct cloudabi64_sys_sock_send_args *); int cloudabi_sys_sock_shutdown(struct thread *, struct cloudabi_sys_sock_shutdown_args *); int cloudabi_sys_sock_stat_get(struct thread *, struct cloudabi_sys_sock_stat_get_args *); int cloudabi64_sys_thread_create(struct thread *, struct cloudabi64_sys_thread_create_args *); int cloudabi_sys_thread_exit(struct thread *, struct cloudabi_sys_thread_exit_args *); int cloudabi_sys_thread_yield(struct thread *, struct cloudabi_sys_thread_yield_args *); #ifdef COMPAT_43 #endif /* COMPAT_43 */ #ifdef COMPAT_FREEBSD4 #endif /* COMPAT_FREEBSD4 */ #ifdef COMPAT_FREEBSD6 #endif /* COMPAT_FREEBSD6 */ #ifdef COMPAT_FREEBSD7 #endif /* COMPAT_FREEBSD7 */ #ifdef COMPAT_FREEBSD10 #endif /* COMPAT_FREEBSD10 */ + +#ifdef COMPAT_FREEBSD11 + + +#endif /* COMPAT_FREEBSD11 */ + #define CLOUDABI64_SYS_AUE_cloudabi_sys_clock_res_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_clock_time_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_condvar_signal AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_close AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_create1 AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_create2 AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_datasync AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_dup AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_fd_pread AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_fd_pwrite AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_fd_read AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_replace AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_seek AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_stat_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_stat_put AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_fd_sync AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_fd_write AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_advise AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_allocate AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_create AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_link AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_open AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_readdir AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_readlink AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_rename AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_stat_fget AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_stat_fput AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_stat_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_stat_put AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_symlink AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_file_unlink AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_lock_unlock AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_advise AUE_NULL -#define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_lock AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_map AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_protect AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_sync AUE_NULL -#define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_unlock AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_mem_unmap AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_poll AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_poll_fd AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_proc_exec AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_proc_exit AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_proc_fork AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_proc_raise AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_random_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_accept AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_bind AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_connect AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_listen AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_sock_recv AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_sock_send AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_shutdown AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_sock_stat_get AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi64_sys_thread_create AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_thread_exit AUE_NULL #define CLOUDABI64_SYS_AUE_cloudabi_sys_thread_yield AUE_NULL #undef PAD_ #undef PADL_ #undef PADR_ #endif /* !_CLOUDABI64_SYSPROTO_H_ */ Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sock.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sock.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sock.c (revision 321585) @@ -1,128 +1,128 @@ /*- * Copyright (c) 2015-2017 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_SOCKET, "socket", "CloudABI socket"); int cloudabi64_sys_sock_recv(struct thread *td, struct cloudabi64_sys_sock_recv_args *uap) { cloudabi64_recv_in_t ri; cloudabi64_recv_out_t ro = {}; cloudabi64_iovec_t iovobj; struct iovec *iov; const cloudabi64_iovec_t *user_iov; size_t i, rdatalen, rfdslen; int error; error = copyin(uap->in, &ri, sizeof(ri)); if (error != 0) return (error); /* Convert iovecs to native format. */ if (ri.ri_data_len > UIO_MAXIOV) return (EINVAL); iov = malloc(ri.ri_data_len * sizeof(struct iovec), M_SOCKET, M_WAITOK); user_iov = TO_PTR(ri.ri_data); for (i = 0; i < ri.ri_data_len; i++) { error = copyin(&user_iov[i], &iovobj, sizeof(iovobj)); if (error != 0) { free(iov, M_SOCKET); return (error); } iov[i].iov_base = TO_PTR(iovobj.buf); iov[i].iov_len = iovobj.buf_len; } error = cloudabi_sock_recv(td, uap->sock, iov, ri.ri_data_len, TO_PTR(ri.ri_fds), ri.ri_fds_len, ri.ri_flags, &rdatalen, - &rfdslen, &ro.ro_peername, &ro.ro_flags); + &rfdslen, &ro.ro_flags); free(iov, M_SOCKET); if (error != 0) return (error); ro.ro_datalen = rdatalen; ro.ro_fdslen = rfdslen; return (copyout(&ro, uap->out, sizeof(ro))); } int cloudabi64_sys_sock_send(struct thread *td, struct cloudabi64_sys_sock_send_args *uap) { cloudabi64_send_in_t si; cloudabi64_send_out_t so = {}; cloudabi64_ciovec_t iovobj; struct iovec *iov; const cloudabi64_ciovec_t *user_iov; size_t datalen, i; int error; error = copyin(uap->in, &si, sizeof(si)); if (error != 0) return (error); /* Convert iovecs to native format. */ if (si.si_data_len > UIO_MAXIOV) return (EINVAL); iov = malloc(si.si_data_len * sizeof(struct iovec), M_SOCKET, M_WAITOK); user_iov = TO_PTR(si.si_data); for (i = 0; i < si.si_data_len; i++) { error = copyin(&user_iov[i], &iovobj, sizeof(iovobj)); if (error != 0) { free(iov, M_SOCKET); return (error); } iov[i].iov_base = TO_PTR(iovobj.buf); iov[i].iov_len = iovobj.buf_len; } error = cloudabi_sock_send(td, uap->sock, iov, si.si_data_len, - TO_PTR(si.si_fds), si.si_fds_len, si.si_flags, &datalen); + TO_PTR(si.si_fds), si.si_fds_len, &datalen); free(iov, M_SOCKET); if (error != 0) return (error); so.so_datalen = datalen; return (copyout(&so, uap->out, sizeof(so))); } Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscall.h =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscall.h (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscall.h (revision 321585) @@ -1,65 +1,63 @@ /* * System call numbers. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #define CLOUDABI64_SYS_cloudabi_sys_clock_res_get 0 #define CLOUDABI64_SYS_cloudabi_sys_clock_time_get 1 #define CLOUDABI64_SYS_cloudabi_sys_condvar_signal 2 #define CLOUDABI64_SYS_cloudabi_sys_fd_close 3 #define CLOUDABI64_SYS_cloudabi_sys_fd_create1 4 #define CLOUDABI64_SYS_cloudabi_sys_fd_create2 5 #define CLOUDABI64_SYS_cloudabi_sys_fd_datasync 6 #define CLOUDABI64_SYS_cloudabi_sys_fd_dup 7 #define CLOUDABI64_SYS_cloudabi64_sys_fd_pread 8 #define CLOUDABI64_SYS_cloudabi64_sys_fd_pwrite 9 #define CLOUDABI64_SYS_cloudabi64_sys_fd_read 10 #define CLOUDABI64_SYS_cloudabi_sys_fd_replace 11 #define CLOUDABI64_SYS_cloudabi_sys_fd_seek 12 #define CLOUDABI64_SYS_cloudabi_sys_fd_stat_get 13 #define CLOUDABI64_SYS_cloudabi_sys_fd_stat_put 14 #define CLOUDABI64_SYS_cloudabi_sys_fd_sync 15 #define CLOUDABI64_SYS_cloudabi64_sys_fd_write 16 #define CLOUDABI64_SYS_cloudabi_sys_file_advise 17 #define CLOUDABI64_SYS_cloudabi_sys_file_allocate 18 #define CLOUDABI64_SYS_cloudabi_sys_file_create 19 #define CLOUDABI64_SYS_cloudabi_sys_file_link 20 #define CLOUDABI64_SYS_cloudabi_sys_file_open 21 #define CLOUDABI64_SYS_cloudabi_sys_file_readdir 22 #define CLOUDABI64_SYS_cloudabi_sys_file_readlink 23 #define CLOUDABI64_SYS_cloudabi_sys_file_rename 24 #define CLOUDABI64_SYS_cloudabi_sys_file_stat_fget 25 #define CLOUDABI64_SYS_cloudabi_sys_file_stat_fput 26 #define CLOUDABI64_SYS_cloudabi_sys_file_stat_get 27 #define CLOUDABI64_SYS_cloudabi_sys_file_stat_put 28 #define CLOUDABI64_SYS_cloudabi_sys_file_symlink 29 #define CLOUDABI64_SYS_cloudabi_sys_file_unlink 30 #define CLOUDABI64_SYS_cloudabi_sys_lock_unlock 31 #define CLOUDABI64_SYS_cloudabi_sys_mem_advise 32 -#define CLOUDABI64_SYS_cloudabi_sys_mem_lock 33 -#define CLOUDABI64_SYS_cloudabi_sys_mem_map 34 -#define CLOUDABI64_SYS_cloudabi_sys_mem_protect 35 -#define CLOUDABI64_SYS_cloudabi_sys_mem_sync 36 -#define CLOUDABI64_SYS_cloudabi_sys_mem_unlock 37 -#define CLOUDABI64_SYS_cloudabi_sys_mem_unmap 38 -#define CLOUDABI64_SYS_cloudabi64_sys_poll 39 -#define CLOUDABI64_SYS_cloudabi64_sys_poll_fd 40 -#define CLOUDABI64_SYS_cloudabi_sys_proc_exec 41 -#define CLOUDABI64_SYS_cloudabi_sys_proc_exit 42 -#define CLOUDABI64_SYS_cloudabi_sys_proc_fork 43 -#define CLOUDABI64_SYS_cloudabi_sys_proc_raise 44 -#define CLOUDABI64_SYS_cloudabi_sys_random_get 45 -#define CLOUDABI64_SYS_cloudabi_sys_sock_accept 46 -#define CLOUDABI64_SYS_cloudabi_sys_sock_bind 47 -#define CLOUDABI64_SYS_cloudabi_sys_sock_connect 48 -#define CLOUDABI64_SYS_cloudabi_sys_sock_listen 49 -#define CLOUDABI64_SYS_cloudabi64_sys_sock_recv 50 -#define CLOUDABI64_SYS_cloudabi64_sys_sock_send 51 -#define CLOUDABI64_SYS_cloudabi_sys_sock_shutdown 52 -#define CLOUDABI64_SYS_cloudabi_sys_sock_stat_get 53 -#define CLOUDABI64_SYS_cloudabi64_sys_thread_create 54 -#define CLOUDABI64_SYS_cloudabi_sys_thread_exit 55 -#define CLOUDABI64_SYS_cloudabi_sys_thread_yield 56 -#define CLOUDABI64_SYS_MAXSYSCALL 57 +#define CLOUDABI64_SYS_cloudabi_sys_mem_map 33 +#define CLOUDABI64_SYS_cloudabi_sys_mem_protect 34 +#define CLOUDABI64_SYS_cloudabi_sys_mem_sync 35 +#define CLOUDABI64_SYS_cloudabi_sys_mem_unmap 36 +#define CLOUDABI64_SYS_cloudabi64_sys_poll 37 +#define CLOUDABI64_SYS_cloudabi64_sys_poll_fd 38 +#define CLOUDABI64_SYS_cloudabi_sys_proc_exec 39 +#define CLOUDABI64_SYS_cloudabi_sys_proc_exit 40 +#define CLOUDABI64_SYS_cloudabi_sys_proc_fork 41 +#define CLOUDABI64_SYS_cloudabi_sys_proc_raise 42 +#define CLOUDABI64_SYS_cloudabi_sys_random_get 43 +#define CLOUDABI64_SYS_cloudabi_sys_sock_accept 44 +#define CLOUDABI64_SYS_cloudabi_sys_sock_bind 45 +#define CLOUDABI64_SYS_cloudabi_sys_sock_connect 46 +#define CLOUDABI64_SYS_cloudabi_sys_sock_listen 47 +#define CLOUDABI64_SYS_cloudabi64_sys_sock_recv 48 +#define CLOUDABI64_SYS_cloudabi64_sys_sock_send 49 +#define CLOUDABI64_SYS_cloudabi_sys_sock_shutdown 50 +#define CLOUDABI64_SYS_cloudabi_sys_sock_stat_get 51 +#define CLOUDABI64_SYS_cloudabi64_sys_thread_create 52 +#define CLOUDABI64_SYS_cloudabi_sys_thread_exit 53 +#define CLOUDABI64_SYS_cloudabi_sys_thread_yield 54 +#define CLOUDABI64_SYS_MAXSYSCALL 55 Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscalls.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscalls.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_syscalls.c (revision 321585) @@ -1,66 +1,64 @@ /* * System call names. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ const char *cloudabi64_syscallnames[] = { "cloudabi_sys_clock_res_get", /* 0 = cloudabi_sys_clock_res_get */ "cloudabi_sys_clock_time_get", /* 1 = cloudabi_sys_clock_time_get */ "cloudabi_sys_condvar_signal", /* 2 = cloudabi_sys_condvar_signal */ "cloudabi_sys_fd_close", /* 3 = cloudabi_sys_fd_close */ "cloudabi_sys_fd_create1", /* 4 = cloudabi_sys_fd_create1 */ "cloudabi_sys_fd_create2", /* 5 = cloudabi_sys_fd_create2 */ "cloudabi_sys_fd_datasync", /* 6 = cloudabi_sys_fd_datasync */ "cloudabi_sys_fd_dup", /* 7 = cloudabi_sys_fd_dup */ "cloudabi64_sys_fd_pread", /* 8 = cloudabi64_sys_fd_pread */ "cloudabi64_sys_fd_pwrite", /* 9 = cloudabi64_sys_fd_pwrite */ "cloudabi64_sys_fd_read", /* 10 = cloudabi64_sys_fd_read */ "cloudabi_sys_fd_replace", /* 11 = cloudabi_sys_fd_replace */ "cloudabi_sys_fd_seek", /* 12 = cloudabi_sys_fd_seek */ "cloudabi_sys_fd_stat_get", /* 13 = cloudabi_sys_fd_stat_get */ "cloudabi_sys_fd_stat_put", /* 14 = cloudabi_sys_fd_stat_put */ "cloudabi_sys_fd_sync", /* 15 = cloudabi_sys_fd_sync */ "cloudabi64_sys_fd_write", /* 16 = cloudabi64_sys_fd_write */ "cloudabi_sys_file_advise", /* 17 = cloudabi_sys_file_advise */ "cloudabi_sys_file_allocate", /* 18 = cloudabi_sys_file_allocate */ "cloudabi_sys_file_create", /* 19 = cloudabi_sys_file_create */ "cloudabi_sys_file_link", /* 20 = cloudabi_sys_file_link */ "cloudabi_sys_file_open", /* 21 = cloudabi_sys_file_open */ "cloudabi_sys_file_readdir", /* 22 = cloudabi_sys_file_readdir */ "cloudabi_sys_file_readlink", /* 23 = cloudabi_sys_file_readlink */ "cloudabi_sys_file_rename", /* 24 = cloudabi_sys_file_rename */ "cloudabi_sys_file_stat_fget", /* 25 = cloudabi_sys_file_stat_fget */ "cloudabi_sys_file_stat_fput", /* 26 = cloudabi_sys_file_stat_fput */ "cloudabi_sys_file_stat_get", /* 27 = cloudabi_sys_file_stat_get */ "cloudabi_sys_file_stat_put", /* 28 = cloudabi_sys_file_stat_put */ "cloudabi_sys_file_symlink", /* 29 = cloudabi_sys_file_symlink */ "cloudabi_sys_file_unlink", /* 30 = cloudabi_sys_file_unlink */ "cloudabi_sys_lock_unlock", /* 31 = cloudabi_sys_lock_unlock */ "cloudabi_sys_mem_advise", /* 32 = cloudabi_sys_mem_advise */ - "cloudabi_sys_mem_lock", /* 33 = cloudabi_sys_mem_lock */ - "cloudabi_sys_mem_map", /* 34 = cloudabi_sys_mem_map */ - "cloudabi_sys_mem_protect", /* 35 = cloudabi_sys_mem_protect */ - "cloudabi_sys_mem_sync", /* 36 = cloudabi_sys_mem_sync */ - "cloudabi_sys_mem_unlock", /* 37 = cloudabi_sys_mem_unlock */ - "cloudabi_sys_mem_unmap", /* 38 = cloudabi_sys_mem_unmap */ - "cloudabi64_sys_poll", /* 39 = cloudabi64_sys_poll */ - "cloudabi64_sys_poll_fd", /* 40 = cloudabi64_sys_poll_fd */ - "cloudabi_sys_proc_exec", /* 41 = cloudabi_sys_proc_exec */ - "cloudabi_sys_proc_exit", /* 42 = cloudabi_sys_proc_exit */ - "cloudabi_sys_proc_fork", /* 43 = cloudabi_sys_proc_fork */ - "cloudabi_sys_proc_raise", /* 44 = cloudabi_sys_proc_raise */ - "cloudabi_sys_random_get", /* 45 = cloudabi_sys_random_get */ - "cloudabi_sys_sock_accept", /* 46 = cloudabi_sys_sock_accept */ - "cloudabi_sys_sock_bind", /* 47 = cloudabi_sys_sock_bind */ - "cloudabi_sys_sock_connect", /* 48 = cloudabi_sys_sock_connect */ - "cloudabi_sys_sock_listen", /* 49 = cloudabi_sys_sock_listen */ - "cloudabi64_sys_sock_recv", /* 50 = cloudabi64_sys_sock_recv */ - "cloudabi64_sys_sock_send", /* 51 = cloudabi64_sys_sock_send */ - "cloudabi_sys_sock_shutdown", /* 52 = cloudabi_sys_sock_shutdown */ - "cloudabi_sys_sock_stat_get", /* 53 = cloudabi_sys_sock_stat_get */ - "cloudabi64_sys_thread_create", /* 54 = cloudabi64_sys_thread_create */ - "cloudabi_sys_thread_exit", /* 55 = cloudabi_sys_thread_exit */ - "cloudabi_sys_thread_yield", /* 56 = cloudabi_sys_thread_yield */ + "cloudabi_sys_mem_map", /* 33 = cloudabi_sys_mem_map */ + "cloudabi_sys_mem_protect", /* 34 = cloudabi_sys_mem_protect */ + "cloudabi_sys_mem_sync", /* 35 = cloudabi_sys_mem_sync */ + "cloudabi_sys_mem_unmap", /* 36 = cloudabi_sys_mem_unmap */ + "cloudabi64_sys_poll", /* 37 = cloudabi64_sys_poll */ + "cloudabi64_sys_poll_fd", /* 38 = cloudabi64_sys_poll_fd */ + "cloudabi_sys_proc_exec", /* 39 = cloudabi_sys_proc_exec */ + "cloudabi_sys_proc_exit", /* 40 = cloudabi_sys_proc_exit */ + "cloudabi_sys_proc_fork", /* 41 = cloudabi_sys_proc_fork */ + "cloudabi_sys_proc_raise", /* 42 = cloudabi_sys_proc_raise */ + "cloudabi_sys_random_get", /* 43 = cloudabi_sys_random_get */ + "cloudabi_sys_sock_accept", /* 44 = cloudabi_sys_sock_accept */ + "cloudabi_sys_sock_bind", /* 45 = cloudabi_sys_sock_bind */ + "cloudabi_sys_sock_connect", /* 46 = cloudabi_sys_sock_connect */ + "cloudabi_sys_sock_listen", /* 47 = cloudabi_sys_sock_listen */ + "cloudabi64_sys_sock_recv", /* 48 = cloudabi64_sys_sock_recv */ + "cloudabi64_sys_sock_send", /* 49 = cloudabi64_sys_sock_send */ + "cloudabi_sys_sock_shutdown", /* 50 = cloudabi_sys_sock_shutdown */ + "cloudabi_sys_sock_stat_get", /* 51 = cloudabi_sys_sock_stat_get */ + "cloudabi64_sys_thread_create", /* 52 = cloudabi64_sys_thread_create */ + "cloudabi_sys_thread_exit", /* 53 = cloudabi_sys_thread_exit */ + "cloudabi_sys_thread_yield", /* 54 = cloudabi_sys_thread_yield */ }; Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sysent.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sysent.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_sysent.c (revision 321585) @@ -1,74 +1,72 @@ /* * System call switch table. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ */ #include #include #include #include #define AS(name) (sizeof(struct name) / sizeof(register_t)) /* The casts are bogus but will do for now. */ struct sysent cloudabi64_sysent[] = { { AS(cloudabi_sys_clock_res_get_args), (sy_call_t *)cloudabi_sys_clock_res_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 0 = cloudabi_sys_clock_res_get */ { AS(cloudabi_sys_clock_time_get_args), (sy_call_t *)cloudabi_sys_clock_time_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 1 = cloudabi_sys_clock_time_get */ { AS(cloudabi_sys_condvar_signal_args), (sy_call_t *)cloudabi_sys_condvar_signal, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 2 = cloudabi_sys_condvar_signal */ { AS(cloudabi_sys_fd_close_args), (sy_call_t *)cloudabi_sys_fd_close, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 3 = cloudabi_sys_fd_close */ { AS(cloudabi_sys_fd_create1_args), (sy_call_t *)cloudabi_sys_fd_create1, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 4 = cloudabi_sys_fd_create1 */ { AS(cloudabi_sys_fd_create2_args), (sy_call_t *)cloudabi_sys_fd_create2, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 5 = cloudabi_sys_fd_create2 */ { AS(cloudabi_sys_fd_datasync_args), (sy_call_t *)cloudabi_sys_fd_datasync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 6 = cloudabi_sys_fd_datasync */ { AS(cloudabi_sys_fd_dup_args), (sy_call_t *)cloudabi_sys_fd_dup, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 7 = cloudabi_sys_fd_dup */ { AS(cloudabi64_sys_fd_pread_args), (sy_call_t *)cloudabi64_sys_fd_pread, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 8 = cloudabi64_sys_fd_pread */ { AS(cloudabi64_sys_fd_pwrite_args), (sy_call_t *)cloudabi64_sys_fd_pwrite, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 9 = cloudabi64_sys_fd_pwrite */ { AS(cloudabi64_sys_fd_read_args), (sy_call_t *)cloudabi64_sys_fd_read, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 10 = cloudabi64_sys_fd_read */ { AS(cloudabi_sys_fd_replace_args), (sy_call_t *)cloudabi_sys_fd_replace, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 11 = cloudabi_sys_fd_replace */ { AS(cloudabi_sys_fd_seek_args), (sy_call_t *)cloudabi_sys_fd_seek, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 12 = cloudabi_sys_fd_seek */ { AS(cloudabi_sys_fd_stat_get_args), (sy_call_t *)cloudabi_sys_fd_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 13 = cloudabi_sys_fd_stat_get */ { AS(cloudabi_sys_fd_stat_put_args), (sy_call_t *)cloudabi_sys_fd_stat_put, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 14 = cloudabi_sys_fd_stat_put */ { AS(cloudabi_sys_fd_sync_args), (sy_call_t *)cloudabi_sys_fd_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 15 = cloudabi_sys_fd_sync */ { AS(cloudabi64_sys_fd_write_args), (sy_call_t *)cloudabi64_sys_fd_write, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 16 = cloudabi64_sys_fd_write */ { AS(cloudabi_sys_file_advise_args), (sy_call_t *)cloudabi_sys_file_advise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 17 = cloudabi_sys_file_advise */ { AS(cloudabi_sys_file_allocate_args), (sy_call_t *)cloudabi_sys_file_allocate, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 18 = cloudabi_sys_file_allocate */ { AS(cloudabi_sys_file_create_args), (sy_call_t *)cloudabi_sys_file_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 19 = cloudabi_sys_file_create */ { AS(cloudabi_sys_file_link_args), (sy_call_t *)cloudabi_sys_file_link, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 20 = cloudabi_sys_file_link */ { AS(cloudabi_sys_file_open_args), (sy_call_t *)cloudabi_sys_file_open, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 21 = cloudabi_sys_file_open */ { AS(cloudabi_sys_file_readdir_args), (sy_call_t *)cloudabi_sys_file_readdir, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 22 = cloudabi_sys_file_readdir */ { AS(cloudabi_sys_file_readlink_args), (sy_call_t *)cloudabi_sys_file_readlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 23 = cloudabi_sys_file_readlink */ { AS(cloudabi_sys_file_rename_args), (sy_call_t *)cloudabi_sys_file_rename, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 24 = cloudabi_sys_file_rename */ { AS(cloudabi_sys_file_stat_fget_args), (sy_call_t *)cloudabi_sys_file_stat_fget, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 25 = cloudabi_sys_file_stat_fget */ { AS(cloudabi_sys_file_stat_fput_args), (sy_call_t *)cloudabi_sys_file_stat_fput, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 26 = cloudabi_sys_file_stat_fput */ { AS(cloudabi_sys_file_stat_get_args), (sy_call_t *)cloudabi_sys_file_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 27 = cloudabi_sys_file_stat_get */ { AS(cloudabi_sys_file_stat_put_args), (sy_call_t *)cloudabi_sys_file_stat_put, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 28 = cloudabi_sys_file_stat_put */ { AS(cloudabi_sys_file_symlink_args), (sy_call_t *)cloudabi_sys_file_symlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 29 = cloudabi_sys_file_symlink */ { AS(cloudabi_sys_file_unlink_args), (sy_call_t *)cloudabi_sys_file_unlink, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 30 = cloudabi_sys_file_unlink */ { AS(cloudabi_sys_lock_unlock_args), (sy_call_t *)cloudabi_sys_lock_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 31 = cloudabi_sys_lock_unlock */ { AS(cloudabi_sys_mem_advise_args), (sy_call_t *)cloudabi_sys_mem_advise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 32 = cloudabi_sys_mem_advise */ - { AS(cloudabi_sys_mem_lock_args), (sy_call_t *)cloudabi_sys_mem_lock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 33 = cloudabi_sys_mem_lock */ - { AS(cloudabi_sys_mem_map_args), (sy_call_t *)cloudabi_sys_mem_map, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 34 = cloudabi_sys_mem_map */ - { AS(cloudabi_sys_mem_protect_args), (sy_call_t *)cloudabi_sys_mem_protect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 35 = cloudabi_sys_mem_protect */ - { AS(cloudabi_sys_mem_sync_args), (sy_call_t *)cloudabi_sys_mem_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 36 = cloudabi_sys_mem_sync */ - { AS(cloudabi_sys_mem_unlock_args), (sy_call_t *)cloudabi_sys_mem_unlock, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 37 = cloudabi_sys_mem_unlock */ - { AS(cloudabi_sys_mem_unmap_args), (sy_call_t *)cloudabi_sys_mem_unmap, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 38 = cloudabi_sys_mem_unmap */ - { AS(cloudabi64_sys_poll_args), (sy_call_t *)cloudabi64_sys_poll, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 39 = cloudabi64_sys_poll */ - { AS(cloudabi64_sys_poll_fd_args), (sy_call_t *)cloudabi64_sys_poll_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 40 = cloudabi64_sys_poll_fd */ - { AS(cloudabi_sys_proc_exec_args), (sy_call_t *)cloudabi_sys_proc_exec, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 41 = cloudabi_sys_proc_exec */ - { AS(cloudabi_sys_proc_exit_args), (sy_call_t *)cloudabi_sys_proc_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 42 = cloudabi_sys_proc_exit */ - { 0, (sy_call_t *)cloudabi_sys_proc_fork, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 43 = cloudabi_sys_proc_fork */ - { AS(cloudabi_sys_proc_raise_args), (sy_call_t *)cloudabi_sys_proc_raise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 44 = cloudabi_sys_proc_raise */ - { AS(cloudabi_sys_random_get_args), (sy_call_t *)cloudabi_sys_random_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 45 = cloudabi_sys_random_get */ - { AS(cloudabi_sys_sock_accept_args), (sy_call_t *)cloudabi_sys_sock_accept, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 46 = cloudabi_sys_sock_accept */ - { AS(cloudabi_sys_sock_bind_args), (sy_call_t *)cloudabi_sys_sock_bind, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 47 = cloudabi_sys_sock_bind */ - { AS(cloudabi_sys_sock_connect_args), (sy_call_t *)cloudabi_sys_sock_connect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 48 = cloudabi_sys_sock_connect */ - { AS(cloudabi_sys_sock_listen_args), (sy_call_t *)cloudabi_sys_sock_listen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 49 = cloudabi_sys_sock_listen */ - { AS(cloudabi64_sys_sock_recv_args), (sy_call_t *)cloudabi64_sys_sock_recv, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 50 = cloudabi64_sys_sock_recv */ - { AS(cloudabi64_sys_sock_send_args), (sy_call_t *)cloudabi64_sys_sock_send, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 51 = cloudabi64_sys_sock_send */ - { AS(cloudabi_sys_sock_shutdown_args), (sy_call_t *)cloudabi_sys_sock_shutdown, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 52 = cloudabi_sys_sock_shutdown */ - { AS(cloudabi_sys_sock_stat_get_args), (sy_call_t *)cloudabi_sys_sock_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 53 = cloudabi_sys_sock_stat_get */ - { AS(cloudabi64_sys_thread_create_args), (sy_call_t *)cloudabi64_sys_thread_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 54 = cloudabi64_sys_thread_create */ - { AS(cloudabi_sys_thread_exit_args), (sy_call_t *)cloudabi_sys_thread_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 55 = cloudabi_sys_thread_exit */ - { 0, (sy_call_t *)cloudabi_sys_thread_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 56 = cloudabi_sys_thread_yield */ + { AS(cloudabi_sys_mem_map_args), (sy_call_t *)cloudabi_sys_mem_map, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 33 = cloudabi_sys_mem_map */ + { AS(cloudabi_sys_mem_protect_args), (sy_call_t *)cloudabi_sys_mem_protect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 34 = cloudabi_sys_mem_protect */ + { AS(cloudabi_sys_mem_sync_args), (sy_call_t *)cloudabi_sys_mem_sync, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 35 = cloudabi_sys_mem_sync */ + { AS(cloudabi_sys_mem_unmap_args), (sy_call_t *)cloudabi_sys_mem_unmap, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 36 = cloudabi_sys_mem_unmap */ + { AS(cloudabi64_sys_poll_args), (sy_call_t *)cloudabi64_sys_poll, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 37 = cloudabi64_sys_poll */ + { AS(cloudabi64_sys_poll_fd_args), (sy_call_t *)cloudabi64_sys_poll_fd, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 38 = cloudabi64_sys_poll_fd */ + { AS(cloudabi_sys_proc_exec_args), (sy_call_t *)cloudabi_sys_proc_exec, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 39 = cloudabi_sys_proc_exec */ + { AS(cloudabi_sys_proc_exit_args), (sy_call_t *)cloudabi_sys_proc_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 40 = cloudabi_sys_proc_exit */ + { 0, (sy_call_t *)cloudabi_sys_proc_fork, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 41 = cloudabi_sys_proc_fork */ + { AS(cloudabi_sys_proc_raise_args), (sy_call_t *)cloudabi_sys_proc_raise, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 42 = cloudabi_sys_proc_raise */ + { AS(cloudabi_sys_random_get_args), (sy_call_t *)cloudabi_sys_random_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 43 = cloudabi_sys_random_get */ + { AS(cloudabi_sys_sock_accept_args), (sy_call_t *)cloudabi_sys_sock_accept, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 44 = cloudabi_sys_sock_accept */ + { AS(cloudabi_sys_sock_bind_args), (sy_call_t *)cloudabi_sys_sock_bind, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 45 = cloudabi_sys_sock_bind */ + { AS(cloudabi_sys_sock_connect_args), (sy_call_t *)cloudabi_sys_sock_connect, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 46 = cloudabi_sys_sock_connect */ + { AS(cloudabi_sys_sock_listen_args), (sy_call_t *)cloudabi_sys_sock_listen, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 47 = cloudabi_sys_sock_listen */ + { AS(cloudabi64_sys_sock_recv_args), (sy_call_t *)cloudabi64_sys_sock_recv, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 48 = cloudabi64_sys_sock_recv */ + { AS(cloudabi64_sys_sock_send_args), (sy_call_t *)cloudabi64_sys_sock_send, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 49 = cloudabi64_sys_sock_send */ + { AS(cloudabi_sys_sock_shutdown_args), (sy_call_t *)cloudabi_sys_sock_shutdown, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 50 = cloudabi_sys_sock_shutdown */ + { AS(cloudabi_sys_sock_stat_get_args), (sy_call_t *)cloudabi_sys_sock_stat_get, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 51 = cloudabi_sys_sock_stat_get */ + { AS(cloudabi64_sys_thread_create_args), (sy_call_t *)cloudabi64_sys_thread_create, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 52 = cloudabi64_sys_thread_create */ + { AS(cloudabi_sys_thread_exit_args), (sy_call_t *)cloudabi_sys_thread_exit, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 53 = cloudabi_sys_thread_exit */ + { 0, (sy_call_t *)cloudabi_sys_thread_yield, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 54 = cloudabi_sys_thread_yield */ }; Index: projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_systrace_args.c =================================================================== --- projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_systrace_args.c (revision 321584) +++ projects/runtime-coverage/sys/compat/cloudabi64/cloudabi64_systrace_args.c (revision 321585) @@ -1,1702 +1,1650 @@ /* * System call argument to DTrace register array converstion. * * DO NOT EDIT-- this file is automatically generated. * $FreeBSD$ * This file is part of the DTrace syscall provider. */ static void systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) { int64_t *iarg = (int64_t *) uarg; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: { struct cloudabi_sys_clock_res_get_args *p = params; iarg[0] = p->clock_id; /* cloudabi_clockid_t */ *n_args = 1; break; } /* cloudabi_sys_clock_time_get */ case 1: { struct cloudabi_sys_clock_time_get_args *p = params; iarg[0] = p->clock_id; /* cloudabi_clockid_t */ iarg[1] = p->precision; /* cloudabi_timestamp_t */ *n_args = 2; break; } /* cloudabi_sys_condvar_signal */ case 2: { struct cloudabi_sys_condvar_signal_args *p = params; uarg[0] = (intptr_t) p->condvar; /* cloudabi_condvar_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ iarg[2] = p->nwaiters; /* cloudabi_nthreads_t */ *n_args = 3; break; } /* cloudabi_sys_fd_close */ case 3: { struct cloudabi_sys_fd_close_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi_sys_fd_create1 */ case 4: { struct cloudabi_sys_fd_create1_args *p = params; iarg[0] = p->type; /* cloudabi_filetype_t */ *n_args = 1; break; } /* cloudabi_sys_fd_create2 */ case 5: { struct cloudabi_sys_fd_create2_args *p = params; iarg[0] = p->type; /* cloudabi_filetype_t */ *n_args = 1; break; } /* cloudabi_sys_fd_datasync */ case 6: { struct cloudabi_sys_fd_datasync_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi_sys_fd_dup */ case 7: { struct cloudabi_sys_fd_dup_args *p = params; iarg[0] = p->from; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi64_sys_fd_pread */ case 8: { struct cloudabi64_sys_fd_pread_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi64_iovec_t * */ uarg[2] = p->iovs_len; /* size_t */ iarg[3] = p->offset; /* cloudabi_filesize_t */ *n_args = 4; break; } /* cloudabi64_sys_fd_pwrite */ case 9: { struct cloudabi64_sys_fd_pwrite_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi64_ciovec_t * */ uarg[2] = p->iovs_len; /* size_t */ iarg[3] = p->offset; /* cloudabi_filesize_t */ *n_args = 4; break; } /* cloudabi64_sys_fd_read */ case 10: { struct cloudabi64_sys_fd_read_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi64_iovec_t * */ uarg[2] = p->iovs_len; /* size_t */ *n_args = 3; break; } /* cloudabi_sys_fd_replace */ case 11: { struct cloudabi_sys_fd_replace_args *p = params; iarg[0] = p->from; /* cloudabi_fd_t */ iarg[1] = p->to; /* cloudabi_fd_t */ *n_args = 2; break; } /* cloudabi_sys_fd_seek */ case 12: { struct cloudabi_sys_fd_seek_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filedelta_t */ iarg[2] = p->whence; /* cloudabi_whence_t */ *n_args = 3; break; } /* cloudabi_sys_fd_stat_get */ case 13: { struct cloudabi_sys_fd_stat_get_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_fdstat_t * */ *n_args = 2; break; } /* cloudabi_sys_fd_stat_put */ case 14: { struct cloudabi_sys_fd_stat_put_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* const cloudabi_fdstat_t * */ iarg[2] = p->flags; /* cloudabi_fdsflags_t */ *n_args = 3; break; } /* cloudabi_sys_fd_sync */ case 15: { struct cloudabi_sys_fd_sync_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ *n_args = 1; break; } /* cloudabi64_sys_fd_write */ case 16: { struct cloudabi64_sys_fd_write_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->iovs; /* const cloudabi64_ciovec_t * */ uarg[2] = p->iovs_len; /* size_t */ *n_args = 3; break; } /* cloudabi_sys_file_advise */ case 17: { struct cloudabi_sys_file_advise_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filesize_t */ iarg[2] = p->len; /* cloudabi_filesize_t */ iarg[3] = p->advice; /* cloudabi_advice_t */ *n_args = 4; break; } /* cloudabi_sys_file_allocate */ case 18: { struct cloudabi_sys_file_allocate_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ iarg[1] = p->offset; /* cloudabi_filesize_t */ iarg[2] = p->len; /* cloudabi_filesize_t */ *n_args = 3; break; } /* cloudabi_sys_file_create */ case 19: { struct cloudabi_sys_file_create_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->type; /* cloudabi_filetype_t */ *n_args = 4; break; } /* cloudabi_sys_file_link */ case 20: { struct cloudabi_sys_file_link_args *p = params; iarg[0] = p->fd1; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path1; /* const char * */ uarg[2] = p->path1_len; /* size_t */ iarg[3] = p->fd2; /* cloudabi_fd_t */ uarg[4] = (intptr_t) p->path2; /* const char * */ uarg[5] = p->path2_len; /* size_t */ *n_args = 6; break; } /* cloudabi_sys_file_open */ case 21: { struct cloudabi_sys_file_open_args *p = params; iarg[0] = p->dirfd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->oflags; /* cloudabi_oflags_t */ uarg[4] = (intptr_t) p->fds; /* const cloudabi_fdstat_t * */ *n_args = 5; break; } /* cloudabi_sys_file_readdir */ case 22: { struct cloudabi_sys_file_readdir_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* void * */ uarg[2] = p->buf_len; /* size_t */ iarg[3] = p->cookie; /* cloudabi_dircookie_t */ *n_args = 4; break; } /* cloudabi_sys_file_readlink */ case 23: { struct cloudabi_sys_file_readlink_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* char * */ uarg[4] = p->buf_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_file_rename */ case 24: { struct cloudabi_sys_file_rename_args *p = params; iarg[0] = p->fd1; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path1; /* const char * */ uarg[2] = p->path1_len; /* size_t */ iarg[3] = p->fd2; /* cloudabi_fd_t */ uarg[4] = (intptr_t) p->path2; /* const char * */ uarg[5] = p->path2_len; /* size_t */ *n_args = 6; break; } /* cloudabi_sys_file_stat_fget */ case 25: { struct cloudabi_sys_file_stat_fget_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_filestat_t * */ *n_args = 2; break; } /* cloudabi_sys_file_stat_fput */ case 26: { struct cloudabi_sys_file_stat_fput_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* const cloudabi_filestat_t * */ iarg[2] = p->flags; /* cloudabi_fsflags_t */ *n_args = 3; break; } /* cloudabi_sys_file_stat_get */ case 27: { struct cloudabi_sys_file_stat_get_args *p = params; iarg[0] = p->fd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* cloudabi_filestat_t * */ *n_args = 4; break; } /* cloudabi_sys_file_stat_put */ case 28: { struct cloudabi_sys_file_stat_put_args *p = params; iarg[0] = p->fd; /* cloudabi_lookup_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ uarg[3] = (intptr_t) p->buf; /* const cloudabi_filestat_t * */ iarg[4] = p->flags; /* cloudabi_fsflags_t */ *n_args = 5; break; } /* cloudabi_sys_file_symlink */ case 29: { struct cloudabi_sys_file_symlink_args *p = params; uarg[0] = (intptr_t) p->path1; /* const char * */ uarg[1] = p->path1_len; /* size_t */ iarg[2] = p->fd; /* cloudabi_fd_t */ uarg[3] = (intptr_t) p->path2; /* const char * */ uarg[4] = p->path2_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_file_unlink */ case 30: { struct cloudabi_sys_file_unlink_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->path; /* const char * */ uarg[2] = p->path_len; /* size_t */ iarg[3] = p->flags; /* cloudabi_ulflags_t */ *n_args = 4; break; } /* cloudabi_sys_lock_unlock */ case 31: { struct cloudabi_sys_lock_unlock_args *p = params; uarg[0] = (intptr_t) p->lock; /* cloudabi_lock_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ *n_args = 2; break; } /* cloudabi_sys_mem_advise */ case 32: { struct cloudabi_sys_mem_advise_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->advice; /* cloudabi_advice_t */ *n_args = 3; break; } - /* cloudabi_sys_mem_lock */ - case 33: { - struct cloudabi_sys_mem_lock_args *p = params; - uarg[0] = (intptr_t) p->mapping; /* const void * */ - uarg[1] = p->mapping_len; /* size_t */ - *n_args = 2; - break; - } /* cloudabi_sys_mem_map */ - case 34: { + case 33: { struct cloudabi_sys_mem_map_args *p = params; uarg[0] = (intptr_t) p->addr; /* void * */ uarg[1] = p->len; /* size_t */ iarg[2] = p->prot; /* cloudabi_mprot_t */ iarg[3] = p->flags; /* cloudabi_mflags_t */ iarg[4] = p->fd; /* cloudabi_fd_t */ iarg[5] = p->off; /* cloudabi_filesize_t */ *n_args = 6; break; } /* cloudabi_sys_mem_protect */ - case 35: { + case 34: { struct cloudabi_sys_mem_protect_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->prot; /* cloudabi_mprot_t */ *n_args = 3; break; } /* cloudabi_sys_mem_sync */ - case 36: { + case 35: { struct cloudabi_sys_mem_sync_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ iarg[2] = p->flags; /* cloudabi_msflags_t */ *n_args = 3; break; } - /* cloudabi_sys_mem_unlock */ - case 37: { - struct cloudabi_sys_mem_unlock_args *p = params; - uarg[0] = (intptr_t) p->mapping; /* const void * */ - uarg[1] = p->mapping_len; /* size_t */ - *n_args = 2; - break; - } /* cloudabi_sys_mem_unmap */ - case 38: { + case 36: { struct cloudabi_sys_mem_unmap_args *p = params; uarg[0] = (intptr_t) p->mapping; /* void * */ uarg[1] = p->mapping_len; /* size_t */ *n_args = 2; break; } /* cloudabi64_sys_poll */ - case 39: { + case 37: { struct cloudabi64_sys_poll_args *p = params; uarg[0] = (intptr_t) p->in; /* const cloudabi64_subscription_t * */ uarg[1] = (intptr_t) p->out; /* cloudabi64_event_t * */ uarg[2] = p->nsubscriptions; /* size_t */ *n_args = 3; break; } /* cloudabi64_sys_poll_fd */ - case 40: { + case 38: { struct cloudabi64_sys_poll_fd_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi64_subscription_t * */ uarg[2] = p->in_len; /* size_t */ uarg[3] = (intptr_t) p->out; /* cloudabi64_event_t * */ uarg[4] = p->out_len; /* size_t */ uarg[5] = (intptr_t) p->timeout; /* const cloudabi64_subscription_t * */ *n_args = 6; break; } /* cloudabi_sys_proc_exec */ - case 41: { + case 39: { struct cloudabi_sys_proc_exec_args *p = params; iarg[0] = p->fd; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->data; /* const void * */ uarg[2] = p->data_len; /* size_t */ uarg[3] = (intptr_t) p->fds; /* const cloudabi_fd_t * */ uarg[4] = p->fds_len; /* size_t */ *n_args = 5; break; } /* cloudabi_sys_proc_exit */ - case 42: { + case 40: { struct cloudabi_sys_proc_exit_args *p = params; iarg[0] = p->rval; /* cloudabi_exitcode_t */ *n_args = 1; break; } /* cloudabi_sys_proc_fork */ - case 43: { + case 41: { *n_args = 0; break; } /* cloudabi_sys_proc_raise */ - case 44: { + case 42: { struct cloudabi_sys_proc_raise_args *p = params; iarg[0] = p->sig; /* cloudabi_signal_t */ *n_args = 1; break; } /* cloudabi_sys_random_get */ - case 45: { + case 43: { struct cloudabi_sys_random_get_args *p = params; uarg[0] = (intptr_t) p->buf; /* void * */ uarg[1] = p->buf_len; /* size_t */ *n_args = 2; break; } /* cloudabi_sys_sock_accept */ - case 46: { + case 44: { struct cloudabi_sys_sock_accept_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ - uarg[1] = (intptr_t) p->buf; /* cloudabi_sockstat_t * */ + uarg[1] = (intptr_t) p->unused; /* void * */ *n_args = 2; break; } /* cloudabi_sys_sock_bind */ - case 47: { + case 45: { struct cloudabi_sys_sock_bind_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->fd; /* cloudabi_fd_t */ uarg[2] = (intptr_t) p->path; /* const char * */ uarg[3] = p->path_len; /* size_t */ *n_args = 4; break; } /* cloudabi_sys_sock_connect */ - case 48: { + case 46: { struct cloudabi_sys_sock_connect_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->fd; /* cloudabi_fd_t */ uarg[2] = (intptr_t) p->path; /* const char * */ uarg[3] = p->path_len; /* size_t */ *n_args = 4; break; } /* cloudabi_sys_sock_listen */ - case 49: { + case 47: { struct cloudabi_sys_sock_listen_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->backlog; /* cloudabi_backlog_t */ *n_args = 2; break; } /* cloudabi64_sys_sock_recv */ - case 50: { + case 48: { struct cloudabi64_sys_sock_recv_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi64_recv_in_t * */ uarg[2] = (intptr_t) p->out; /* cloudabi64_recv_out_t * */ *n_args = 3; break; } /* cloudabi64_sys_sock_send */ - case 51: { + case 49: { struct cloudabi64_sys_sock_send_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->in; /* const cloudabi64_send_in_t * */ uarg[2] = (intptr_t) p->out; /* cloudabi64_send_out_t * */ *n_args = 3; break; } /* cloudabi_sys_sock_shutdown */ - case 52: { + case 50: { struct cloudabi_sys_sock_shutdown_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ iarg[1] = p->how; /* cloudabi_sdflags_t */ *n_args = 2; break; } /* cloudabi_sys_sock_stat_get */ - case 53: { + case 51: { struct cloudabi_sys_sock_stat_get_args *p = params; iarg[0] = p->sock; /* cloudabi_fd_t */ uarg[1] = (intptr_t) p->buf; /* cloudabi_sockstat_t * */ iarg[2] = p->flags; /* cloudabi_ssflags_t */ *n_args = 3; break; } /* cloudabi64_sys_thread_create */ - case 54: { + case 52: { struct cloudabi64_sys_thread_create_args *p = params; uarg[0] = (intptr_t) p->attr; /* cloudabi64_threadattr_t * */ *n_args = 1; break; } /* cloudabi_sys_thread_exit */ - case 55: { + case 53: { struct cloudabi_sys_thread_exit_args *p = params; uarg[0] = (intptr_t) p->lock; /* cloudabi_lock_t * */ iarg[1] = p->scope; /* cloudabi_scope_t */ *n_args = 2; break; } /* cloudabi_sys_thread_yield */ - case 56: { + case 54: { *n_args = 0; break; } default: *n_args = 0; break; }; } static void systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) { const char *p = NULL; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: switch(ndx) { case 0: p = "cloudabi_clockid_t"; break; default: break; }; break; /* cloudabi_sys_clock_time_get */ case 1: switch(ndx) { case 0: p = "cloudabi_clockid_t"; break; case 1: p = "cloudabi_timestamp_t"; break; default: break; }; break; /* cloudabi_sys_condvar_signal */ case 2: switch(ndx) { case 0: p = "userland cloudabi_condvar_t *"; break; case 1: p = "cloudabi_scope_t"; break; case 2: p = "cloudabi_nthreads_t"; break; default: break; }; break; /* cloudabi_sys_fd_close */ case 3: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_create1 */ case 4: switch(ndx) { case 0: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_fd_create2 */ case 5: switch(ndx) { case 0: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_fd_datasync */ case 6: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_dup */ case 7: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi64_sys_fd_pread */ case 8: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_iovec_t *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi64_sys_fd_pwrite */ case 9: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_ciovec_t *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi64_sys_fd_read */ case 10: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_iovec_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_fd_replace */ case 11: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi_sys_fd_seek */ case 12: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filedelta_t"; break; case 2: p = "cloudabi_whence_t"; break; default: break; }; break; /* cloudabi_sys_fd_stat_get */ case 13: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_fdstat_t *"; break; default: break; }; break; /* cloudabi_sys_fd_stat_put */ case 14: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi_fdstat_t *"; break; case 2: p = "cloudabi_fdsflags_t"; break; default: break; }; break; /* cloudabi_sys_fd_sync */ case 15: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; default: break; }; break; /* cloudabi64_sys_fd_write */ case 16: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_ciovec_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_advise */ case 17: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filesize_t"; break; case 2: p = "cloudabi_filesize_t"; break; case 3: p = "cloudabi_advice_t"; break; default: break; }; break; /* cloudabi_sys_file_allocate */ case 18: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_filesize_t"; break; case 2: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi_sys_file_create */ case 19: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_filetype_t"; break; default: break; }; break; /* cloudabi_sys_file_link */ case 20: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_fd_t"; break; case 4: p = "userland const char *"; break; case 5: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_open */ case 21: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_oflags_t"; break; case 4: p = "userland const cloudabi_fdstat_t *"; break; default: break; }; break; /* cloudabi_sys_file_readdir */ case 22: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland void *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_dircookie_t"; break; default: break; }; break; /* cloudabi_sys_file_readlink */ case 23: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland char *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_rename */ case 24: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_fd_t"; break; case 4: p = "userland const char *"; break; case 5: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_stat_fget */ case 25: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_filestat_t *"; break; default: break; }; break; /* cloudabi_sys_file_stat_fput */ case 26: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi_filestat_t *"; break; case 2: p = "cloudabi_fsflags_t"; break; default: break; }; break; /* cloudabi_sys_file_stat_get */ case 27: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland cloudabi_filestat_t *"; break; default: break; }; break; /* cloudabi_sys_file_stat_put */ case 28: switch(ndx) { case 0: p = "cloudabi_lookup_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "userland const cloudabi_filestat_t *"; break; case 4: p = "cloudabi_fsflags_t"; break; default: break; }; break; /* cloudabi_sys_file_symlink */ case 29: switch(ndx) { case 0: p = "userland const char *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_fd_t"; break; case 3: p = "userland const char *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_file_unlink */ case 30: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const char *"; break; case 2: p = "size_t"; break; case 3: p = "cloudabi_ulflags_t"; break; default: break; }; break; /* cloudabi_sys_lock_unlock */ case 31: switch(ndx) { case 0: p = "userland cloudabi_lock_t *"; break; case 1: p = "cloudabi_scope_t"; break; default: break; }; break; /* cloudabi_sys_mem_advise */ case 32: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_advice_t"; break; default: break; }; break; - /* cloudabi_sys_mem_lock */ + /* cloudabi_sys_mem_map */ case 33: switch(ndx) { case 0: - p = "userland const void *"; - break; - case 1: - p = "size_t"; - break; - default: - break; - }; - break; - /* cloudabi_sys_mem_map */ - case 34: - switch(ndx) { - case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_mprot_t"; break; case 3: p = "cloudabi_mflags_t"; break; case 4: p = "cloudabi_fd_t"; break; case 5: p = "cloudabi_filesize_t"; break; default: break; }; break; /* cloudabi_sys_mem_protect */ - case 35: + case 34: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_mprot_t"; break; default: break; }; break; /* cloudabi_sys_mem_sync */ - case 36: + case 35: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; case 2: p = "cloudabi_msflags_t"; break; default: break; }; break; - /* cloudabi_sys_mem_unlock */ - case 37: - switch(ndx) { - case 0: - p = "userland const void *"; - break; - case 1: - p = "size_t"; - break; - default: - break; - }; - break; /* cloudabi_sys_mem_unmap */ - case 38: + case 36: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; default: break; }; break; /* cloudabi64_sys_poll */ - case 39: + case 37: switch(ndx) { case 0: p = "userland const cloudabi64_subscription_t *"; break; case 1: p = "userland cloudabi64_event_t *"; break; case 2: p = "size_t"; break; default: break; }; break; /* cloudabi64_sys_poll_fd */ - case 40: + case 38: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_subscription_t *"; break; case 2: p = "size_t"; break; case 3: p = "userland cloudabi64_event_t *"; break; case 4: p = "size_t"; break; case 5: p = "userland const cloudabi64_subscription_t *"; break; default: break; }; break; /* cloudabi_sys_proc_exec */ - case 41: + case 39: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const void *"; break; case 2: p = "size_t"; break; case 3: p = "userland const cloudabi_fd_t *"; break; case 4: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_proc_exit */ - case 42: + case 40: switch(ndx) { case 0: p = "cloudabi_exitcode_t"; break; default: break; }; break; /* cloudabi_sys_proc_fork */ - case 43: + case 41: break; /* cloudabi_sys_proc_raise */ - case 44: + case 42: switch(ndx) { case 0: p = "cloudabi_signal_t"; break; default: break; }; break; /* cloudabi_sys_random_get */ - case 45: + case 43: switch(ndx) { case 0: p = "userland void *"; break; case 1: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_accept */ - case 46: + case 44: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: - p = "userland cloudabi_sockstat_t *"; + p = "userland void *"; break; default: break; }; break; /* cloudabi_sys_sock_bind */ - case 47: + case 45: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; case 2: p = "userland const char *"; break; case 3: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_connect */ - case 48: + case 46: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_fd_t"; break; case 2: p = "userland const char *"; break; case 3: p = "size_t"; break; default: break; }; break; /* cloudabi_sys_sock_listen */ - case 49: + case 47: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_backlog_t"; break; default: break; }; break; /* cloudabi64_sys_sock_recv */ - case 50: + case 48: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_recv_in_t *"; break; case 2: p = "userland cloudabi64_recv_out_t *"; break; default: break; }; break; /* cloudabi64_sys_sock_send */ - case 51: + case 49: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland const cloudabi64_send_in_t *"; break; case 2: p = "userland cloudabi64_send_out_t *"; break; default: break; }; break; /* cloudabi_sys_sock_shutdown */ - case 52: + case 50: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "cloudabi_sdflags_t"; break; default: break; }; break; /* cloudabi_sys_sock_stat_get */ - case 53: + case 51: switch(ndx) { case 0: p = "cloudabi_fd_t"; break; case 1: p = "userland cloudabi_sockstat_t *"; break; case 2: p = "cloudabi_ssflags_t"; break; default: break; }; break; /* cloudabi64_sys_thread_create */ - case 54: + case 52: switch(ndx) { case 0: p = "userland cloudabi64_threadattr_t *"; break; default: break; }; break; /* cloudabi_sys_thread_exit */ - case 55: + case 53: switch(ndx) { case 0: p = "userland cloudabi_lock_t *"; break; case 1: p = "cloudabi_scope_t"; break; default: break; }; break; /* cloudabi_sys_thread_yield */ - case 56: + case 54: break; default: break; }; if (p != NULL) strlcpy(desc, p, descsz); } static void systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) { const char *p = NULL; switch (sysnum) { /* cloudabi_sys_clock_res_get */ case 0: if (ndx == 0 || ndx == 1) p = "cloudabi_timestamp_t"; break; /* cloudabi_sys_clock_time_get */ case 1: if (ndx == 0 || ndx == 1) p = "cloudabi_timestamp_t"; break; /* cloudabi_sys_condvar_signal */ case 2: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_close */ case 3: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_create1 */ case 4: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_fd_create2 */ case 5: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_datasync */ case 6: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_dup */ case 7: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi64_sys_fd_pread */ case 8: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi64_sys_fd_pwrite */ case 9: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi64_sys_fd_read */ case 10: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_fd_replace */ case 11: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_seek */ case 12: if (ndx == 0 || ndx == 1) p = "cloudabi_filesize_t"; break; /* cloudabi_sys_fd_stat_get */ case 13: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_stat_put */ case 14: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_fd_sync */ case 15: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi64_sys_fd_write */ case 16: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_advise */ case 17: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_allocate */ case 18: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_create */ case 19: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_link */ case 20: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_open */ case 21: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_file_readdir */ case 22: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_readlink */ case 23: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_file_rename */ case 24: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_fget */ case 25: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_fput */ case 26: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_get */ case 27: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_stat_put */ case 28: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_symlink */ case 29: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_file_unlink */ case 30: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_lock_unlock */ case 31: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_mem_advise */ case 32: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_lock */ + /* cloudabi_sys_mem_map */ case 33: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_map */ + /* cloudabi_sys_mem_protect */ case 34: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_protect */ + /* cloudabi_sys_mem_sync */ case 35: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_sync */ + /* cloudabi_sys_mem_unmap */ case 36: if (ndx == 0 || ndx == 1) p = "void"; break; - /* cloudabi_sys_mem_unlock */ + /* cloudabi64_sys_poll */ case 37: if (ndx == 0 || ndx == 1) - p = "void"; - break; - /* cloudabi_sys_mem_unmap */ - case 38: - if (ndx == 0 || ndx == 1) - p = "void"; - break; - /* cloudabi64_sys_poll */ - case 39: - if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi64_sys_poll_fd */ - case 40: + case 38: if (ndx == 0 || ndx == 1) p = "size_t"; break; /* cloudabi_sys_proc_exec */ - case 41: + case 39: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_proc_exit */ - case 42: + case 40: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_proc_fork */ - case 43: + case 41: /* cloudabi_sys_proc_raise */ - case 44: + case 42: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_random_get */ - case 45: + case 43: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_accept */ - case 46: + case 44: if (ndx == 0 || ndx == 1) p = "cloudabi_fd_t"; break; /* cloudabi_sys_sock_bind */ - case 47: + case 45: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_connect */ - case 48: + case 46: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_listen */ - case 49: + case 47: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi64_sys_sock_recv */ - case 50: + case 48: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi64_sys_sock_send */ - case 51: + case 49: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_shutdown */ - case 52: + case 50: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_sock_stat_get */ - case 53: + case 51: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi64_sys_thread_create */ - case 54: + case 52: if (ndx == 0 || ndx == 1) p = "cloudabi_tid_t"; break; /* cloudabi_sys_thread_exit */ - case 55: + case 53: if (ndx == 0 || ndx == 1) p = "void"; break; /* cloudabi_sys_thread_yield */ - case 56: + case 54: default: break; }; if (p != NULL) strlcpy(desc, p, descsz); } Index: projects/runtime-coverage/sys/conf/kern.post.mk =================================================================== --- projects/runtime-coverage/sys/conf/kern.post.mk (revision 321584) +++ projects/runtime-coverage/sys/conf/kern.post.mk (revision 321585) @@ -1,415 +1,415 @@ # $FreeBSD$ # Part of a unified Makefile for building kernels. This part includes all # the definitions that need to be after all the % directives except %RULES # and ones that act like they are part of %RULES. # # Most make variables should not be defined in this file. Instead, they # should be defined in the kern.pre.mk so that port makefiles can # override or augment them. # In case the config had a makeoptions DESTDIR... .if defined(DESTDIR) MKMODULESENV+= DESTDIR="${DESTDIR}" .endif SYSDIR?= ${S:C;^[^/];${.CURDIR}/&;:tA} MKMODULESENV+= KERNBUILDDIR="${.CURDIR}" SYSDIR="${SYSDIR}" .if defined(CONF_CFLAGS) MKMODULESENV+= CONF_CFLAGS="${CONF_CFLAGS}" .endif .if defined(WITH_CTF) MKMODULESENV+= WITH_CTF="${WITH_CTF}" .endif .if defined(WITH_EXTRA_TCP_STACKS) MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}" .endif # Allow overriding the kernel debug directory, so kernel and user debug may be # installed in different directories. Setting it to "" restores the historical # behavior of installing debug files in the kernel directory. KERN_DEBUGDIR?= ${DEBUGDIR} .MAIN: all .for target in all clean cleandepend cleandir clobber depend install \ ${_obj} reinstall tags ${target}: kernel-${target} .if !defined(MODULES_WITH_WORLD) && !defined(NO_MODULES) && exists($S/modules) ${target}: modules-${target} modules-${target}: cd $S/modules; ${MKMODULESENV} ${MAKE} \ ${target:S/^reinstall$/install/:S/^clobber$/cleandir/} .endif .endfor # Handle ports (as defined by the user) that build kernel modules .if !defined(NO_MODULES) && defined(PORTS_MODULES) # # The ports tree needs some environment variables defined to match the new kernel # # Ports search for some dependencies in PATH, so add the location of the installed files LOCALBASE?= /usr/local # SRC_BASE is how the ports tree refers to the location of the base source files .if !defined(SRC_BASE) SRC_BASE= ${SYSDIR:H:tA} .endif # OSVERSION is used by some ports to determine build options .if !defined(OSRELDATE) # Definition copied from src/Makefile.inc1 OSRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ ${MAKEOBJDIRPREFIX}${SRC_BASE}/include/osreldate.h .endif # Keep the related ports builds in the obj directory so that they are only rebuilt once per kernel build WRKDIRPREFIX?= ${.OBJDIR} PORTSMODULESENV=\ env \ -u CC \ -u CXX \ -u CPP \ PATH=${PATH}:${LOCALBASE}/bin:${LOCALBASE}/sbin \ SRC_BASE=${SRC_BASE} \ OSVERSION=${OSRELDATE} \ WRKDIRPREFIX=${WRKDIRPREFIX} # The WRKDIR needs to be cleaned before building, and trying to change the target # with a :C pattern below results in install -> instclean all: .for __i in ${PORTS_MODULES} @${ECHO} "===> Ports module ${__i} (all)" cd $${PORTSDIR:-/usr/ports}/${__i}; ${PORTSMODULESENV} ${MAKE} -B clean build .endfor .for __target in install reinstall clean ${__target}: ports-${__target} ports-${__target}: .for __i in ${PORTS_MODULES} @${ECHO} "===> Ports module ${__i} (${__target})" cd $${PORTSDIR:-/usr/ports}/${__i}; ${PORTSMODULESENV} ${MAKE} -B ${__target:C/install/deinstall reinstall/:C/reinstall/deinstall reinstall/} .endfor .endfor .endif .ORDER: kernel-install modules-install beforebuild: .PHONY kernel-all: beforebuild .WAIT ${KERNEL_KO} ${KERNEL_EXTRA} kernel-cleandir: kernel-clean kernel-cleandepend kernel-clobber: find . -maxdepth 1 ! -type d ! -name version -delete kernel-obj: .if !defined(MODULES_WITH_WORLD) && !defined(NO_MODULES) && exists($S/modules) modules: modules-all .if !defined(NO_MODULES_OBJ) modules-all modules-depend: modules-obj .endif .endif .if !defined(DEBUG) FULLKERNEL= ${KERNEL_KO} .else FULLKERNEL= ${KERNEL_KO}.full ${KERNEL_KO}: ${FULLKERNEL} ${KERNEL_KO}.debug ${OBJCOPY} --strip-debug --add-gnu-debuglink=${KERNEL_KO}.debug \ ${FULLKERNEL} ${.TARGET} ${KERNEL_KO}.debug: ${FULLKERNEL} ${OBJCOPY} --only-keep-debug ${FULLKERNEL} ${.TARGET} install.debug reinstall.debug: gdbinit cd ${.CURDIR}; ${MAKE} ${.TARGET:R} # Install gdbinit files for kernel debugging. gdbinit: grep -v '# XXX' ${S}/../tools/debugscripts/dot.gdbinit | \ sed "s:MODPATH:${.OBJDIR}/modules:" > .gdbinit cp ${S}/../tools/debugscripts/gdbinit.kernel ${.CURDIR} .if exists(${S}/../tools/debugscripts/gdbinit.${MACHINE_CPUARCH}) cp ${S}/../tools/debugscripts/gdbinit.${MACHINE_CPUARCH} \ ${.CURDIR}/gdbinit.machine .endif .endif ${FULLKERNEL}: ${SYSTEM_DEP} vers.o @rm -f ${.TARGET} @echo linking ${.TARGET} ${SYSTEM_LD} .if !empty(MD_ROOT_SIZE_CONFIGURED) && defined(MFS_IMAGE) @sh ${S}/tools/embed_mfs.sh ${.TARGET} ${MFS_IMAGE} .endif .if ${MK_CTF} != "no" @echo ${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ... @${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ${SYSTEM_OBJS} vers.o .endif .if !defined(DEBUG) ${OBJCOPY} --strip-debug ${.TARGET} .endif ${SYSTEM_LD_TAIL} OBJS_DEPEND_GUESS+= assym.s vnode_if.h ${BEFORE_DEPEND:M*.h} \ ${MFILES:T:S/.m$/.h/} LNFILES= ${CFILES:T:S/.c$/.ln/} .for mfile in ${MFILES} # XXX the low quality .m.o rules gnerated by config are normally used # instead of the .m.c rules here. ${mfile:T:S/.m$/.c/}: ${mfile} ${AWK} -f $S/tools/makeobjops.awk ${mfile} -c ${mfile:T:S/.m$/.h/}: ${mfile} ${AWK} -f $S/tools/makeobjops.awk ${mfile} -h .endfor kernel-clean: rm -f *.o *.so *.pico *.ko *.s eddep errs \ ${FULLKERNEL} ${KERNEL_KO} ${KERNEL_KO}.debug \ linterrs tags vers.c \ vnode_if.c vnode_if.h vnode_if_newproto.h vnode_if_typedef.h \ ${MFILES:T:S/.m$/.c/} ${MFILES:T:S/.m$/.h/} \ ${CLEAN} lint: ${LNFILES} ${LINT} ${LINTKERNFLAGS} ${CFLAGS:M-[DILU]*} ${.ALLSRC} 2>&1 | \ tee -a linterrs # This is a hack. BFD "optimizes" away dynamic mode if there are no # dynamic references. We could probably do a '-Bforcedynamic' mode like # in the a.out ld. For now, this works. HACK_EXTRA_FLAGS?= -shared hack.pico: Makefile :> hack.c ${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico rm -f hack.c assym.s: $S/kern/genassym.sh genassym.o NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET} genassym.o: $S/$M/$M/genassym.c ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c ${SYSTEM_OBJS} genassym.o vers.o: opt_global.h .if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon) _meta_filemon= 1 .endif # Skip reading .depend when not needed to speed up tree-walks and simple # lookups. For install, only do this if no other targets are specified. # Also skip generating or including .depend.* files if in meta+filemon mode # since it will track dependencies itself. OBJS_DEPEND_GUESS is still used # for _meta_filemon but not for _SKIP_DEPEND. -.if !empty(.MAKEFLAGS:M-V${_V_READ_DEPEND}) || make(*obj) || \ +.if !defined(NO_SKIP_DEPEND) && (make(*obj) || \ ${.TARGETS:M*clean*} == ${.TARGETS} || \ - ${.TARGETS:M*install*} == ${.TARGETS} + ${.TARGETS:M*install*} == ${.TARGETS}) _SKIP_DEPEND= 1 .endif .if defined(_SKIP_DEPEND) || defined(_meta_filemon) .MAKE.DEPENDFILE= /dev/null .endif kernel-depend: .depend SRCS= assym.s vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ ${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \ ${MFILES:T:S/.m$/.h/} DEPENDFILES= .depend .depend.* DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o DEPENDFILES_OBJS= ${DEPENDOBJS:O:u:C/^/.depend./} .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP .endif .if defined(_SKIP_DEPEND) # Don't bother reading any .meta files ${DEPENDOBJS}: .NOMETA .depend: .NOMETA # Unset these to avoid looping/statting on them later. .undef DEPENDSRCS .undef DEPENDOBJS .undef DEPENDFILES_OBJS .endif # defined(_SKIP_DEPEND) DEPEND_CFLAGS+= -MD ${DEPEND_MP} -MF.depend.${.TARGET} DEPEND_CFLAGS+= -MT${.TARGET} .if !defined(_meta_filemon) .if !empty(DEPEND_CFLAGS) # Only add in DEPEND_CFLAGS for CFLAGS on files we expect from DEPENDOBJS # as those are the only ones we will include. DEPEND_CFLAGS_CONDITION= "${DEPENDOBJS:M${.TARGET}}" != "" CFLAGS+= ${${DEPEND_CFLAGS_CONDITION}:?${DEPEND_CFLAGS}:} .endif .for __depend_obj in ${DEPENDFILES_OBJS} .if ${MAKE_VERSION} < 20160220 .sinclude "${.OBJDIR}/${__depend_obj}" .else .dinclude "${.OBJDIR}/${__depend_obj}" .endif .endfor .endif # !defined(_meta_filemon) # Always run 'make depend' to generate dependencies early and to avoid the # need for manually running it. For the kernel this is mostly a NOP since # all dependencies are correctly added or accounted for. This is mostly to # ensure downstream uses of kernel-depend are handled. beforebuild: kernel-depend # Guess some dependencies for when no ${DEPENDFILE}.OBJ is generated yet. # For meta+filemon the .meta file is checked for since it is the dependency # file used. .for __obj in ${DEPENDOBJS:O:u} .if defined(_meta_filemon) _depfile= ${.OBJDIR}/${__obj}.meta .else _depfile= ${.OBJDIR}/.depend.${__obj} .endif .if !exists(${_depfile}) .if ${SYSTEM_OBJS:M${__obj}} ${__obj}: ${OBJS_DEPEND_GUESS} .endif ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .elif defined(_meta_filemon) # For meta mode we still need to know which file to depend on to avoid # ambiguous suffix transformation rules from .PATH. Meta mode does not # use .depend files. We really only need source files, not headers since # they are typically in SRCS/beforebuild already. For target-specific # guesses do include headers though since they may not be in SRCS. .if ${SYSTEM_OBJS:M${__obj}} ${__obj}: ${OBJS_DEPEND_GUESS:N*.h} .endif ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .endif # !exists(${_depfile}) .endfor .NOPATH: .depend ${DEPENDFILES_OBJS} .depend: .PRECIOUS ${SRCS} _ILINKS= machine .if ${MACHINE} != ${MACHINE_CPUARCH} && ${MACHINE} != "arm64" _ILINKS+= ${MACHINE_CPUARCH} .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _ILINKS+= x86 .endif # Ensure that the link exists without depending on it when it exists. .for _link in ${_ILINKS} .if !exists(${.OBJDIR}/${_link}) ${SRCS} ${CLEAN:M*.o}: ${_link} .endif .endfor ${_ILINKS}: @case ${.TARGET} in \ machine) \ path=${S}/${MACHINE}/include ;; \ *) \ path=${S}/${.TARGET}/include ;; \ esac ; \ ${ECHO} ${.TARGET} "->" $$path ; \ ln -fhs $$path ${.TARGET} # .depend needs include links so we remove them only together. kernel-cleandepend: .PHONY rm -f ${DEPENDFILES} ${_ILINKS} kernel-tags: @[ -f .depend ] || { echo "you must make depend first"; exit 1; } sh $S/conf/systags.sh kernel-install: .PHONY @if [ ! -f ${KERNEL_KO} ] ; then \ echo "You must build a kernel first." ; \ exit 1 ; \ fi .if exists(${DESTDIR}${KODIR}) -thiskernel=`sysctl -n kern.bootfile` ; \ if [ ! "`dirname "$$thiskernel"`" -ef ${DESTDIR}${KODIR} ] ; then \ chflags -R noschg ${DESTDIR}${KODIR} ; \ rm -rf ${DESTDIR}${KODIR} ; \ rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ; \ else \ if [ -d ${DESTDIR}${KODIR}.old ] ; then \ chflags -R noschg ${DESTDIR}${KODIR}.old ; \ rm -rf ${DESTDIR}${KODIR}.old ; \ fi ; \ mv ${DESTDIR}${KODIR} ${DESTDIR}${KODIR}.old ; \ if [ -n "${KERN_DEBUGDIR}" -a \ -d ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ]; then \ rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \ mv ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \ fi ; \ sysctl kern.bootfile=${DESTDIR}${KODIR}.old/"`basename "$$thiskernel"`" ; \ fi .endif mkdir -p ${DESTDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/ .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ .endif .if defined(KERNEL_EXTRA_INSTALL) ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/ .endif kernel-reinstall: @-chflags -R noschg ${DESTDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/ .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ .endif config.o env.o hints.o vers.o vnode_if.o: ${NORMAL_C} ${NORMAL_CTFCONVERT} config.ln env.ln hints.ln vers.ln vnode_if.ln: ${NORMAL_LINT} .if ${MK_REPRODUCIBLE_BUILD} != "no" REPRO_FLAG="-r" .endif vers.c: $S/conf/newvers.sh $S/sys/param.h ${SYSTEM_DEP} MAKE=${MAKE} sh $S/conf/newvers.sh ${REPRO_FLAG} ${KERN_IDENT} vnode_if.c: $S/tools/vnode_if.awk $S/kern/vnode_if.src ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -c vnode_if.h vnode_if_newproto.h vnode_if_typedef.h: $S/tools/vnode_if.awk \ $S/kern/vnode_if.src vnode_if.h: vnode_if_newproto.h vnode_if_typedef.h ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -h vnode_if_newproto.h: ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -p vnode_if_typedef.h: ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -q .if ${MFS_IMAGE:Uno} != "no" .if empty(MD_ROOT_SIZE_CONFIGURED) # Generate an object file from the file system image to embed in the kernel # via linking. Make sure the contents are in the mfs section and rename the # start/end/size variables to __start_mfs, __stop_mfs, and mfs_size, # respectively. embedfs_${MFS_IMAGE:T:R}.o: ${MFS_IMAGE} ${OBJCOPY} --input-target binary \ --output-target ${EMBEDFS_FORMAT.${MACHINE_ARCH}} \ --binary-architecture ${EMBEDFS_ARCH.${MACHINE_ARCH}} \ ${MFS_IMAGE} ${.TARGET} ${OBJCOPY} \ --rename-section .data=mfs,contents,alloc,load,readonly,data \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_size=__mfs_root_size \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_start=mfs_root \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_end=mfs_root_end \ ${.TARGET} .endif .endif # XXX strictly, everything depends on Makefile because changes to ${PROF} # only appear there, but we don't handle that. .include "kern.mk" Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi32_types.h =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi32_types.h (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi32_types.h (revision 321585) @@ -1,232 +1,230 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #ifndef CLOUDABI32_TYPES_H #define CLOUDABI32_TYPES_H #include "cloudabi_types_common.h" typedef struct { _Alignas(4) cloudabi_auxtype_t a_type; union { _Alignas(4) uint32_t a_val; _Alignas(4) uint32_t a_ptr; }; } cloudabi32_auxv_t; _Static_assert(offsetof(cloudabi32_auxv_t, a_type) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_auxv_t, a_val) == 4, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_auxv_t, a_ptr) == 4, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_auxv_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_auxv_t) == 4, "Incorrect layout"); typedef struct { _Alignas(4) uint32_t buf; _Alignas(4) uint32_t buf_len; } cloudabi32_ciovec_t; _Static_assert(offsetof(cloudabi32_ciovec_t, buf) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_ciovec_t, buf_len) == 4, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_ciovec_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_ciovec_t) == 4, "Incorrect layout"); typedef struct { _Alignas(8) cloudabi_userdata_t userdata; _Alignas(2) cloudabi_errno_t error; _Alignas(1) cloudabi_eventtype_t type; union { struct { _Alignas(8) cloudabi_userdata_t identifier; } clock; struct { _Alignas(4) uint32_t condvar; } condvar; struct { _Alignas(8) cloudabi_filesize_t nbytes; _Alignas(4) cloudabi_fd_t fd; _Alignas(2) cloudabi_eventrwflags_t flags; } fd_readwrite; struct { _Alignas(4) uint32_t lock; } lock; struct { _Alignas(4) cloudabi_fd_t fd; _Alignas(1) cloudabi_signal_t signal; _Alignas(4) cloudabi_exitcode_t exitcode; } proc_terminate; }; } cloudabi32_event_t; _Static_assert(offsetof(cloudabi32_event_t, userdata) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, error) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, type) == 10, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, clock.identifier) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, condvar.condvar) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, fd_readwrite.nbytes) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, fd_readwrite.fd) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, fd_readwrite.flags) == 28, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, lock.lock) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, proc_terminate.fd) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, proc_terminate.signal) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_event_t, proc_terminate.exitcode) == 24, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_event_t) == 32, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_event_t) == 8, "Incorrect layout"); typedef struct { _Alignas(4) uint32_t buf; _Alignas(4) uint32_t buf_len; } cloudabi32_iovec_t; _Static_assert(offsetof(cloudabi32_iovec_t, buf) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_iovec_t, buf_len) == 4, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_iovec_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_iovec_t) == 4, "Incorrect layout"); typedef void cloudabi32_processentry_t(uint32_t auxv); typedef struct { _Alignas(4) uint32_t ri_data; _Alignas(4) uint32_t ri_data_len; _Alignas(4) uint32_t ri_fds; _Alignas(4) uint32_t ri_fds_len; - _Alignas(2) cloudabi_msgflags_t ri_flags; + _Alignas(2) cloudabi_riflags_t ri_flags; } cloudabi32_recv_in_t; _Static_assert(offsetof(cloudabi32_recv_in_t, ri_data) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_recv_in_t, ri_data_len) == 4, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_recv_in_t, ri_fds) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_recv_in_t, ri_fds_len) == 12, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_recv_in_t, ri_flags) == 16, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_recv_in_t) == 20, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_recv_in_t) == 4, "Incorrect layout"); typedef struct { + _Alignas(4) uint32_t ro_datalen; + _Alignas(4) uint32_t ro_fdslen; + _Alignas(1) char ro_unused[40]; + _Alignas(2) cloudabi_roflags_t ro_flags; +} cloudabi32_recv_out_t; +_Static_assert(offsetof(cloudabi32_recv_out_t, ro_datalen) == 0, "Incorrect layout"); +_Static_assert(offsetof(cloudabi32_recv_out_t, ro_fdslen) == 4, "Incorrect layout"); +_Static_assert(offsetof(cloudabi32_recv_out_t, ro_unused) == 8, "Incorrect layout"); +_Static_assert(offsetof(cloudabi32_recv_out_t, ro_flags) == 48, "Incorrect layout"); +_Static_assert(sizeof(cloudabi32_recv_out_t) == 52, "Incorrect layout"); +_Static_assert(_Alignof(cloudabi32_recv_out_t) == 4, "Incorrect layout"); + +typedef struct { _Alignas(4) uint32_t si_data; _Alignas(4) uint32_t si_data_len; _Alignas(4) uint32_t si_fds; _Alignas(4) uint32_t si_fds_len; - _Alignas(2) cloudabi_msgflags_t si_flags; + _Alignas(2) cloudabi_siflags_t si_flags; } cloudabi32_send_in_t; _Static_assert(offsetof(cloudabi32_send_in_t, si_data) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_send_in_t, si_data_len) == 4, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_send_in_t, si_fds) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_send_in_t, si_fds_len) == 12, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_send_in_t, si_flags) == 16, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_send_in_t) == 20, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_send_in_t) == 4, "Incorrect layout"); typedef struct { _Alignas(4) uint32_t so_datalen; } cloudabi32_send_out_t; _Static_assert(offsetof(cloudabi32_send_out_t, so_datalen) == 0, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_send_out_t) == 4, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_send_out_t) == 4, "Incorrect layout"); typedef struct { _Alignas(8) cloudabi_userdata_t userdata; _Alignas(2) cloudabi_subflags_t flags; _Alignas(1) cloudabi_eventtype_t type; union { struct { _Alignas(8) cloudabi_userdata_t identifier; _Alignas(4) cloudabi_clockid_t clock_id; _Alignas(8) cloudabi_timestamp_t timeout; _Alignas(8) cloudabi_timestamp_t precision; _Alignas(2) cloudabi_subclockflags_t flags; } clock; struct { _Alignas(4) uint32_t condvar; _Alignas(4) uint32_t lock; _Alignas(1) cloudabi_scope_t condvar_scope; _Alignas(1) cloudabi_scope_t lock_scope; } condvar; struct { _Alignas(4) cloudabi_fd_t fd; _Alignas(2) cloudabi_subrwflags_t flags; } fd_readwrite; struct { _Alignas(4) uint32_t lock; _Alignas(1) cloudabi_scope_t lock_scope; } lock; struct { _Alignas(4) cloudabi_fd_t fd; } proc_terminate; }; } cloudabi32_subscription_t; _Static_assert(offsetof(cloudabi32_subscription_t, userdata) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, flags) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, type) == 10, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, clock.identifier) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, clock.clock_id) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, clock.timeout) == 32, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, clock.precision) == 40, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, clock.flags) == 48, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, condvar.condvar) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, condvar.lock) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, condvar.condvar_scope) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, condvar.lock_scope) == 25, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, fd_readwrite.fd) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, fd_readwrite.flags) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, lock.lock) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, lock.lock_scope) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_subscription_t, proc_terminate.fd) == 16, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_subscription_t) == 56, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_subscription_t) == 8, "Incorrect layout"); typedef struct { _Alignas(4) uint32_t parent; } cloudabi32_tcb_t; _Static_assert(offsetof(cloudabi32_tcb_t, parent) == 0, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_tcb_t) == 4, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_tcb_t) == 4, "Incorrect layout"); typedef void cloudabi32_threadentry_t(cloudabi_tid_t tid, uint32_t aux); - -typedef struct { - _Alignas(4) uint32_t ro_datalen; - _Alignas(4) uint32_t ro_fdslen; - _Alignas(2) cloudabi_sockaddr_t ro_sockname; - _Alignas(2) cloudabi_sockaddr_t ro_peername; - _Alignas(2) cloudabi_msgflags_t ro_flags; -} cloudabi32_recv_out_t; -_Static_assert(offsetof(cloudabi32_recv_out_t, ro_datalen) == 0, "Incorrect layout"); -_Static_assert(offsetof(cloudabi32_recv_out_t, ro_fdslen) == 4, "Incorrect layout"); -_Static_assert(offsetof(cloudabi32_recv_out_t, ro_sockname) == 8, "Incorrect layout"); -_Static_assert(offsetof(cloudabi32_recv_out_t, ro_peername) == 28, "Incorrect layout"); -_Static_assert(offsetof(cloudabi32_recv_out_t, ro_flags) == 48, "Incorrect layout"); -_Static_assert(sizeof(cloudabi32_recv_out_t) == 52, "Incorrect layout"); -_Static_assert(_Alignof(cloudabi32_recv_out_t) == 4, "Incorrect layout"); typedef struct { _Alignas(4) uint32_t entry_point; _Alignas(4) uint32_t stack; _Alignas(4) uint32_t stack_len; _Alignas(4) uint32_t argument; } cloudabi32_threadattr_t; _Static_assert(offsetof(cloudabi32_threadattr_t, entry_point) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_threadattr_t, stack) == 4, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_threadattr_t, stack_len) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi32_threadattr_t, argument) == 12, "Incorrect layout"); _Static_assert(sizeof(cloudabi32_threadattr_t) == 16, "Incorrect layout"); _Static_assert(_Alignof(cloudabi32_threadattr_t) == 4, "Incorrect layout"); #endif Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi64_types.h =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi64_types.h (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi64_types.h (revision 321585) @@ -1,232 +1,230 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #ifndef CLOUDABI64_TYPES_H #define CLOUDABI64_TYPES_H #include "cloudabi_types_common.h" typedef struct { _Alignas(4) cloudabi_auxtype_t a_type; union { _Alignas(8) uint64_t a_val; _Alignas(8) uint64_t a_ptr; }; } cloudabi64_auxv_t; _Static_assert(offsetof(cloudabi64_auxv_t, a_type) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_auxv_t, a_val) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_auxv_t, a_ptr) == 8, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_auxv_t) == 16, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_auxv_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) uint64_t buf; _Alignas(8) uint64_t buf_len; } cloudabi64_ciovec_t; _Static_assert(offsetof(cloudabi64_ciovec_t, buf) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_ciovec_t, buf_len) == 8, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_ciovec_t) == 16, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_ciovec_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) cloudabi_userdata_t userdata; _Alignas(2) cloudabi_errno_t error; _Alignas(1) cloudabi_eventtype_t type; union { struct { _Alignas(8) cloudabi_userdata_t identifier; } clock; struct { _Alignas(8) uint64_t condvar; } condvar; struct { _Alignas(8) cloudabi_filesize_t nbytes; _Alignas(4) cloudabi_fd_t fd; _Alignas(2) cloudabi_eventrwflags_t flags; } fd_readwrite; struct { _Alignas(8) uint64_t lock; } lock; struct { _Alignas(4) cloudabi_fd_t fd; _Alignas(1) cloudabi_signal_t signal; _Alignas(4) cloudabi_exitcode_t exitcode; } proc_terminate; }; } cloudabi64_event_t; _Static_assert(offsetof(cloudabi64_event_t, userdata) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, error) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, type) == 10, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, clock.identifier) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, condvar.condvar) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, fd_readwrite.nbytes) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, fd_readwrite.fd) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, fd_readwrite.flags) == 28, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, lock.lock) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, proc_terminate.fd) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, proc_terminate.signal) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_event_t, proc_terminate.exitcode) == 24, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_event_t) == 32, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_event_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) uint64_t buf; _Alignas(8) uint64_t buf_len; } cloudabi64_iovec_t; _Static_assert(offsetof(cloudabi64_iovec_t, buf) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_iovec_t, buf_len) == 8, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_iovec_t) == 16, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_iovec_t) == 8, "Incorrect layout"); typedef void cloudabi64_processentry_t(uint64_t auxv); typedef struct { _Alignas(8) uint64_t ri_data; _Alignas(8) uint64_t ri_data_len; _Alignas(8) uint64_t ri_fds; _Alignas(8) uint64_t ri_fds_len; - _Alignas(2) cloudabi_msgflags_t ri_flags; + _Alignas(2) cloudabi_riflags_t ri_flags; } cloudabi64_recv_in_t; _Static_assert(offsetof(cloudabi64_recv_in_t, ri_data) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_recv_in_t, ri_data_len) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_recv_in_t, ri_fds) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_recv_in_t, ri_fds_len) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_recv_in_t, ri_flags) == 32, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_recv_in_t) == 40, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_recv_in_t) == 8, "Incorrect layout"); typedef struct { + _Alignas(8) uint64_t ro_datalen; + _Alignas(8) uint64_t ro_fdslen; + _Alignas(1) char ro_unused[40]; + _Alignas(2) cloudabi_roflags_t ro_flags; +} cloudabi64_recv_out_t; +_Static_assert(offsetof(cloudabi64_recv_out_t, ro_datalen) == 0, "Incorrect layout"); +_Static_assert(offsetof(cloudabi64_recv_out_t, ro_fdslen) == 8, "Incorrect layout"); +_Static_assert(offsetof(cloudabi64_recv_out_t, ro_unused) == 16, "Incorrect layout"); +_Static_assert(offsetof(cloudabi64_recv_out_t, ro_flags) == 56, "Incorrect layout"); +_Static_assert(sizeof(cloudabi64_recv_out_t) == 64, "Incorrect layout"); +_Static_assert(_Alignof(cloudabi64_recv_out_t) == 8, "Incorrect layout"); + +typedef struct { _Alignas(8) uint64_t si_data; _Alignas(8) uint64_t si_data_len; _Alignas(8) uint64_t si_fds; _Alignas(8) uint64_t si_fds_len; - _Alignas(2) cloudabi_msgflags_t si_flags; + _Alignas(2) cloudabi_siflags_t si_flags; } cloudabi64_send_in_t; _Static_assert(offsetof(cloudabi64_send_in_t, si_data) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_send_in_t, si_data_len) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_send_in_t, si_fds) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_send_in_t, si_fds_len) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_send_in_t, si_flags) == 32, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_send_in_t) == 40, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_send_in_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) uint64_t so_datalen; } cloudabi64_send_out_t; _Static_assert(offsetof(cloudabi64_send_out_t, so_datalen) == 0, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_send_out_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_send_out_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) cloudabi_userdata_t userdata; _Alignas(2) cloudabi_subflags_t flags; _Alignas(1) cloudabi_eventtype_t type; union { struct { _Alignas(8) cloudabi_userdata_t identifier; _Alignas(4) cloudabi_clockid_t clock_id; _Alignas(8) cloudabi_timestamp_t timeout; _Alignas(8) cloudabi_timestamp_t precision; _Alignas(2) cloudabi_subclockflags_t flags; } clock; struct { _Alignas(8) uint64_t condvar; _Alignas(8) uint64_t lock; _Alignas(1) cloudabi_scope_t condvar_scope; _Alignas(1) cloudabi_scope_t lock_scope; } condvar; struct { _Alignas(4) cloudabi_fd_t fd; _Alignas(2) cloudabi_subrwflags_t flags; } fd_readwrite; struct { _Alignas(8) uint64_t lock; _Alignas(1) cloudabi_scope_t lock_scope; } lock; struct { _Alignas(4) cloudabi_fd_t fd; } proc_terminate; }; } cloudabi64_subscription_t; _Static_assert(offsetof(cloudabi64_subscription_t, userdata) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, flags) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, type) == 10, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, clock.identifier) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, clock.clock_id) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, clock.timeout) == 32, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, clock.precision) == 40, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, clock.flags) == 48, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, condvar.condvar) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, condvar.lock) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, condvar.condvar_scope) == 32, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, condvar.lock_scope) == 33, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, fd_readwrite.fd) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, fd_readwrite.flags) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, lock.lock) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, lock.lock_scope) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_subscription_t, proc_terminate.fd) == 16, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_subscription_t) == 56, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_subscription_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) uint64_t parent; } cloudabi64_tcb_t; _Static_assert(offsetof(cloudabi64_tcb_t, parent) == 0, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_tcb_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_tcb_t) == 8, "Incorrect layout"); typedef void cloudabi64_threadentry_t(cloudabi_tid_t tid, uint64_t aux); - -typedef struct { - _Alignas(8) uint64_t ro_datalen; - _Alignas(8) uint64_t ro_fdslen; - _Alignas(2) cloudabi_sockaddr_t ro_sockname; - _Alignas(2) cloudabi_sockaddr_t ro_peername; - _Alignas(2) cloudabi_msgflags_t ro_flags; -} cloudabi64_recv_out_t; -_Static_assert(offsetof(cloudabi64_recv_out_t, ro_datalen) == 0, "Incorrect layout"); -_Static_assert(offsetof(cloudabi64_recv_out_t, ro_fdslen) == 8, "Incorrect layout"); -_Static_assert(offsetof(cloudabi64_recv_out_t, ro_sockname) == 16, "Incorrect layout"); -_Static_assert(offsetof(cloudabi64_recv_out_t, ro_peername) == 36, "Incorrect layout"); -_Static_assert(offsetof(cloudabi64_recv_out_t, ro_flags) == 56, "Incorrect layout"); -_Static_assert(sizeof(cloudabi64_recv_out_t) == 64, "Incorrect layout"); -_Static_assert(_Alignof(cloudabi64_recv_out_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) uint64_t entry_point; _Alignas(8) uint64_t stack; _Alignas(8) uint64_t stack_len; _Alignas(8) uint64_t argument; } cloudabi64_threadattr_t; _Static_assert(offsetof(cloudabi64_threadattr_t, entry_point) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_threadattr_t, stack) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_threadattr_t, stack_len) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi64_threadattr_t, argument) == 24, "Incorrect layout"); _Static_assert(sizeof(cloudabi64_threadattr_t) == 32, "Incorrect layout"); _Static_assert(_Alignof(cloudabi64_threadattr_t) == 8, "Incorrect layout"); #endif Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_types_common.h =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_types_common.h (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_types_common.h (revision 321585) @@ -1,464 +1,437 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #ifndef CLOUDABI_TYPES_COMMON_H #define CLOUDABI_TYPES_COMMON_H #if defined(__FreeBSD__) && defined(_KERNEL) #include #elif defined(__linux__) && defined(__KERNEL__) #include #else #include #include #endif typedef uint8_t cloudabi_advice_t; #define CLOUDABI_ADVICE_DONTNEED 1 #define CLOUDABI_ADVICE_NOREUSE 2 #define CLOUDABI_ADVICE_NORMAL 3 #define CLOUDABI_ADVICE_RANDOM 4 #define CLOUDABI_ADVICE_SEQUENTIAL 5 #define CLOUDABI_ADVICE_WILLNEED 6 typedef uint32_t cloudabi_auxtype_t; #define CLOUDABI_AT_ARGDATA 256 #define CLOUDABI_AT_ARGDATALEN 257 #define CLOUDABI_AT_BASE 7 #define CLOUDABI_AT_CANARY 258 #define CLOUDABI_AT_CANARYLEN 259 #define CLOUDABI_AT_NCPUS 260 #define CLOUDABI_AT_NULL 0 #define CLOUDABI_AT_PAGESZ 6 #define CLOUDABI_AT_PHDR 3 #define CLOUDABI_AT_PHNUM 4 #define CLOUDABI_AT_SYSINFO_EHDR 262 #define CLOUDABI_AT_TID 261 typedef uint32_t cloudabi_backlog_t; typedef uint32_t cloudabi_clockid_t; #define CLOUDABI_CLOCK_MONOTONIC 1 #define CLOUDABI_CLOCK_PROCESS_CPUTIME_ID 2 #define CLOUDABI_CLOCK_REALTIME 3 #define CLOUDABI_CLOCK_THREAD_CPUTIME_ID 4 typedef uint32_t cloudabi_condvar_t; #define CLOUDABI_CONDVAR_HAS_NO_WAITERS 0 typedef uint64_t cloudabi_device_t; typedef uint64_t cloudabi_dircookie_t; #define CLOUDABI_DIRCOOKIE_START 0 typedef uint16_t cloudabi_errno_t; #define CLOUDABI_E2BIG 1 #define CLOUDABI_EACCES 2 #define CLOUDABI_EADDRINUSE 3 #define CLOUDABI_EADDRNOTAVAIL 4 #define CLOUDABI_EAFNOSUPPORT 5 #define CLOUDABI_EAGAIN 6 #define CLOUDABI_EALREADY 7 #define CLOUDABI_EBADF 8 #define CLOUDABI_EBADMSG 9 #define CLOUDABI_EBUSY 10 #define CLOUDABI_ECANCELED 11 #define CLOUDABI_ECHILD 12 #define CLOUDABI_ECONNABORTED 13 #define CLOUDABI_ECONNREFUSED 14 #define CLOUDABI_ECONNRESET 15 #define CLOUDABI_EDEADLK 16 #define CLOUDABI_EDESTADDRREQ 17 #define CLOUDABI_EDOM 18 #define CLOUDABI_EDQUOT 19 #define CLOUDABI_EEXIST 20 #define CLOUDABI_EFAULT 21 #define CLOUDABI_EFBIG 22 #define CLOUDABI_EHOSTUNREACH 23 #define CLOUDABI_EIDRM 24 #define CLOUDABI_EILSEQ 25 #define CLOUDABI_EINPROGRESS 26 #define CLOUDABI_EINTR 27 #define CLOUDABI_EINVAL 28 #define CLOUDABI_EIO 29 #define CLOUDABI_EISCONN 30 #define CLOUDABI_EISDIR 31 #define CLOUDABI_ELOOP 32 #define CLOUDABI_EMFILE 33 #define CLOUDABI_EMLINK 34 #define CLOUDABI_EMSGSIZE 35 #define CLOUDABI_EMULTIHOP 36 #define CLOUDABI_ENAMETOOLONG 37 #define CLOUDABI_ENETDOWN 38 #define CLOUDABI_ENETRESET 39 #define CLOUDABI_ENETUNREACH 40 #define CLOUDABI_ENFILE 41 #define CLOUDABI_ENOBUFS 42 #define CLOUDABI_ENODEV 43 #define CLOUDABI_ENOENT 44 #define CLOUDABI_ENOEXEC 45 #define CLOUDABI_ENOLCK 46 #define CLOUDABI_ENOLINK 47 #define CLOUDABI_ENOMEM 48 #define CLOUDABI_ENOMSG 49 #define CLOUDABI_ENOPROTOOPT 50 #define CLOUDABI_ENOSPC 51 #define CLOUDABI_ENOSYS 52 #define CLOUDABI_ENOTCONN 53 #define CLOUDABI_ENOTDIR 54 #define CLOUDABI_ENOTEMPTY 55 #define CLOUDABI_ENOTRECOVERABLE 56 #define CLOUDABI_ENOTSOCK 57 #define CLOUDABI_ENOTSUP 58 #define CLOUDABI_ENOTTY 59 #define CLOUDABI_ENXIO 60 #define CLOUDABI_EOVERFLOW 61 #define CLOUDABI_EOWNERDEAD 62 #define CLOUDABI_EPERM 63 #define CLOUDABI_EPIPE 64 #define CLOUDABI_EPROTO 65 #define CLOUDABI_EPROTONOSUPPORT 66 #define CLOUDABI_EPROTOTYPE 67 #define CLOUDABI_ERANGE 68 #define CLOUDABI_EROFS 69 #define CLOUDABI_ESPIPE 70 #define CLOUDABI_ESRCH 71 #define CLOUDABI_ESTALE 72 #define CLOUDABI_ETIMEDOUT 73 #define CLOUDABI_ETXTBSY 74 #define CLOUDABI_EXDEV 75 #define CLOUDABI_ENOTCAPABLE 76 typedef uint16_t cloudabi_eventrwflags_t; #define CLOUDABI_EVENT_FD_READWRITE_HANGUP 0x0001 typedef uint8_t cloudabi_eventtype_t; #define CLOUDABI_EVENTTYPE_CLOCK 1 #define CLOUDABI_EVENTTYPE_CONDVAR 2 #define CLOUDABI_EVENTTYPE_FD_READ 3 #define CLOUDABI_EVENTTYPE_FD_WRITE 4 #define CLOUDABI_EVENTTYPE_LOCK_RDLOCK 5 #define CLOUDABI_EVENTTYPE_LOCK_WRLOCK 6 #define CLOUDABI_EVENTTYPE_PROC_TERMINATE 7 typedef uint32_t cloudabi_exitcode_t; typedef uint32_t cloudabi_fd_t; #define CLOUDABI_PROCESS_CHILD 0xffffffff #define CLOUDABI_MAP_ANON_FD 0xffffffff typedef uint16_t cloudabi_fdflags_t; #define CLOUDABI_FDFLAG_APPEND 0x0001 #define CLOUDABI_FDFLAG_DSYNC 0x0002 #define CLOUDABI_FDFLAG_NONBLOCK 0x0004 #define CLOUDABI_FDFLAG_RSYNC 0x0008 #define CLOUDABI_FDFLAG_SYNC 0x0010 typedef uint16_t cloudabi_fdsflags_t; #define CLOUDABI_FDSTAT_FLAGS 0x0001 #define CLOUDABI_FDSTAT_RIGHTS 0x0002 typedef int64_t cloudabi_filedelta_t; typedef uint64_t cloudabi_filesize_t; typedef uint8_t cloudabi_filetype_t; #define CLOUDABI_FILETYPE_UNKNOWN 0 #define CLOUDABI_FILETYPE_BLOCK_DEVICE 16 #define CLOUDABI_FILETYPE_CHARACTER_DEVICE 17 #define CLOUDABI_FILETYPE_DIRECTORY 32 #define CLOUDABI_FILETYPE_FIFO 48 #define CLOUDABI_FILETYPE_POLL 64 #define CLOUDABI_FILETYPE_PROCESS 80 #define CLOUDABI_FILETYPE_REGULAR_FILE 96 #define CLOUDABI_FILETYPE_SHARED_MEMORY 112 #define CLOUDABI_FILETYPE_SOCKET_DGRAM 128 -#define CLOUDABI_FILETYPE_SOCKET_SEQPACKET 129 #define CLOUDABI_FILETYPE_SOCKET_STREAM 130 #define CLOUDABI_FILETYPE_SYMBOLIC_LINK 144 typedef uint16_t cloudabi_fsflags_t; #define CLOUDABI_FILESTAT_ATIM 0x0001 #define CLOUDABI_FILESTAT_ATIM_NOW 0x0002 #define CLOUDABI_FILESTAT_MTIM 0x0004 #define CLOUDABI_FILESTAT_MTIM_NOW 0x0008 #define CLOUDABI_FILESTAT_SIZE 0x0010 typedef uint64_t cloudabi_inode_t; typedef uint32_t cloudabi_linkcount_t; typedef uint32_t cloudabi_lock_t; #define CLOUDABI_LOCK_UNLOCKED 0x00000000 #define CLOUDABI_LOCK_WRLOCKED 0x40000000 #define CLOUDABI_LOCK_KERNEL_MANAGED 0x80000000 #define CLOUDABI_LOCK_BOGUS 0x80000000 typedef uint32_t cloudabi_lookupflags_t; #define CLOUDABI_LOOKUP_SYMLINK_FOLLOW 0x00000001 typedef uint8_t cloudabi_mflags_t; #define CLOUDABI_MAP_ANON 0x01 #define CLOUDABI_MAP_FIXED 0x02 #define CLOUDABI_MAP_PRIVATE 0x04 #define CLOUDABI_MAP_SHARED 0x08 typedef uint8_t cloudabi_mprot_t; #define CLOUDABI_PROT_EXEC 0x01 #define CLOUDABI_PROT_WRITE 0x02 #define CLOUDABI_PROT_READ 0x04 typedef uint8_t cloudabi_msflags_t; #define CLOUDABI_MS_ASYNC 0x01 #define CLOUDABI_MS_INVALIDATE 0x02 #define CLOUDABI_MS_SYNC 0x04 -typedef uint16_t cloudabi_msgflags_t; -#define CLOUDABI_MSG_CTRUNC 0x0001 -#define CLOUDABI_MSG_EOR 0x0002 -#define CLOUDABI_MSG_PEEK 0x0004 -#define CLOUDABI_MSG_TRUNC 0x0008 -#define CLOUDABI_MSG_WAITALL 0x0010 - typedef uint32_t cloudabi_nthreads_t; typedef uint16_t cloudabi_oflags_t; #define CLOUDABI_O_CREAT 0x0001 #define CLOUDABI_O_DIRECTORY 0x0002 #define CLOUDABI_O_EXCL 0x0004 #define CLOUDABI_O_TRUNC 0x0008 +typedef uint16_t cloudabi_riflags_t; +#define CLOUDABI_SOCK_RECV_PEEK 0x0004 +#define CLOUDABI_SOCK_RECV_WAITALL 0x0010 + typedef uint64_t cloudabi_rights_t; #define CLOUDABI_RIGHT_FD_DATASYNC 0x0000000000000001 #define CLOUDABI_RIGHT_FD_READ 0x0000000000000002 #define CLOUDABI_RIGHT_FD_SEEK 0x0000000000000004 #define CLOUDABI_RIGHT_FD_STAT_PUT_FLAGS 0x0000000000000008 #define CLOUDABI_RIGHT_FD_SYNC 0x0000000000000010 #define CLOUDABI_RIGHT_FD_TELL 0x0000000000000020 #define CLOUDABI_RIGHT_FD_WRITE 0x0000000000000040 #define CLOUDABI_RIGHT_FILE_ADVISE 0x0000000000000080 #define CLOUDABI_RIGHT_FILE_ALLOCATE 0x0000000000000100 #define CLOUDABI_RIGHT_FILE_CREATE_DIRECTORY 0x0000000000000200 #define CLOUDABI_RIGHT_FILE_CREATE_FILE 0x0000000000000400 #define CLOUDABI_RIGHT_FILE_CREATE_FIFO 0x0000000000000800 #define CLOUDABI_RIGHT_FILE_LINK_SOURCE 0x0000000000001000 #define CLOUDABI_RIGHT_FILE_LINK_TARGET 0x0000000000002000 #define CLOUDABI_RIGHT_FILE_OPEN 0x0000000000004000 #define CLOUDABI_RIGHT_FILE_READDIR 0x0000000000008000 #define CLOUDABI_RIGHT_FILE_READLINK 0x0000000000010000 #define CLOUDABI_RIGHT_FILE_RENAME_SOURCE 0x0000000000020000 #define CLOUDABI_RIGHT_FILE_RENAME_TARGET 0x0000000000040000 #define CLOUDABI_RIGHT_FILE_STAT_FGET 0x0000000000080000 #define CLOUDABI_RIGHT_FILE_STAT_FPUT_SIZE 0x0000000000100000 #define CLOUDABI_RIGHT_FILE_STAT_FPUT_TIMES 0x0000000000200000 #define CLOUDABI_RIGHT_FILE_STAT_GET 0x0000000000400000 #define CLOUDABI_RIGHT_FILE_STAT_PUT_TIMES 0x0000000000800000 #define CLOUDABI_RIGHT_FILE_SYMLINK 0x0000000001000000 #define CLOUDABI_RIGHT_FILE_UNLINK 0x0000000002000000 #define CLOUDABI_RIGHT_MEM_MAP 0x0000000004000000 #define CLOUDABI_RIGHT_MEM_MAP_EXEC 0x0000000008000000 #define CLOUDABI_RIGHT_POLL_FD_READWRITE 0x0000000010000000 #define CLOUDABI_RIGHT_POLL_MODIFY 0x0000000020000000 #define CLOUDABI_RIGHT_POLL_PROC_TERMINATE 0x0000000040000000 #define CLOUDABI_RIGHT_POLL_WAIT 0x0000000080000000 #define CLOUDABI_RIGHT_PROC_EXEC 0x0000000100000000 #define CLOUDABI_RIGHT_SOCK_ACCEPT 0x0000000200000000 #define CLOUDABI_RIGHT_SOCK_BIND_DIRECTORY 0x0000000400000000 #define CLOUDABI_RIGHT_SOCK_BIND_SOCKET 0x0000000800000000 #define CLOUDABI_RIGHT_SOCK_CONNECT_DIRECTORY 0x0000001000000000 #define CLOUDABI_RIGHT_SOCK_CONNECT_SOCKET 0x0000002000000000 #define CLOUDABI_RIGHT_SOCK_LISTEN 0x0000004000000000 #define CLOUDABI_RIGHT_SOCK_SHUTDOWN 0x0000008000000000 #define CLOUDABI_RIGHT_SOCK_STAT_GET 0x0000010000000000 -typedef uint8_t cloudabi_sa_family_t; -#define CLOUDABI_AF_UNSPEC 0 -#define CLOUDABI_AF_INET 1 -#define CLOUDABI_AF_INET6 2 -#define CLOUDABI_AF_UNIX 3 +typedef uint16_t cloudabi_roflags_t; +#define CLOUDABI_SOCK_RECV_FDS_TRUNCATED 0x0001 +#define CLOUDABI_SOCK_RECV_DATA_TRUNCATED 0x0008 typedef uint8_t cloudabi_scope_t; #define CLOUDABI_SCOPE_PRIVATE 4 #define CLOUDABI_SCOPE_SHARED 8 typedef uint8_t cloudabi_sdflags_t; #define CLOUDABI_SHUT_RD 0x01 #define CLOUDABI_SHUT_WR 0x02 +typedef uint16_t cloudabi_siflags_t; + typedef uint8_t cloudabi_signal_t; #define CLOUDABI_SIGABRT 1 #define CLOUDABI_SIGALRM 2 #define CLOUDABI_SIGBUS 3 #define CLOUDABI_SIGCHLD 4 #define CLOUDABI_SIGCONT 5 #define CLOUDABI_SIGFPE 6 #define CLOUDABI_SIGHUP 7 #define CLOUDABI_SIGILL 8 #define CLOUDABI_SIGINT 9 #define CLOUDABI_SIGKILL 10 #define CLOUDABI_SIGPIPE 11 #define CLOUDABI_SIGQUIT 12 #define CLOUDABI_SIGSEGV 13 #define CLOUDABI_SIGSTOP 14 #define CLOUDABI_SIGSYS 15 #define CLOUDABI_SIGTERM 16 #define CLOUDABI_SIGTRAP 17 #define CLOUDABI_SIGTSTP 18 #define CLOUDABI_SIGTTIN 19 #define CLOUDABI_SIGTTOU 20 #define CLOUDABI_SIGURG 21 #define CLOUDABI_SIGUSR1 22 #define CLOUDABI_SIGUSR2 23 #define CLOUDABI_SIGVTALRM 24 #define CLOUDABI_SIGXCPU 25 #define CLOUDABI_SIGXFSZ 26 typedef uint8_t cloudabi_ssflags_t; #define CLOUDABI_SOCKSTAT_CLEAR_ERROR 0x01 typedef uint32_t cloudabi_sstate_t; #define CLOUDABI_SOCKSTATE_ACCEPTCONN 0x00000001 typedef uint16_t cloudabi_subclockflags_t; #define CLOUDABI_SUBSCRIPTION_CLOCK_ABSTIME 0x0001 typedef uint16_t cloudabi_subflags_t; #define CLOUDABI_SUBSCRIPTION_ADD 0x0001 #define CLOUDABI_SUBSCRIPTION_CLEAR 0x0002 #define CLOUDABI_SUBSCRIPTION_DELETE 0x0004 #define CLOUDABI_SUBSCRIPTION_DISABLE 0x0008 #define CLOUDABI_SUBSCRIPTION_ENABLE 0x0010 #define CLOUDABI_SUBSCRIPTION_ONESHOT 0x0020 typedef uint16_t cloudabi_subrwflags_t; #define CLOUDABI_SUBSCRIPTION_FD_READWRITE_POLL 0x0001 typedef uint32_t cloudabi_tid_t; typedef uint64_t cloudabi_timestamp_t; typedef uint8_t cloudabi_ulflags_t; #define CLOUDABI_UNLINK_REMOVEDIR 0x01 typedef uint64_t cloudabi_userdata_t; typedef uint8_t cloudabi_whence_t; #define CLOUDABI_WHENCE_CUR 1 #define CLOUDABI_WHENCE_END 2 #define CLOUDABI_WHENCE_SET 3 typedef struct { _Alignas(8) cloudabi_dircookie_t d_next; _Alignas(8) cloudabi_inode_t d_ino; _Alignas(4) uint32_t d_namlen; _Alignas(1) cloudabi_filetype_t d_type; } cloudabi_dirent_t; _Static_assert(offsetof(cloudabi_dirent_t, d_next) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi_dirent_t, d_ino) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi_dirent_t, d_namlen) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi_dirent_t, d_type) == 20, "Incorrect layout"); _Static_assert(sizeof(cloudabi_dirent_t) == 24, "Incorrect layout"); _Static_assert(_Alignof(cloudabi_dirent_t) == 8, "Incorrect layout"); typedef struct { _Alignas(1) cloudabi_filetype_t fs_filetype; _Alignas(2) cloudabi_fdflags_t fs_flags; _Alignas(8) cloudabi_rights_t fs_rights_base; _Alignas(8) cloudabi_rights_t fs_rights_inheriting; } cloudabi_fdstat_t; _Static_assert(offsetof(cloudabi_fdstat_t, fs_filetype) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi_fdstat_t, fs_flags) == 2, "Incorrect layout"); _Static_assert(offsetof(cloudabi_fdstat_t, fs_rights_base) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi_fdstat_t, fs_rights_inheriting) == 16, "Incorrect layout"); _Static_assert(sizeof(cloudabi_fdstat_t) == 24, "Incorrect layout"); _Static_assert(_Alignof(cloudabi_fdstat_t) == 8, "Incorrect layout"); typedef struct { _Alignas(8) cloudabi_device_t st_dev; _Alignas(8) cloudabi_inode_t st_ino; _Alignas(1) cloudabi_filetype_t st_filetype; _Alignas(4) cloudabi_linkcount_t st_nlink; _Alignas(8) cloudabi_filesize_t st_size; _Alignas(8) cloudabi_timestamp_t st_atim; _Alignas(8) cloudabi_timestamp_t st_mtim; _Alignas(8) cloudabi_timestamp_t st_ctim; } cloudabi_filestat_t; _Static_assert(offsetof(cloudabi_filestat_t, st_dev) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_ino) == 8, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_filetype) == 16, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_nlink) == 20, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_size) == 24, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_atim) == 32, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_mtim) == 40, "Incorrect layout"); _Static_assert(offsetof(cloudabi_filestat_t, st_ctim) == 48, "Incorrect layout"); _Static_assert(sizeof(cloudabi_filestat_t) == 56, "Incorrect layout"); _Static_assert(_Alignof(cloudabi_filestat_t) == 8, "Incorrect layout"); typedef struct { _Alignas(4) cloudabi_fd_t fd; _Alignas(4) cloudabi_lookupflags_t flags; } cloudabi_lookup_t; _Static_assert(offsetof(cloudabi_lookup_t, fd) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi_lookup_t, flags) == 4, "Incorrect layout"); _Static_assert(sizeof(cloudabi_lookup_t) == 8, "Incorrect layout"); _Static_assert(_Alignof(cloudabi_lookup_t) == 4, "Incorrect layout"); typedef struct { - _Alignas(1) cloudabi_sa_family_t sa_family; - union { - struct { - _Alignas(1) uint8_t addr[4]; - _Alignas(2) uint16_t port; - } sa_inet; - struct { - _Alignas(1) uint8_t addr[16]; - _Alignas(2) uint16_t port; - } sa_inet6; - }; -} cloudabi_sockaddr_t; -_Static_assert(offsetof(cloudabi_sockaddr_t, sa_family) == 0, "Incorrect layout"); -_Static_assert(offsetof(cloudabi_sockaddr_t, sa_inet.addr) == 2, "Incorrect layout"); -_Static_assert(offsetof(cloudabi_sockaddr_t, sa_inet.port) == 6, "Incorrect layout"); -_Static_assert(offsetof(cloudabi_sockaddr_t, sa_inet6.addr) == 2, "Incorrect layout"); -_Static_assert(offsetof(cloudabi_sockaddr_t, sa_inet6.port) == 18, "Incorrect layout"); -_Static_assert(sizeof(cloudabi_sockaddr_t) == 20, "Incorrect layout"); -_Static_assert(_Alignof(cloudabi_sockaddr_t) == 2, "Incorrect layout"); - -typedef struct { - _Alignas(2) cloudabi_sockaddr_t ss_sockname; - _Alignas(2) cloudabi_sockaddr_t ss_peername; + _Alignas(1) char ss_unused[40]; _Alignas(2) cloudabi_errno_t ss_error; _Alignas(4) cloudabi_sstate_t ss_state; } cloudabi_sockstat_t; -_Static_assert(offsetof(cloudabi_sockstat_t, ss_sockname) == 0, "Incorrect layout"); -_Static_assert(offsetof(cloudabi_sockstat_t, ss_peername) == 20, "Incorrect layout"); +_Static_assert(offsetof(cloudabi_sockstat_t, ss_unused) == 0, "Incorrect layout"); _Static_assert(offsetof(cloudabi_sockstat_t, ss_error) == 40, "Incorrect layout"); _Static_assert(offsetof(cloudabi_sockstat_t, ss_state) == 44, "Incorrect layout"); _Static_assert(sizeof(cloudabi_sockstat_t) == 48, "Incorrect layout"); _Static_assert(_Alignof(cloudabi_sockstat_t) == 4, "Incorrect layout"); #endif Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_aarch64.S =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_aarch64.S (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_aarch64.S (revision 321585) @@ -1,491 +1,479 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #define ENTRY(name) \ .text; \ .p2align 2; \ .global name; \ .type name, @function; \ name: #define END(name) .size name, . - name ENTRY(cloudabi_sys_clock_res_get) str x1, [sp, #-8] mov w8, #0 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_clock_res_get) ENTRY(cloudabi_sys_clock_time_get) str x2, [sp, #-8] mov w8, #1 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_clock_time_get) ENTRY(cloudabi_sys_condvar_signal) mov w8, #2 svc #0 ret END(cloudabi_sys_condvar_signal) ENTRY(cloudabi_sys_fd_close) mov w8, #3 svc #0 ret END(cloudabi_sys_fd_close) ENTRY(cloudabi_sys_fd_create1) str x1, [sp, #-8] mov w8, #4 svc #0 ldr x2, [sp, #-8] b.cs 1f str w0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_create1) ENTRY(cloudabi_sys_fd_create2) stp x1, x2, [sp, #-16] mov w8, #5 svc #0 ldp x2, x3, [sp, #-16] b.cs 1f str w0, [x2] str w1, [x3] mov w0, wzr 1: ret END(cloudabi_sys_fd_create2) ENTRY(cloudabi_sys_fd_datasync) mov w8, #6 svc #0 ret END(cloudabi_sys_fd_datasync) ENTRY(cloudabi_sys_fd_dup) str x1, [sp, #-8] mov w8, #7 svc #0 ldr x2, [sp, #-8] b.cs 1f str w0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_dup) ENTRY(cloudabi_sys_fd_pread) str x4, [sp, #-8] mov w8, #8 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_pread) ENTRY(cloudabi_sys_fd_pwrite) str x4, [sp, #-8] mov w8, #9 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_pwrite) ENTRY(cloudabi_sys_fd_read) str x3, [sp, #-8] mov w8, #10 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_read) ENTRY(cloudabi_sys_fd_replace) mov w8, #11 svc #0 ret END(cloudabi_sys_fd_replace) ENTRY(cloudabi_sys_fd_seek) str x3, [sp, #-8] mov w8, #12 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_seek) ENTRY(cloudabi_sys_fd_stat_get) mov w8, #13 svc #0 ret END(cloudabi_sys_fd_stat_get) ENTRY(cloudabi_sys_fd_stat_put) mov w8, #14 svc #0 ret END(cloudabi_sys_fd_stat_put) ENTRY(cloudabi_sys_fd_sync) mov w8, #15 svc #0 ret END(cloudabi_sys_fd_sync) ENTRY(cloudabi_sys_fd_write) str x3, [sp, #-8] mov w8, #16 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_fd_write) ENTRY(cloudabi_sys_file_advise) mov w8, #17 svc #0 ret END(cloudabi_sys_file_advise) ENTRY(cloudabi_sys_file_allocate) mov w8, #18 svc #0 ret END(cloudabi_sys_file_allocate) ENTRY(cloudabi_sys_file_create) mov w8, #19 svc #0 ret END(cloudabi_sys_file_create) ENTRY(cloudabi_sys_file_link) mov w8, #20 svc #0 ret END(cloudabi_sys_file_link) ENTRY(cloudabi_sys_file_open) str x5, [sp, #-8] mov w8, #21 svc #0 ldr x2, [sp, #-8] b.cs 1f str w0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_file_open) ENTRY(cloudabi_sys_file_readdir) str x4, [sp, #-8] mov w8, #22 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_file_readdir) ENTRY(cloudabi_sys_file_readlink) str x5, [sp, #-8] mov w8, #23 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_file_readlink) ENTRY(cloudabi_sys_file_rename) mov w8, #24 svc #0 ret END(cloudabi_sys_file_rename) ENTRY(cloudabi_sys_file_stat_fget) mov w8, #25 svc #0 ret END(cloudabi_sys_file_stat_fget) ENTRY(cloudabi_sys_file_stat_fput) mov w8, #26 svc #0 ret END(cloudabi_sys_file_stat_fput) ENTRY(cloudabi_sys_file_stat_get) mov w8, #27 svc #0 ret END(cloudabi_sys_file_stat_get) ENTRY(cloudabi_sys_file_stat_put) mov w8, #28 svc #0 ret END(cloudabi_sys_file_stat_put) ENTRY(cloudabi_sys_file_symlink) mov w8, #29 svc #0 ret END(cloudabi_sys_file_symlink) ENTRY(cloudabi_sys_file_unlink) mov w8, #30 svc #0 ret END(cloudabi_sys_file_unlink) ENTRY(cloudabi_sys_lock_unlock) mov w8, #31 svc #0 ret END(cloudabi_sys_lock_unlock) ENTRY(cloudabi_sys_mem_advise) mov w8, #32 svc #0 ret END(cloudabi_sys_mem_advise) -ENTRY(cloudabi_sys_mem_lock) - mov w8, #33 - svc #0 - ret -END(cloudabi_sys_mem_lock) - ENTRY(cloudabi_sys_mem_map) str x6, [sp, #-8] - mov w8, #34 + mov w8, #33 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_mem_map) ENTRY(cloudabi_sys_mem_protect) - mov w8, #35 + mov w8, #34 svc #0 ret END(cloudabi_sys_mem_protect) ENTRY(cloudabi_sys_mem_sync) - mov w8, #36 + mov w8, #35 svc #0 ret END(cloudabi_sys_mem_sync) -ENTRY(cloudabi_sys_mem_unlock) - mov w8, #37 - svc #0 - ret -END(cloudabi_sys_mem_unlock) - ENTRY(cloudabi_sys_mem_unmap) - mov w8, #38 + mov w8, #36 svc #0 ret END(cloudabi_sys_mem_unmap) ENTRY(cloudabi_sys_poll) str x3, [sp, #-8] - mov w8, #39 + mov w8, #37 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_poll) ENTRY(cloudabi_sys_poll_fd) str x6, [sp, #-8] - mov w8, #40 + mov w8, #38 svc #0 ldr x2, [sp, #-8] b.cs 1f str x0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_poll_fd) ENTRY(cloudabi_sys_proc_exec) - mov w8, #41 + mov w8, #39 svc #0 ret END(cloudabi_sys_proc_exec) ENTRY(cloudabi_sys_proc_exit) - mov w8, #42 + mov w8, #40 svc #0 END(cloudabi_sys_proc_exit) ENTRY(cloudabi_sys_proc_fork) stp x0, x1, [sp, #-16] - mov w8, #43 + mov w8, #41 svc #0 ldp x2, x3, [sp, #-16] b.cs 1f str w0, [x2] str w1, [x3] mov w0, wzr 1: ret END(cloudabi_sys_proc_fork) ENTRY(cloudabi_sys_proc_raise) - mov w8, #44 + mov w8, #42 svc #0 ret END(cloudabi_sys_proc_raise) ENTRY(cloudabi_sys_random_get) - mov w8, #45 + mov w8, #43 svc #0 ret END(cloudabi_sys_random_get) ENTRY(cloudabi_sys_sock_accept) str x2, [sp, #-8] - mov w8, #46 + mov w8, #44 svc #0 ldr x2, [sp, #-8] b.cs 1f str w0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_sock_accept) ENTRY(cloudabi_sys_sock_bind) - mov w8, #47 + mov w8, #45 svc #0 ret END(cloudabi_sys_sock_bind) ENTRY(cloudabi_sys_sock_connect) - mov w8, #48 + mov w8, #46 svc #0 ret END(cloudabi_sys_sock_connect) ENTRY(cloudabi_sys_sock_listen) - mov w8, #49 + mov w8, #47 svc #0 ret END(cloudabi_sys_sock_listen) ENTRY(cloudabi_sys_sock_recv) - mov w8, #50 + mov w8, #48 svc #0 ret END(cloudabi_sys_sock_recv) ENTRY(cloudabi_sys_sock_send) - mov w8, #51 + mov w8, #49 svc #0 ret END(cloudabi_sys_sock_send) ENTRY(cloudabi_sys_sock_shutdown) - mov w8, #52 + mov w8, #50 svc #0 ret END(cloudabi_sys_sock_shutdown) ENTRY(cloudabi_sys_sock_stat_get) - mov w8, #53 + mov w8, #51 svc #0 ret END(cloudabi_sys_sock_stat_get) ENTRY(cloudabi_sys_thread_create) str x1, [sp, #-8] - mov w8, #54 + mov w8, #52 svc #0 ldr x2, [sp, #-8] b.cs 1f str w0, [x2] mov w0, wzr 1: ret END(cloudabi_sys_thread_create) ENTRY(cloudabi_sys_thread_exit) - mov w8, #55 + mov w8, #53 svc #0 END(cloudabi_sys_thread_exit) ENTRY(cloudabi_sys_thread_yield) - mov w8, #56 + mov w8, #54 svc #0 ret END(cloudabi_sys_thread_yield) Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_armv6.S =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_armv6.S (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_armv6.S (revision 321585) @@ -1,451 +1,439 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #define ENTRY(name) \ .text; \ .p2align 2; \ .global name; \ .type name, %function; \ name: #define END(name) .size name, . - name ENTRY(cloudabi_sys_clock_res_get) str r1, [sp, #-4] mov ip, #0 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2, 0] strcc r1, [r2, 4] movcc r0, $0 bx lr END(cloudabi_sys_clock_res_get) ENTRY(cloudabi_sys_clock_time_get) mov ip, #1 swi 0 ldrcc r2, [sp, #0] strcc r0, [r2, 0] strcc r1, [r2, 4] movcc r0, $0 bx lr END(cloudabi_sys_clock_time_get) ENTRY(cloudabi_sys_condvar_signal) mov ip, #2 swi 0 bx lr END(cloudabi_sys_condvar_signal) ENTRY(cloudabi_sys_fd_close) mov ip, #3 swi 0 bx lr END(cloudabi_sys_fd_close) ENTRY(cloudabi_sys_fd_create1) str r1, [sp, #-4] mov ip, #4 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_create1) ENTRY(cloudabi_sys_fd_create2) str r1, [sp, #-4] str r2, [sp, #-8] mov ip, #5 swi 0 ldrcc r2, [sp, #-4] ldrcc r3, [sp, #-8] strcc r0, [r2] strcc r1, [r3] movcc r0, $0 bx lr END(cloudabi_sys_fd_create2) ENTRY(cloudabi_sys_fd_datasync) mov ip, #6 swi 0 bx lr END(cloudabi_sys_fd_datasync) ENTRY(cloudabi_sys_fd_dup) str r1, [sp, #-4] mov ip, #7 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_dup) ENTRY(cloudabi_sys_fd_pread) mov ip, #8 swi 0 ldrcc r2, [sp, #8] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_pread) ENTRY(cloudabi_sys_fd_pwrite) mov ip, #9 swi 0 ldrcc r2, [sp, #8] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_pwrite) ENTRY(cloudabi_sys_fd_read) str r3, [sp, #-4] mov ip, #10 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_read) ENTRY(cloudabi_sys_fd_replace) mov ip, #11 swi 0 bx lr END(cloudabi_sys_fd_replace) ENTRY(cloudabi_sys_fd_seek) mov ip, #12 swi 0 ldrcc r2, [sp, #4] strcc r0, [r2, 0] strcc r1, [r2, 4] movcc r0, $0 bx lr END(cloudabi_sys_fd_seek) ENTRY(cloudabi_sys_fd_stat_get) mov ip, #13 swi 0 bx lr END(cloudabi_sys_fd_stat_get) ENTRY(cloudabi_sys_fd_stat_put) mov ip, #14 swi 0 bx lr END(cloudabi_sys_fd_stat_put) ENTRY(cloudabi_sys_fd_sync) mov ip, #15 swi 0 bx lr END(cloudabi_sys_fd_sync) ENTRY(cloudabi_sys_fd_write) str r3, [sp, #-4] mov ip, #16 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_fd_write) ENTRY(cloudabi_sys_file_advise) mov ip, #17 swi 0 bx lr END(cloudabi_sys_file_advise) ENTRY(cloudabi_sys_file_allocate) mov ip, #18 swi 0 bx lr END(cloudabi_sys_file_allocate) ENTRY(cloudabi_sys_file_create) mov ip, #19 swi 0 bx lr END(cloudabi_sys_file_create) ENTRY(cloudabi_sys_file_link) mov ip, #20 swi 0 bx lr END(cloudabi_sys_file_link) ENTRY(cloudabi_sys_file_open) mov ip, #21 swi 0 ldrcc r2, [sp, #8] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_file_open) ENTRY(cloudabi_sys_file_readdir) mov ip, #22 swi 0 ldrcc r2, [sp, #8] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_file_readdir) ENTRY(cloudabi_sys_file_readlink) mov ip, #23 swi 0 ldrcc r2, [sp, #4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_file_readlink) ENTRY(cloudabi_sys_file_rename) mov ip, #24 swi 0 bx lr END(cloudabi_sys_file_rename) ENTRY(cloudabi_sys_file_stat_fget) mov ip, #25 swi 0 bx lr END(cloudabi_sys_file_stat_fget) ENTRY(cloudabi_sys_file_stat_fput) mov ip, #26 swi 0 bx lr END(cloudabi_sys_file_stat_fput) ENTRY(cloudabi_sys_file_stat_get) mov ip, #27 swi 0 bx lr END(cloudabi_sys_file_stat_get) ENTRY(cloudabi_sys_file_stat_put) mov ip, #28 swi 0 bx lr END(cloudabi_sys_file_stat_put) ENTRY(cloudabi_sys_file_symlink) mov ip, #29 swi 0 bx lr END(cloudabi_sys_file_symlink) ENTRY(cloudabi_sys_file_unlink) mov ip, #30 swi 0 bx lr END(cloudabi_sys_file_unlink) ENTRY(cloudabi_sys_lock_unlock) mov ip, #31 swi 0 bx lr END(cloudabi_sys_lock_unlock) ENTRY(cloudabi_sys_mem_advise) mov ip, #32 swi 0 bx lr END(cloudabi_sys_mem_advise) -ENTRY(cloudabi_sys_mem_lock) +ENTRY(cloudabi_sys_mem_map) mov ip, #33 swi 0 - bx lr -END(cloudabi_sys_mem_lock) - -ENTRY(cloudabi_sys_mem_map) - mov ip, #34 - swi 0 ldrcc r2, [sp, #16] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_mem_map) ENTRY(cloudabi_sys_mem_protect) - mov ip, #35 + mov ip, #34 swi 0 bx lr END(cloudabi_sys_mem_protect) ENTRY(cloudabi_sys_mem_sync) - mov ip, #36 + mov ip, #35 swi 0 bx lr END(cloudabi_sys_mem_sync) -ENTRY(cloudabi_sys_mem_unlock) - mov ip, #37 - swi 0 - bx lr -END(cloudabi_sys_mem_unlock) - ENTRY(cloudabi_sys_mem_unmap) - mov ip, #38 + mov ip, #36 swi 0 bx lr END(cloudabi_sys_mem_unmap) ENTRY(cloudabi_sys_poll) str r3, [sp, #-4] - mov ip, #39 + mov ip, #37 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_poll) ENTRY(cloudabi_sys_poll_fd) - mov ip, #40 + mov ip, #38 swi 0 ldrcc r2, [sp, #8] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_poll_fd) ENTRY(cloudabi_sys_proc_exec) - mov ip, #41 + mov ip, #39 swi 0 bx lr END(cloudabi_sys_proc_exec) ENTRY(cloudabi_sys_proc_exit) - mov ip, #42 + mov ip, #40 swi 0 END(cloudabi_sys_proc_exit) ENTRY(cloudabi_sys_proc_fork) str r0, [sp, #-4] str r1, [sp, #-8] - mov ip, #43 + mov ip, #41 swi 0 ldrcc r2, [sp, #-4] ldrcc r3, [sp, #-8] strcc r0, [r2] strcc r1, [r3] movcc r0, $0 bx lr END(cloudabi_sys_proc_fork) ENTRY(cloudabi_sys_proc_raise) - mov ip, #44 + mov ip, #42 swi 0 bx lr END(cloudabi_sys_proc_raise) ENTRY(cloudabi_sys_random_get) - mov ip, #45 + mov ip, #43 swi 0 bx lr END(cloudabi_sys_random_get) ENTRY(cloudabi_sys_sock_accept) str r2, [sp, #-4] - mov ip, #46 + mov ip, #44 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_sock_accept) ENTRY(cloudabi_sys_sock_bind) - mov ip, #47 + mov ip, #45 swi 0 bx lr END(cloudabi_sys_sock_bind) ENTRY(cloudabi_sys_sock_connect) - mov ip, #48 + mov ip, #46 swi 0 bx lr END(cloudabi_sys_sock_connect) ENTRY(cloudabi_sys_sock_listen) - mov ip, #49 + mov ip, #47 swi 0 bx lr END(cloudabi_sys_sock_listen) ENTRY(cloudabi_sys_sock_recv) - mov ip, #50 + mov ip, #48 swi 0 bx lr END(cloudabi_sys_sock_recv) ENTRY(cloudabi_sys_sock_send) - mov ip, #51 + mov ip, #49 swi 0 bx lr END(cloudabi_sys_sock_send) ENTRY(cloudabi_sys_sock_shutdown) - mov ip, #52 + mov ip, #50 swi 0 bx lr END(cloudabi_sys_sock_shutdown) ENTRY(cloudabi_sys_sock_stat_get) - mov ip, #53 + mov ip, #51 swi 0 bx lr END(cloudabi_sys_sock_stat_get) ENTRY(cloudabi_sys_thread_create) str r1, [sp, #-4] - mov ip, #54 + mov ip, #52 swi 0 ldrcc r2, [sp, #-4] strcc r0, [r2] movcc r0, $0 bx lr END(cloudabi_sys_thread_create) ENTRY(cloudabi_sys_thread_exit) - mov ip, #55 + mov ip, #53 swi 0 END(cloudabi_sys_thread_exit) ENTRY(cloudabi_sys_thread_yield) - mov ip, #56 + mov ip, #54 swi 0 bx lr END(cloudabi_sys_thread_yield) Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686.S =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686.S (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686.S (revision 321585) @@ -1,477 +1,465 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #define ENTRY(name) \ .text; \ .p2align 2, 0x90; \ .global name; \ .type name, @function; \ name: #define END(name) .size name, . - name ENTRY(cloudabi_sys_clock_res_get) mov $0, %eax int $0x80 jc 1f mov 8(%esp), %ecx mov %eax, 0(%ecx) mov %edx, 4(%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_clock_res_get) ENTRY(cloudabi_sys_clock_time_get) mov $1, %eax int $0x80 jc 1f mov 16(%esp), %ecx mov %eax, 0(%ecx) mov %edx, 4(%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_clock_time_get) ENTRY(cloudabi_sys_condvar_signal) mov $2, %eax int $0x80 ret END(cloudabi_sys_condvar_signal) ENTRY(cloudabi_sys_fd_close) mov $3, %eax int $0x80 ret END(cloudabi_sys_fd_close) ENTRY(cloudabi_sys_fd_create1) mov $4, %eax int $0x80 jc 1f mov 8(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_create1) ENTRY(cloudabi_sys_fd_create2) mov $5, %eax int $0x80 jc 1f mov 8(%esp), %ecx mov %eax, (%ecx) mov 12(%esp), %ecx mov %edx, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_create2) ENTRY(cloudabi_sys_fd_datasync) mov $6, %eax int $0x80 ret END(cloudabi_sys_fd_datasync) ENTRY(cloudabi_sys_fd_dup) mov $7, %eax int $0x80 jc 1f mov 8(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_dup) ENTRY(cloudabi_sys_fd_pread) mov $8, %eax int $0x80 jc 1f mov 24(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_pread) ENTRY(cloudabi_sys_fd_pwrite) mov $9, %eax int $0x80 jc 1f mov 24(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_pwrite) ENTRY(cloudabi_sys_fd_read) mov $10, %eax int $0x80 jc 1f mov 16(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_read) ENTRY(cloudabi_sys_fd_replace) mov $11, %eax int $0x80 ret END(cloudabi_sys_fd_replace) ENTRY(cloudabi_sys_fd_seek) mov $12, %eax int $0x80 jc 1f mov 20(%esp), %ecx mov %eax, 0(%ecx) mov %edx, 4(%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_seek) ENTRY(cloudabi_sys_fd_stat_get) mov $13, %eax int $0x80 ret END(cloudabi_sys_fd_stat_get) ENTRY(cloudabi_sys_fd_stat_put) mov $14, %eax int $0x80 ret END(cloudabi_sys_fd_stat_put) ENTRY(cloudabi_sys_fd_sync) mov $15, %eax int $0x80 ret END(cloudabi_sys_fd_sync) ENTRY(cloudabi_sys_fd_write) mov $16, %eax int $0x80 jc 1f mov 16(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_write) ENTRY(cloudabi_sys_file_advise) mov $17, %eax int $0x80 ret END(cloudabi_sys_file_advise) ENTRY(cloudabi_sys_file_allocate) mov $18, %eax int $0x80 ret END(cloudabi_sys_file_allocate) ENTRY(cloudabi_sys_file_create) mov $19, %eax int $0x80 ret END(cloudabi_sys_file_create) ENTRY(cloudabi_sys_file_link) mov $20, %eax int $0x80 ret END(cloudabi_sys_file_link) ENTRY(cloudabi_sys_file_open) mov $21, %eax int $0x80 jc 1f mov 28(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_file_open) ENTRY(cloudabi_sys_file_readdir) mov $22, %eax int $0x80 jc 1f mov 24(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_file_readdir) ENTRY(cloudabi_sys_file_readlink) mov $23, %eax int $0x80 jc 1f mov 24(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_file_readlink) ENTRY(cloudabi_sys_file_rename) mov $24, %eax int $0x80 ret END(cloudabi_sys_file_rename) ENTRY(cloudabi_sys_file_stat_fget) mov $25, %eax int $0x80 ret END(cloudabi_sys_file_stat_fget) ENTRY(cloudabi_sys_file_stat_fput) mov $26, %eax int $0x80 ret END(cloudabi_sys_file_stat_fput) ENTRY(cloudabi_sys_file_stat_get) mov $27, %eax int $0x80 ret END(cloudabi_sys_file_stat_get) ENTRY(cloudabi_sys_file_stat_put) mov $28, %eax int $0x80 ret END(cloudabi_sys_file_stat_put) ENTRY(cloudabi_sys_file_symlink) mov $29, %eax int $0x80 ret END(cloudabi_sys_file_symlink) ENTRY(cloudabi_sys_file_unlink) mov $30, %eax int $0x80 ret END(cloudabi_sys_file_unlink) ENTRY(cloudabi_sys_lock_unlock) mov $31, %eax int $0x80 ret END(cloudabi_sys_lock_unlock) ENTRY(cloudabi_sys_mem_advise) mov $32, %eax int $0x80 ret END(cloudabi_sys_mem_advise) -ENTRY(cloudabi_sys_mem_lock) +ENTRY(cloudabi_sys_mem_map) mov $33, %eax int $0x80 - ret -END(cloudabi_sys_mem_lock) - -ENTRY(cloudabi_sys_mem_map) - mov $34, %eax - int $0x80 jc 1f mov 32(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_mem_map) ENTRY(cloudabi_sys_mem_protect) - mov $35, %eax + mov $34, %eax int $0x80 ret END(cloudabi_sys_mem_protect) ENTRY(cloudabi_sys_mem_sync) - mov $36, %eax + mov $35, %eax int $0x80 ret END(cloudabi_sys_mem_sync) -ENTRY(cloudabi_sys_mem_unlock) - mov $37, %eax - int $0x80 - ret -END(cloudabi_sys_mem_unlock) - ENTRY(cloudabi_sys_mem_unmap) - mov $38, %eax + mov $36, %eax int $0x80 ret END(cloudabi_sys_mem_unmap) ENTRY(cloudabi_sys_poll) - mov $39, %eax + mov $37, %eax int $0x80 jc 1f mov 16(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_poll) ENTRY(cloudabi_sys_poll_fd) - mov $40, %eax + mov $38, %eax int $0x80 jc 1f mov 28(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_poll_fd) ENTRY(cloudabi_sys_proc_exec) - mov $41, %eax + mov $39, %eax int $0x80 ret END(cloudabi_sys_proc_exec) ENTRY(cloudabi_sys_proc_exit) - mov $42, %eax + mov $40, %eax int $0x80 END(cloudabi_sys_proc_exit) ENTRY(cloudabi_sys_proc_fork) - mov $43, %eax + mov $41, %eax int $0x80 jc 1f mov 4(%esp), %ecx mov %eax, (%ecx) mov 8(%esp), %ecx mov %edx, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_proc_fork) ENTRY(cloudabi_sys_proc_raise) - mov $44, %eax + mov $42, %eax int $0x80 ret END(cloudabi_sys_proc_raise) ENTRY(cloudabi_sys_random_get) - mov $45, %eax + mov $43, %eax int $0x80 ret END(cloudabi_sys_random_get) ENTRY(cloudabi_sys_sock_accept) - mov $46, %eax + mov $44, %eax int $0x80 jc 1f mov 12(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_sock_accept) ENTRY(cloudabi_sys_sock_bind) - mov $47, %eax + mov $45, %eax int $0x80 ret END(cloudabi_sys_sock_bind) ENTRY(cloudabi_sys_sock_connect) - mov $48, %eax + mov $46, %eax int $0x80 ret END(cloudabi_sys_sock_connect) ENTRY(cloudabi_sys_sock_listen) - mov $49, %eax + mov $47, %eax int $0x80 ret END(cloudabi_sys_sock_listen) ENTRY(cloudabi_sys_sock_recv) - mov $50, %eax + mov $48, %eax int $0x80 ret END(cloudabi_sys_sock_recv) ENTRY(cloudabi_sys_sock_send) - mov $51, %eax + mov $49, %eax int $0x80 ret END(cloudabi_sys_sock_send) ENTRY(cloudabi_sys_sock_shutdown) - mov $52, %eax + mov $50, %eax int $0x80 ret END(cloudabi_sys_sock_shutdown) ENTRY(cloudabi_sys_sock_stat_get) - mov $53, %eax + mov $51, %eax int $0x80 ret END(cloudabi_sys_sock_stat_get) ENTRY(cloudabi_sys_thread_create) - mov $54, %eax + mov $52, %eax int $0x80 jc 1f mov 8(%esp), %ecx mov %eax, (%ecx) xor %eax, %eax 1: ret END(cloudabi_sys_thread_create) ENTRY(cloudabi_sys_thread_exit) - mov $55, %eax + mov $53, %eax int $0x80 END(cloudabi_sys_thread_exit) ENTRY(cloudabi_sys_thread_yield) - mov $56, %eax + mov $54, %eax int $0x80 ret END(cloudabi_sys_thread_yield) Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S (revision 321585) @@ -1,1223 +1,1189 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #define ENTRY(name) \ .text; \ .p2align 2, 0x90; \ .global name; \ .type name, @function; \ name: #define END(name) .size name, . - name ENTRY(cloudabi_sys_clock_res_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $0, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 12(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) mov -12(%ebp), %edx mov %edx, 4(%ecx) 1: pop %ebp ret END(cloudabi_sys_clock_res_get) ENTRY(cloudabi_sys_clock_time_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) mov 16(%ebp), %ecx mov %ecx, -4(%ebp) mov $1, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 20(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) mov -12(%ebp), %edx mov %edx, 4(%ecx) 1: pop %ebp ret END(cloudabi_sys_clock_time_get) ENTRY(cloudabi_sys_condvar_signal) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) mov $2, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_condvar_signal) ENTRY(cloudabi_sys_fd_close) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $3, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_close) ENTRY(cloudabi_sys_fd_create1) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $4, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 12(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_create1) ENTRY(cloudabi_sys_fd_create2) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $5, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 12(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) mov 16(%ebp), %ecx mov -8(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_create2) ENTRY(cloudabi_sys_fd_datasync) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $6, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_datasync) ENTRY(cloudabi_sys_fd_dup) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $7, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 12(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_dup) ENTRY(cloudabi_sys_fd_pread) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov 24(%ebp), %ecx mov %ecx, -4(%ebp) mov $8, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 test %eax, %eax jnz 1f mov 28(%ebp), %ecx mov -32(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_pread) ENTRY(cloudabi_sys_fd_pwrite) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov 24(%ebp), %ecx mov %ecx, -4(%ebp) mov $9, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 test %eax, %eax jnz 1f mov 28(%ebp), %ecx mov -32(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_pwrite) ENTRY(cloudabi_sys_fd_read) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $10, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 test %eax, %eax jnz 1f mov 20(%ebp), %ecx mov -24(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_read) ENTRY(cloudabi_sys_fd_replace) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) mov $11, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_replace) ENTRY(cloudabi_sys_fd_seek) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) mov 16(%ebp), %ecx mov %ecx, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov $12, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 test %eax, %eax jnz 1f mov 24(%ebp), %ecx mov -24(%ebp), %edx mov %edx, 0(%ecx) mov -20(%ebp), %edx mov %edx, 4(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_seek) ENTRY(cloudabi_sys_fd_stat_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $13, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_stat_get) ENTRY(cloudabi_sys_fd_stat_put) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) mov $14, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_stat_put) ENTRY(cloudabi_sys_fd_sync) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov $15, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_fd_sync) ENTRY(cloudabi_sys_fd_write) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $16, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 test %eax, %eax jnz 1f mov 20(%ebp), %ecx mov -24(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_fd_write) ENTRY(cloudabi_sys_file_advise) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) mov 16(%ebp), %ecx mov %ecx, -20(%ebp) mov 20(%ebp), %ecx mov %ecx, -16(%ebp) mov 24(%ebp), %ecx mov %ecx, -12(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) mov $17, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_advise) ENTRY(cloudabi_sys_file_allocate) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) mov 16(%ebp), %ecx mov %ecx, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov 24(%ebp), %ecx mov %ecx, -4(%ebp) mov $18, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_allocate) ENTRY(cloudabi_sys_file_create) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov $19, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_create) ENTRY(cloudabi_sys_file_link) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -48(%ebp) mov 12(%ebp), %ecx mov %ecx, -44(%ebp) mov 16(%ebp), %ecx mov %ecx, -40(%ebp) movl $0, -36(%ebp) mov 20(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 24(%ebp), %ecx mov %ecx, -24(%ebp) mov 28(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 32(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $20, %eax mov %ebp, %ecx sub $48, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_link) ENTRY(cloudabi_sys_file_open) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -40(%ebp) mov 12(%ebp), %ecx mov %ecx, -36(%ebp) mov 16(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 20(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 24(%ebp), %ecx mov %ecx, -16(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $21, %eax mov %ebp, %ecx sub $40, %ecx int $0x80 test %eax, %eax jnz 1f mov 32(%ebp), %ecx mov -40(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_file_open) ENTRY(cloudabi_sys_file_readdir) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov 24(%ebp), %ecx mov %ecx, -4(%ebp) mov $22, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 test %eax, %eax jnz 1f mov 28(%ebp), %ecx mov -32(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_file_readdir) ENTRY(cloudabi_sys_file_readlink) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -40(%ebp) mov 12(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 16(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 20(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 24(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $23, %eax mov %ebp, %ecx sub $40, %ecx int $0x80 test %eax, %eax jnz 1f mov 28(%ebp), %ecx mov -40(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_file_readlink) ENTRY(cloudabi_sys_file_rename) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -48(%ebp) mov 12(%ebp), %ecx mov %ecx, -40(%ebp) movl $0, -36(%ebp) mov 16(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 20(%ebp), %ecx mov %ecx, -24(%ebp) mov 24(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $24, %eax mov %ebp, %ecx sub $48, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_rename) ENTRY(cloudabi_sys_file_stat_fget) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $25, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_stat_fget) ENTRY(cloudabi_sys_file_stat_fput) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) mov $26, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_stat_fput) ENTRY(cloudabi_sys_file_stat_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -28(%ebp) mov 16(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 20(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 24(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $27, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_stat_get) ENTRY(cloudabi_sys_file_stat_put) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -40(%ebp) mov 12(%ebp), %ecx mov %ecx, -36(%ebp) mov 16(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 20(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 24(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) mov $28, %eax mov %ebp, %ecx sub $40, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_stat_put) ENTRY(cloudabi_sys_file_symlink) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -40(%ebp) movl $0, -36(%ebp) mov 12(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 16(%ebp), %ecx mov %ecx, -24(%ebp) mov 20(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 24(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) mov $29, %eax mov %ebp, %ecx sub $40, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_symlink) ENTRY(cloudabi_sys_file_unlink) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) mov $30, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_file_unlink) ENTRY(cloudabi_sys_lock_unlock) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) mov $31, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_lock_unlock) ENTRY(cloudabi_sys_mem_advise) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) mov $32, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_mem_advise) -ENTRY(cloudabi_sys_mem_lock) - push %ebp - mov %esp, %ebp - mov 8(%ebp), %ecx - mov %ecx, -16(%ebp) - movl $0, -12(%ebp) - mov 12(%ebp), %ecx - mov %ecx, -8(%ebp) - movl $0, -4(%ebp) - mov $33, %eax - mov %ebp, %ecx - sub $16, %ecx - int $0x80 - pop %ebp - ret -END(cloudabi_sys_mem_lock) - ENTRY(cloudabi_sys_mem_map) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -48(%ebp) movl $0, -44(%ebp) mov 12(%ebp), %ecx mov %ecx, -40(%ebp) movl $0, -36(%ebp) mov 16(%ebp), %ecx mov %ecx, -32(%ebp) mov 20(%ebp), %ecx mov %ecx, -24(%ebp) mov 24(%ebp), %ecx mov %ecx, -16(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) mov 32(%ebp), %ecx mov %ecx, -4(%ebp) - mov $34, %eax + mov $33, %eax mov %ebp, %ecx sub $48, %ecx int $0x80 test %eax, %eax jnz 1f mov 36(%ebp), %ecx mov -48(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_mem_map) ENTRY(cloudabi_sys_mem_protect) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) - mov $35, %eax + mov $34, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_mem_protect) ENTRY(cloudabi_sys_mem_sync) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) - mov $36, %eax + mov $35, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_mem_sync) -ENTRY(cloudabi_sys_mem_unlock) - push %ebp - mov %esp, %ebp - mov 8(%ebp), %ecx - mov %ecx, -16(%ebp) - movl $0, -12(%ebp) - mov 12(%ebp), %ecx - mov %ecx, -8(%ebp) - movl $0, -4(%ebp) - mov $37, %eax - mov %ebp, %ecx - sub $16, %ecx - int $0x80 - pop %ebp - ret -END(cloudabi_sys_mem_unlock) - ENTRY(cloudabi_sys_mem_unmap) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $38, %eax + mov $36, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_mem_unmap) ENTRY(cloudabi_sys_poll) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $39, %eax + mov $37, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 test %eax, %eax jnz 1f mov 20(%ebp), %ecx mov -24(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_poll) ENTRY(cloudabi_sys_poll_fd) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -48(%ebp) mov 12(%ebp), %ecx mov %ecx, -40(%ebp) movl $0, -36(%ebp) mov 16(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 20(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 24(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 28(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $40, %eax + mov $38, %eax mov %ebp, %ecx sub $48, %ecx int $0x80 test %eax, %eax jnz 1f mov 32(%ebp), %ecx mov -48(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_poll_fd) ENTRY(cloudabi_sys_proc_exec) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -40(%ebp) mov 12(%ebp), %ecx mov %ecx, -32(%ebp) movl $0, -28(%ebp) mov 16(%ebp), %ecx mov %ecx, -24(%ebp) movl $0, -20(%ebp) mov 20(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 24(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $41, %eax + mov $39, %eax mov %ebp, %ecx sub $40, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_proc_exec) ENTRY(cloudabi_sys_proc_exit) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) - mov $42, %eax + mov $40, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 END(cloudabi_sys_proc_exit) ENTRY(cloudabi_sys_proc_fork) push %ebp mov %esp, %ebp - mov $43, %eax + mov $41, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 8(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) mov 12(%ebp), %ecx mov -8(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_proc_fork) ENTRY(cloudabi_sys_proc_raise) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) - mov $44, %eax + mov $42, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_proc_raise) ENTRY(cloudabi_sys_random_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $45, %eax + mov $43, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_random_get) ENTRY(cloudabi_sys_sock_accept) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $46, %eax + mov $44, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 16(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_sock_accept) ENTRY(cloudabi_sys_sock_bind) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $47, %eax + mov $45, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_bind) ENTRY(cloudabi_sys_sock_connect) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -32(%ebp) mov 12(%ebp), %ecx mov %ecx, -24(%ebp) mov 16(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 20(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $48, %eax + mov $46, %eax mov %ebp, %ecx sub $32, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_connect) ENTRY(cloudabi_sys_sock_listen) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) - mov $49, %eax + mov $47, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_listen) ENTRY(cloudabi_sys_sock_recv) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $50, %eax + mov $48, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_recv) ENTRY(cloudabi_sys_sock_send) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) movl $0, -4(%ebp) - mov $51, %eax + mov $49, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_send) ENTRY(cloudabi_sys_sock_shutdown) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) - mov $52, %eax + mov $50, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_shutdown) ENTRY(cloudabi_sys_sock_stat_get) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -24(%ebp) mov 12(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 16(%ebp), %ecx mov %ecx, -8(%ebp) - mov $53, %eax + mov $51, %eax mov %ebp, %ecx sub $24, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_sock_stat_get) ENTRY(cloudabi_sys_thread_create) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) - mov $54, %eax + mov $52, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 test %eax, %eax jnz 1f mov 12(%ebp), %ecx mov -16(%ebp), %edx mov %edx, 0(%ecx) 1: pop %ebp ret END(cloudabi_sys_thread_create) ENTRY(cloudabi_sys_thread_exit) push %ebp mov %esp, %ebp mov 8(%ebp), %ecx mov %ecx, -16(%ebp) movl $0, -12(%ebp) mov 12(%ebp), %ecx mov %ecx, -8(%ebp) - mov $55, %eax + mov $53, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 END(cloudabi_sys_thread_exit) ENTRY(cloudabi_sys_thread_yield) push %ebp mov %esp, %ebp - mov $56, %eax + mov $54, %eax mov %ebp, %ecx sub $16, %ecx int $0x80 pop %ebp ret END(cloudabi_sys_thread_yield) Index: projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_x86_64.S =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_x86_64.S (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/cloudabi_vdso_x86_64.S (revision 321585) @@ -1,511 +1,499 @@ // Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. // // This file is automatically generated. Do not edit. // // Source: https://github.com/NuxiNL/cloudabi #define ENTRY(name) \ .text; \ .p2align 4, 0x90; \ .global name; \ .type name, @function; \ name: #define END(name) .size name, . - name ENTRY(cloudabi_sys_clock_res_get) push %rsi mov $0, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_clock_res_get) ENTRY(cloudabi_sys_clock_time_get) push %rdx mov $1, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_clock_time_get) ENTRY(cloudabi_sys_condvar_signal) mov $2, %eax syscall ret END(cloudabi_sys_condvar_signal) ENTRY(cloudabi_sys_fd_close) mov $3, %eax syscall ret END(cloudabi_sys_fd_close) ENTRY(cloudabi_sys_fd_create1) push %rsi mov $4, %eax syscall pop %rcx jc 1f mov %eax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_create1) ENTRY(cloudabi_sys_fd_create2) push %rsi push %rdx mov $5, %eax syscall pop %rsi pop %rcx jc 1f mov %eax, (%rcx) mov %edx, (%rsi) xor %eax, %eax 1: ret END(cloudabi_sys_fd_create2) ENTRY(cloudabi_sys_fd_datasync) mov $6, %eax syscall ret END(cloudabi_sys_fd_datasync) ENTRY(cloudabi_sys_fd_dup) push %rsi mov $7, %eax syscall pop %rcx jc 1f mov %eax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_dup) ENTRY(cloudabi_sys_fd_pread) mov %rcx, %r10 push %r8 mov $8, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_pread) ENTRY(cloudabi_sys_fd_pwrite) mov %rcx, %r10 push %r8 mov $9, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_pwrite) ENTRY(cloudabi_sys_fd_read) push %rcx mov $10, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_read) ENTRY(cloudabi_sys_fd_replace) mov $11, %eax syscall ret END(cloudabi_sys_fd_replace) ENTRY(cloudabi_sys_fd_seek) push %rcx mov $12, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_seek) ENTRY(cloudabi_sys_fd_stat_get) mov $13, %eax syscall ret END(cloudabi_sys_fd_stat_get) ENTRY(cloudabi_sys_fd_stat_put) mov $14, %eax syscall ret END(cloudabi_sys_fd_stat_put) ENTRY(cloudabi_sys_fd_sync) mov $15, %eax syscall ret END(cloudabi_sys_fd_sync) ENTRY(cloudabi_sys_fd_write) push %rcx mov $16, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_fd_write) ENTRY(cloudabi_sys_file_advise) mov %rcx, %r10 mov $17, %eax syscall ret END(cloudabi_sys_file_advise) ENTRY(cloudabi_sys_file_allocate) mov $18, %eax syscall ret END(cloudabi_sys_file_allocate) ENTRY(cloudabi_sys_file_create) mov %rcx, %r10 mov $19, %eax syscall ret END(cloudabi_sys_file_create) ENTRY(cloudabi_sys_file_link) mov %rcx, %r10 mov $20, %eax syscall ret END(cloudabi_sys_file_link) ENTRY(cloudabi_sys_file_open) mov %rcx, %r10 push %r9 mov $21, %eax syscall pop %rcx jc 1f mov %eax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_file_open) ENTRY(cloudabi_sys_file_readdir) mov %rcx, %r10 push %r8 mov $22, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_file_readdir) ENTRY(cloudabi_sys_file_readlink) mov %rcx, %r10 push %r9 mov $23, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_file_readlink) ENTRY(cloudabi_sys_file_rename) mov %rcx, %r10 mov $24, %eax syscall ret END(cloudabi_sys_file_rename) ENTRY(cloudabi_sys_file_stat_fget) mov $25, %eax syscall ret END(cloudabi_sys_file_stat_fget) ENTRY(cloudabi_sys_file_stat_fput) mov $26, %eax syscall ret END(cloudabi_sys_file_stat_fput) ENTRY(cloudabi_sys_file_stat_get) mov %rcx, %r10 mov $27, %eax syscall ret END(cloudabi_sys_file_stat_get) ENTRY(cloudabi_sys_file_stat_put) mov %rcx, %r10 mov $28, %eax syscall ret END(cloudabi_sys_file_stat_put) ENTRY(cloudabi_sys_file_symlink) mov %rcx, %r10 mov $29, %eax syscall ret END(cloudabi_sys_file_symlink) ENTRY(cloudabi_sys_file_unlink) mov %rcx, %r10 mov $30, %eax syscall ret END(cloudabi_sys_file_unlink) ENTRY(cloudabi_sys_lock_unlock) mov $31, %eax syscall ret END(cloudabi_sys_lock_unlock) ENTRY(cloudabi_sys_mem_advise) mov $32, %eax syscall ret END(cloudabi_sys_mem_advise) -ENTRY(cloudabi_sys_mem_lock) - mov $33, %eax - syscall - ret -END(cloudabi_sys_mem_lock) - ENTRY(cloudabi_sys_mem_map) mov %rcx, %r10 - mov $34, %eax + mov $33, %eax syscall jc 1f mov 8(%rsp), %rcx mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_mem_map) ENTRY(cloudabi_sys_mem_protect) - mov $35, %eax + mov $34, %eax syscall ret END(cloudabi_sys_mem_protect) ENTRY(cloudabi_sys_mem_sync) - mov $36, %eax + mov $35, %eax syscall ret END(cloudabi_sys_mem_sync) -ENTRY(cloudabi_sys_mem_unlock) - mov $37, %eax - syscall - ret -END(cloudabi_sys_mem_unlock) - ENTRY(cloudabi_sys_mem_unmap) - mov $38, %eax + mov $36, %eax syscall ret END(cloudabi_sys_mem_unmap) ENTRY(cloudabi_sys_poll) push %rcx - mov $39, %eax + mov $37, %eax syscall pop %rcx jc 1f mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_poll) ENTRY(cloudabi_sys_poll_fd) mov %rcx, %r10 - mov $40, %eax + mov $38, %eax syscall jc 1f mov 8(%rsp), %rcx mov %rax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_poll_fd) ENTRY(cloudabi_sys_proc_exec) mov %rcx, %r10 - mov $41, %eax + mov $39, %eax syscall ret END(cloudabi_sys_proc_exec) ENTRY(cloudabi_sys_proc_exit) - mov $42, %eax + mov $40, %eax syscall END(cloudabi_sys_proc_exit) ENTRY(cloudabi_sys_proc_fork) push %rdi push %rsi - mov $43, %eax + mov $41, %eax syscall pop %rsi pop %rcx jc 1f mov %eax, (%rcx) mov %edx, (%rsi) xor %eax, %eax 1: ret END(cloudabi_sys_proc_fork) ENTRY(cloudabi_sys_proc_raise) - mov $44, %eax + mov $42, %eax syscall ret END(cloudabi_sys_proc_raise) ENTRY(cloudabi_sys_random_get) - mov $45, %eax + mov $43, %eax syscall ret END(cloudabi_sys_random_get) ENTRY(cloudabi_sys_sock_accept) push %rdx - mov $46, %eax + mov $44, %eax syscall pop %rcx jc 1f mov %eax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_sock_accept) ENTRY(cloudabi_sys_sock_bind) mov %rcx, %r10 - mov $47, %eax + mov $45, %eax syscall ret END(cloudabi_sys_sock_bind) ENTRY(cloudabi_sys_sock_connect) mov %rcx, %r10 - mov $48, %eax + mov $46, %eax syscall ret END(cloudabi_sys_sock_connect) ENTRY(cloudabi_sys_sock_listen) - mov $49, %eax + mov $47, %eax syscall ret END(cloudabi_sys_sock_listen) ENTRY(cloudabi_sys_sock_recv) - mov $50, %eax + mov $48, %eax syscall ret END(cloudabi_sys_sock_recv) ENTRY(cloudabi_sys_sock_send) - mov $51, %eax + mov $49, %eax syscall ret END(cloudabi_sys_sock_send) ENTRY(cloudabi_sys_sock_shutdown) - mov $52, %eax + mov $50, %eax syscall ret END(cloudabi_sys_sock_shutdown) ENTRY(cloudabi_sys_sock_stat_get) - mov $53, %eax + mov $51, %eax syscall ret END(cloudabi_sys_sock_stat_get) ENTRY(cloudabi_sys_thread_create) push %rsi - mov $54, %eax + mov $52, %eax syscall pop %rcx jc 1f mov %eax, (%rcx) xor %eax, %eax 1: ret END(cloudabi_sys_thread_create) ENTRY(cloudabi_sys_thread_exit) - mov $55, %eax + mov $53, %eax syscall END(cloudabi_sys_thread_exit) ENTRY(cloudabi_sys_thread_yield) - mov $56, %eax + mov $54, %eax syscall ret END(cloudabi_sys_thread_yield) Index: projects/runtime-coverage/sys/contrib/cloudabi/syscalls32.master =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/syscalls32.master (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/syscalls32.master (revision 321585) @@ -1,315 +1,307 @@ $FreeBSD$ ; Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions ; are met: ; 1. Redistributions of source code must retain the above copyright ; notice, this list of conditions and the following disclaimer. ; 2. Redistributions in binary form must reproduce the above copyright ; notice, this list of conditions and the following disclaimer in the ; documentation and/or other materials provided with the distribution. ; ; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE ; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL ; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS ; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY ; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ; SUCH DAMAGE. ; ; This file is automatically generated. Do not edit. ; ; Source: https://github.com/NuxiNL/cloudabi #include #include #include #include 0 AUE_NULL STD { cloudabi_timestamp_t \ cloudabi_sys_clock_res_get( \ cloudabi_clockid_t clock_id); } 1 AUE_NULL STD { cloudabi_timestamp_t \ cloudabi_sys_clock_time_get( \ cloudabi_clockid_t clock_id, \ cloudabi_timestamp_t precision); } 2 AUE_NULL STD { void cloudabi_sys_condvar_signal( \ cloudabi_condvar_t *condvar, \ cloudabi_scope_t scope, \ cloudabi_nthreads_t nwaiters); } 3 AUE_NULL STD { void cloudabi_sys_fd_close( \ cloudabi_fd_t fd); } 4 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_fd_create1( \ cloudabi_filetype_t type); } 5 AUE_NULL STD { void cloudabi_sys_fd_create2( \ cloudabi_filetype_t type); } 6 AUE_NULL STD { void cloudabi_sys_fd_datasync( \ cloudabi_fd_t fd); } 7 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_fd_dup( \ cloudabi_fd_t from); } 8 AUE_NULL STD { size_t cloudabi32_sys_fd_pread( \ cloudabi_fd_t fd, \ const cloudabi32_iovec_t *iovs, \ size_t iovs_len, \ cloudabi_filesize_t offset); } 9 AUE_NULL STD { size_t cloudabi32_sys_fd_pwrite( \ cloudabi_fd_t fd, \ const cloudabi32_ciovec_t *iovs, \ size_t iovs_len, \ cloudabi_filesize_t offset); } 10 AUE_NULL STD { size_t cloudabi32_sys_fd_read( \ cloudabi_fd_t fd, \ const cloudabi32_iovec_t *iovs, \ size_t iovs_len); } 11 AUE_NULL STD { void cloudabi_sys_fd_replace( \ cloudabi_fd_t from, \ cloudabi_fd_t to); } 12 AUE_NULL STD { cloudabi_filesize_t \ cloudabi_sys_fd_seek( \ cloudabi_fd_t fd, \ cloudabi_filedelta_t offset, \ cloudabi_whence_t whence); } 13 AUE_NULL STD { void cloudabi_sys_fd_stat_get( \ cloudabi_fd_t fd, \ cloudabi_fdstat_t *buf); } 14 AUE_NULL STD { void cloudabi_sys_fd_stat_put( \ cloudabi_fd_t fd, \ const cloudabi_fdstat_t *buf, \ cloudabi_fdsflags_t flags); } 15 AUE_NULL STD { void cloudabi_sys_fd_sync( \ cloudabi_fd_t fd); } 16 AUE_NULL STD { size_t cloudabi32_sys_fd_write( \ cloudabi_fd_t fd, \ const cloudabi32_ciovec_t *iovs, \ size_t iovs_len); } 17 AUE_NULL STD { void cloudabi_sys_file_advise( \ cloudabi_fd_t fd, \ cloudabi_filesize_t offset, \ cloudabi_filesize_t len, \ cloudabi_advice_t advice); } 18 AUE_NULL STD { void cloudabi_sys_file_allocate( \ cloudabi_fd_t fd, \ cloudabi_filesize_t offset, \ cloudabi_filesize_t len); } 19 AUE_NULL STD { void cloudabi_sys_file_create( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ cloudabi_filetype_t type); } 20 AUE_NULL STD { void cloudabi_sys_file_link( \ cloudabi_lookup_t fd1, \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd2, \ const char *path2, \ size_t path2_len); } 21 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_file_open( \ cloudabi_lookup_t dirfd, \ const char *path, \ size_t path_len, \ cloudabi_oflags_t oflags, \ const cloudabi_fdstat_t *fds); } 22 AUE_NULL STD { size_t cloudabi_sys_file_readdir( \ cloudabi_fd_t fd, \ void *buf, \ size_t buf_len, \ cloudabi_dircookie_t cookie); } 23 AUE_NULL STD { size_t cloudabi_sys_file_readlink( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ char *buf, \ size_t buf_len); } 24 AUE_NULL STD { void cloudabi_sys_file_rename( \ cloudabi_fd_t fd1, \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd2, \ const char *path2, \ size_t path2_len); } 25 AUE_NULL STD { void cloudabi_sys_file_stat_fget( \ cloudabi_fd_t fd, \ cloudabi_filestat_t *buf); } 26 AUE_NULL STD { void cloudabi_sys_file_stat_fput( \ cloudabi_fd_t fd, \ const cloudabi_filestat_t *buf, \ cloudabi_fsflags_t flags); } 27 AUE_NULL STD { void cloudabi_sys_file_stat_get( \ cloudabi_lookup_t fd, \ const char *path, \ size_t path_len, \ cloudabi_filestat_t *buf); } 28 AUE_NULL STD { void cloudabi_sys_file_stat_put( \ cloudabi_lookup_t fd, \ const char *path, \ size_t path_len, \ const cloudabi_filestat_t *buf, \ cloudabi_fsflags_t flags); } 29 AUE_NULL STD { void cloudabi_sys_file_symlink( \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd, \ const char *path2, \ size_t path2_len); } 30 AUE_NULL STD { void cloudabi_sys_file_unlink( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ cloudabi_ulflags_t flags); } 31 AUE_NULL STD { void cloudabi_sys_lock_unlock( \ cloudabi_lock_t *lock, \ cloudabi_scope_t scope); } 32 AUE_NULL STD { void cloudabi_sys_mem_advise( \ void *mapping, \ size_t mapping_len, \ cloudabi_advice_t advice); } -33 AUE_NULL STD { void cloudabi_sys_mem_lock( \ - const void *mapping, \ - size_t mapping_len); } - -34 AUE_NULL STD { void cloudabi_sys_mem_map( \ +33 AUE_NULL STD { void cloudabi_sys_mem_map( \ void *addr, \ size_t len, \ cloudabi_mprot_t prot, \ cloudabi_mflags_t flags, \ cloudabi_fd_t fd, \ cloudabi_filesize_t off); } -35 AUE_NULL STD { void cloudabi_sys_mem_protect( \ +34 AUE_NULL STD { void cloudabi_sys_mem_protect( \ void *mapping, \ size_t mapping_len, \ cloudabi_mprot_t prot); } -36 AUE_NULL STD { void cloudabi_sys_mem_sync( \ +35 AUE_NULL STD { void cloudabi_sys_mem_sync( \ void *mapping, \ size_t mapping_len, \ cloudabi_msflags_t flags); } -37 AUE_NULL STD { void cloudabi_sys_mem_unlock( \ - const void *mapping, \ - size_t mapping_len); } - -38 AUE_NULL STD { void cloudabi_sys_mem_unmap( \ +36 AUE_NULL STD { void cloudabi_sys_mem_unmap( \ void *mapping, \ size_t mapping_len); } -39 AUE_NULL STD { size_t cloudabi32_sys_poll( \ +37 AUE_NULL STD { size_t cloudabi32_sys_poll( \ const cloudabi32_subscription_t *in, \ cloudabi32_event_t *out, \ size_t nsubscriptions); } -40 AUE_NULL STD { size_t cloudabi32_sys_poll_fd( \ +38 AUE_NULL STD { size_t cloudabi32_sys_poll_fd( \ cloudabi_fd_t fd, \ const cloudabi32_subscription_t *in, \ size_t in_len, \ cloudabi32_event_t *out, \ size_t out_len, \ const cloudabi32_subscription_t *timeout); } -41 AUE_NULL STD { void cloudabi_sys_proc_exec( \ +39 AUE_NULL STD { void cloudabi_sys_proc_exec( \ cloudabi_fd_t fd, \ const void *data, \ size_t data_len, \ const cloudabi_fd_t *fds, \ size_t fds_len); } -42 AUE_NULL STD { void cloudabi_sys_proc_exit( \ +40 AUE_NULL STD { void cloudabi_sys_proc_exit( \ cloudabi_exitcode_t rval); } -43 AUE_NULL STD { void cloudabi_sys_proc_fork(); } +41 AUE_NULL STD { void cloudabi_sys_proc_fork(); } -44 AUE_NULL STD { void cloudabi_sys_proc_raise( \ +42 AUE_NULL STD { void cloudabi_sys_proc_raise( \ cloudabi_signal_t sig); } -45 AUE_NULL STD { void cloudabi_sys_random_get( \ +43 AUE_NULL STD { void cloudabi_sys_random_get( \ void *buf, \ size_t buf_len); } -46 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_sock_accept( \ +44 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_sock_accept( \ cloudabi_fd_t sock, \ - cloudabi_sockstat_t *buf); } + void *unused); } -47 AUE_NULL STD { void cloudabi_sys_sock_bind( \ +45 AUE_NULL STD { void cloudabi_sys_sock_bind( \ cloudabi_fd_t sock, \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len); } -48 AUE_NULL STD { void cloudabi_sys_sock_connect( \ +46 AUE_NULL STD { void cloudabi_sys_sock_connect( \ cloudabi_fd_t sock, \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len); } -49 AUE_NULL STD { void cloudabi_sys_sock_listen( \ +47 AUE_NULL STD { void cloudabi_sys_sock_listen( \ cloudabi_fd_t sock, \ cloudabi_backlog_t backlog); } -50 AUE_NULL STD { void cloudabi32_sys_sock_recv( \ +48 AUE_NULL STD { void cloudabi32_sys_sock_recv( \ cloudabi_fd_t sock, \ const cloudabi32_recv_in_t *in, \ cloudabi32_recv_out_t *out); } -51 AUE_NULL STD { void cloudabi32_sys_sock_send( \ +49 AUE_NULL STD { void cloudabi32_sys_sock_send( \ cloudabi_fd_t sock, \ const cloudabi32_send_in_t *in, \ cloudabi32_send_out_t *out); } -52 AUE_NULL STD { void cloudabi_sys_sock_shutdown( \ +50 AUE_NULL STD { void cloudabi_sys_sock_shutdown( \ cloudabi_fd_t sock, \ cloudabi_sdflags_t how); } -53 AUE_NULL STD { void cloudabi_sys_sock_stat_get( \ +51 AUE_NULL STD { void cloudabi_sys_sock_stat_get( \ cloudabi_fd_t sock, \ cloudabi_sockstat_t *buf, \ cloudabi_ssflags_t flags); } -54 AUE_NULL STD { cloudabi_tid_t cloudabi32_sys_thread_create( \ +52 AUE_NULL STD { cloudabi_tid_t cloudabi32_sys_thread_create( \ cloudabi32_threadattr_t *attr); } -55 AUE_NULL STD { void cloudabi_sys_thread_exit( \ +53 AUE_NULL STD { void cloudabi_sys_thread_exit( \ cloudabi_lock_t *lock, \ cloudabi_scope_t scope); } -56 AUE_NULL STD { void cloudabi_sys_thread_yield(); } +54 AUE_NULL STD { void cloudabi_sys_thread_yield(); } Index: projects/runtime-coverage/sys/contrib/cloudabi/syscalls64.master =================================================================== --- projects/runtime-coverage/sys/contrib/cloudabi/syscalls64.master (revision 321584) +++ projects/runtime-coverage/sys/contrib/cloudabi/syscalls64.master (revision 321585) @@ -1,315 +1,307 @@ $FreeBSD$ ; Copyright (c) 2016 Nuxi (https://nuxi.nl/) and contributors. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions ; are met: ; 1. Redistributions of source code must retain the above copyright ; notice, this list of conditions and the following disclaimer. ; 2. Redistributions in binary form must reproduce the above copyright ; notice, this list of conditions and the following disclaimer in the ; documentation and/or other materials provided with the distribution. ; ; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE ; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL ; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS ; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY ; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ; SUCH DAMAGE. ; ; This file is automatically generated. Do not edit. ; ; Source: https://github.com/NuxiNL/cloudabi #include #include #include #include 0 AUE_NULL STD { cloudabi_timestamp_t \ cloudabi_sys_clock_res_get( \ cloudabi_clockid_t clock_id); } 1 AUE_NULL STD { cloudabi_timestamp_t \ cloudabi_sys_clock_time_get( \ cloudabi_clockid_t clock_id, \ cloudabi_timestamp_t precision); } 2 AUE_NULL STD { void cloudabi_sys_condvar_signal( \ cloudabi_condvar_t *condvar, \ cloudabi_scope_t scope, \ cloudabi_nthreads_t nwaiters); } 3 AUE_NULL STD { void cloudabi_sys_fd_close( \ cloudabi_fd_t fd); } 4 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_fd_create1( \ cloudabi_filetype_t type); } 5 AUE_NULL STD { void cloudabi_sys_fd_create2( \ cloudabi_filetype_t type); } 6 AUE_NULL STD { void cloudabi_sys_fd_datasync( \ cloudabi_fd_t fd); } 7 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_fd_dup( \ cloudabi_fd_t from); } 8 AUE_NULL STD { size_t cloudabi64_sys_fd_pread( \ cloudabi_fd_t fd, \ const cloudabi64_iovec_t *iovs, \ size_t iovs_len, \ cloudabi_filesize_t offset); } 9 AUE_NULL STD { size_t cloudabi64_sys_fd_pwrite( \ cloudabi_fd_t fd, \ const cloudabi64_ciovec_t *iovs, \ size_t iovs_len, \ cloudabi_filesize_t offset); } 10 AUE_NULL STD { size_t cloudabi64_sys_fd_read( \ cloudabi_fd_t fd, \ const cloudabi64_iovec_t *iovs, \ size_t iovs_len); } 11 AUE_NULL STD { void cloudabi_sys_fd_replace( \ cloudabi_fd_t from, \ cloudabi_fd_t to); } 12 AUE_NULL STD { cloudabi_filesize_t \ cloudabi_sys_fd_seek( \ cloudabi_fd_t fd, \ cloudabi_filedelta_t offset, \ cloudabi_whence_t whence); } 13 AUE_NULL STD { void cloudabi_sys_fd_stat_get( \ cloudabi_fd_t fd, \ cloudabi_fdstat_t *buf); } 14 AUE_NULL STD { void cloudabi_sys_fd_stat_put( \ cloudabi_fd_t fd, \ const cloudabi_fdstat_t *buf, \ cloudabi_fdsflags_t flags); } 15 AUE_NULL STD { void cloudabi_sys_fd_sync( \ cloudabi_fd_t fd); } 16 AUE_NULL STD { size_t cloudabi64_sys_fd_write( \ cloudabi_fd_t fd, \ const cloudabi64_ciovec_t *iovs, \ size_t iovs_len); } 17 AUE_NULL STD { void cloudabi_sys_file_advise( \ cloudabi_fd_t fd, \ cloudabi_filesize_t offset, \ cloudabi_filesize_t len, \ cloudabi_advice_t advice); } 18 AUE_NULL STD { void cloudabi_sys_file_allocate( \ cloudabi_fd_t fd, \ cloudabi_filesize_t offset, \ cloudabi_filesize_t len); } 19 AUE_NULL STD { void cloudabi_sys_file_create( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ cloudabi_filetype_t type); } 20 AUE_NULL STD { void cloudabi_sys_file_link( \ cloudabi_lookup_t fd1, \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd2, \ const char *path2, \ size_t path2_len); } 21 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_file_open( \ cloudabi_lookup_t dirfd, \ const char *path, \ size_t path_len, \ cloudabi_oflags_t oflags, \ const cloudabi_fdstat_t *fds); } 22 AUE_NULL STD { size_t cloudabi_sys_file_readdir( \ cloudabi_fd_t fd, \ void *buf, \ size_t buf_len, \ cloudabi_dircookie_t cookie); } 23 AUE_NULL STD { size_t cloudabi_sys_file_readlink( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ char *buf, \ size_t buf_len); } 24 AUE_NULL STD { void cloudabi_sys_file_rename( \ cloudabi_fd_t fd1, \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd2, \ const char *path2, \ size_t path2_len); } 25 AUE_NULL STD { void cloudabi_sys_file_stat_fget( \ cloudabi_fd_t fd, \ cloudabi_filestat_t *buf); } 26 AUE_NULL STD { void cloudabi_sys_file_stat_fput( \ cloudabi_fd_t fd, \ const cloudabi_filestat_t *buf, \ cloudabi_fsflags_t flags); } 27 AUE_NULL STD { void cloudabi_sys_file_stat_get( \ cloudabi_lookup_t fd, \ const char *path, \ size_t path_len, \ cloudabi_filestat_t *buf); } 28 AUE_NULL STD { void cloudabi_sys_file_stat_put( \ cloudabi_lookup_t fd, \ const char *path, \ size_t path_len, \ const cloudabi_filestat_t *buf, \ cloudabi_fsflags_t flags); } 29 AUE_NULL STD { void cloudabi_sys_file_symlink( \ const char *path1, \ size_t path1_len, \ cloudabi_fd_t fd, \ const char *path2, \ size_t path2_len); } 30 AUE_NULL STD { void cloudabi_sys_file_unlink( \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len, \ cloudabi_ulflags_t flags); } 31 AUE_NULL STD { void cloudabi_sys_lock_unlock( \ cloudabi_lock_t *lock, \ cloudabi_scope_t scope); } 32 AUE_NULL STD { void cloudabi_sys_mem_advise( \ void *mapping, \ size_t mapping_len, \ cloudabi_advice_t advice); } -33 AUE_NULL STD { void cloudabi_sys_mem_lock( \ - const void *mapping, \ - size_t mapping_len); } - -34 AUE_NULL STD { void cloudabi_sys_mem_map( \ +33 AUE_NULL STD { void cloudabi_sys_mem_map( \ void *addr, \ size_t len, \ cloudabi_mprot_t prot, \ cloudabi_mflags_t flags, \ cloudabi_fd_t fd, \ cloudabi_filesize_t off); } -35 AUE_NULL STD { void cloudabi_sys_mem_protect( \ +34 AUE_NULL STD { void cloudabi_sys_mem_protect( \ void *mapping, \ size_t mapping_len, \ cloudabi_mprot_t prot); } -36 AUE_NULL STD { void cloudabi_sys_mem_sync( \ +35 AUE_NULL STD { void cloudabi_sys_mem_sync( \ void *mapping, \ size_t mapping_len, \ cloudabi_msflags_t flags); } -37 AUE_NULL STD { void cloudabi_sys_mem_unlock( \ - const void *mapping, \ - size_t mapping_len); } - -38 AUE_NULL STD { void cloudabi_sys_mem_unmap( \ +36 AUE_NULL STD { void cloudabi_sys_mem_unmap( \ void *mapping, \ size_t mapping_len); } -39 AUE_NULL STD { size_t cloudabi64_sys_poll( \ +37 AUE_NULL STD { size_t cloudabi64_sys_poll( \ const cloudabi64_subscription_t *in, \ cloudabi64_event_t *out, \ size_t nsubscriptions); } -40 AUE_NULL STD { size_t cloudabi64_sys_poll_fd( \ +38 AUE_NULL STD { size_t cloudabi64_sys_poll_fd( \ cloudabi_fd_t fd, \ const cloudabi64_subscription_t *in, \ size_t in_len, \ cloudabi64_event_t *out, \ size_t out_len, \ const cloudabi64_subscription_t *timeout); } -41 AUE_NULL STD { void cloudabi_sys_proc_exec( \ +39 AUE_NULL STD { void cloudabi_sys_proc_exec( \ cloudabi_fd_t fd, \ const void *data, \ size_t data_len, \ const cloudabi_fd_t *fds, \ size_t fds_len); } -42 AUE_NULL STD { void cloudabi_sys_proc_exit( \ +40 AUE_NULL STD { void cloudabi_sys_proc_exit( \ cloudabi_exitcode_t rval); } -43 AUE_NULL STD { void cloudabi_sys_proc_fork(); } +41 AUE_NULL STD { void cloudabi_sys_proc_fork(); } -44 AUE_NULL STD { void cloudabi_sys_proc_raise( \ +42 AUE_NULL STD { void cloudabi_sys_proc_raise( \ cloudabi_signal_t sig); } -45 AUE_NULL STD { void cloudabi_sys_random_get( \ +43 AUE_NULL STD { void cloudabi_sys_random_get( \ void *buf, \ size_t buf_len); } -46 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_sock_accept( \ +44 AUE_NULL STD { cloudabi_fd_t cloudabi_sys_sock_accept( \ cloudabi_fd_t sock, \ - cloudabi_sockstat_t *buf); } + void *unused); } -47 AUE_NULL STD { void cloudabi_sys_sock_bind( \ +45 AUE_NULL STD { void cloudabi_sys_sock_bind( \ cloudabi_fd_t sock, \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len); } -48 AUE_NULL STD { void cloudabi_sys_sock_connect( \ +46 AUE_NULL STD { void cloudabi_sys_sock_connect( \ cloudabi_fd_t sock, \ cloudabi_fd_t fd, \ const char *path, \ size_t path_len); } -49 AUE_NULL STD { void cloudabi_sys_sock_listen( \ +47 AUE_NULL STD { void cloudabi_sys_sock_listen( \ cloudabi_fd_t sock, \ cloudabi_backlog_t backlog); } -50 AUE_NULL STD { void cloudabi64_sys_sock_recv( \ +48 AUE_NULL STD { void cloudabi64_sys_sock_recv( \ cloudabi_fd_t sock, \ const cloudabi64_recv_in_t *in, \ cloudabi64_recv_out_t *out); } -51 AUE_NULL STD { void cloudabi64_sys_sock_send( \ +49 AUE_NULL STD { void cloudabi64_sys_sock_send( \ cloudabi_fd_t sock, \ const cloudabi64_send_in_t *in, \ cloudabi64_send_out_t *out); } -52 AUE_NULL STD { void cloudabi_sys_sock_shutdown( \ +50 AUE_NULL STD { void cloudabi_sys_sock_shutdown( \ cloudabi_fd_t sock, \ cloudabi_sdflags_t how); } -53 AUE_NULL STD { void cloudabi_sys_sock_stat_get( \ +51 AUE_NULL STD { void cloudabi_sys_sock_stat_get( \ cloudabi_fd_t sock, \ cloudabi_sockstat_t *buf, \ cloudabi_ssflags_t flags); } -54 AUE_NULL STD { cloudabi_tid_t cloudabi64_sys_thread_create( \ +52 AUE_NULL STD { cloudabi_tid_t cloudabi64_sys_thread_create( \ cloudabi64_threadattr_t *attr); } -55 AUE_NULL STD { void cloudabi_sys_thread_exit( \ +53 AUE_NULL STD { void cloudabi_sys_thread_exit( \ cloudabi_lock_t *lock, \ cloudabi_scope_t scope); } -56 AUE_NULL STD { void cloudabi_sys_thread_yield(); } +54 AUE_NULL STD { void cloudabi_sys_thread_yield(); } Index: projects/runtime-coverage/sys/dev/bnxt/bnxt.h =================================================================== --- projects/runtime-coverage/sys/dev/bnxt/bnxt.h (revision 321584) +++ projects/runtime-coverage/sys/dev/bnxt/bnxt.h (revision 321585) @@ -1,606 +1,632 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifndef _BNXT_H #define _BNXT_H #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" /* PCI IDs */ #define BROADCOM_VENDOR_ID 0x14E4 #define BCM57301 0x16c8 #define BCM57302 0x16c9 #define BCM57304 0x16ca #define BCM57311 0x16ce #define BCM57312 0x16cf #define BCM57314 0x16df #define BCM57402 0x16d0 #define BCM57402_NPAR 0x16d4 #define BCM57404 0x16d1 #define BCM57404_NPAR 0x16e7 #define BCM57406 0x16d2 #define BCM57406_NPAR 0x16e8 #define BCM57407 0x16d5 #define BCM57407_NPAR 0x16ea #define BCM57407_SFP 0x16e9 #define BCM57412 0x16d6 #define BCM57412_NPAR1 0x16de #define BCM57412_NPAR2 0x16eb #define BCM57414 0x16d7 #define BCM57414_NPAR1 0x16ec #define BCM57414_NPAR2 0x16ed #define BCM57416 0x16d8 #define BCM57416_NPAR1 0x16ee #define BCM57416_NPAR2 0x16ef #define BCM57416_SFP 0x16e3 #define BCM57417 0x16d9 #define BCM57417_NPAR1 0x16c0 #define BCM57417_NPAR2 0x16cc #define BCM57417_SFP 0x16e2 #define BCM57454 0x1614 #define BCM58700 0x16cd #define NETXTREME_C_VF1 0x16cb #define NETXTREME_C_VF2 0x16e1 #define NETXTREME_C_VF3 0x16e5 #define NETXTREME_E_VF1 0x16c1 #define NETXTREME_E_VF2 0x16d3 #define NETXTREME_E_VF3 0x16dc #define CSUM_OFFLOAD (CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \ CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \ CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP) #define BNXT_MAX_MTU 9000 #define BNXT_RSS_HASH_TYPE_TCPV4 0 #define BNXT_RSS_HASH_TYPE_UDPV4 1 #define BNXT_RSS_HASH_TYPE_IPV4 2 #define BNXT_RSS_HASH_TYPE_TCPV6 3 #define BNXT_RSS_HASH_TYPE_UDPV6 4 #define BNXT_RSS_HASH_TYPE_IPV6 5 #define BNXT_GET_RSS_PROFILE_ID(rss_hash_type) ((rss_hash_type >> 1) & 0x1F) #define BNXT_NO_MORE_WOL_FILTERS 0xFFFF #define bnxt_wol_supported(softc) ((softc)->flags & BNXT_FLAG_WOL_CAP) /* Completion related defines */ #define CMP_VALID(cmp, v_bit) \ ((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) ) #define NEXT_CP_CONS_V(ring, cons, v_bit) do { \ if (__predict_false(++(cons) == (ring)->ring_size)) \ ((cons) = 0, (v_bit) = !v_bit); \ } while (0) #define RING_NEXT(ring, idx) (__predict_false(idx + 1 == (ring)->ring_size) ? \ 0 : idx + 1) #define CMPL_PREFETCH_NEXT(cpr, idx) \ __builtin_prefetch(&((struct cmpl_base *)(cpr)->ring.vaddr)[((idx) +\ (CACHE_LINE_SIZE / sizeof(struct cmpl_base))) & \ ((cpr)->ring.ring_size - 1)]) /* * If we update the index, a write barrier is needed after the write to ensure * the completion ring has space before the RX/TX ring does. Since we can't * make the RX and AG doorbells covered by the same barrier without remapping * MSI-X vectors, we create the barrier over the enture doorbell bar. * TODO: Remap the MSI-X vectors to allow a barrier to only cover the doorbells * for a single ring group. * * A barrier of just the size of the write is used to ensure the ordering * remains correct and no writes are lost. */ #define BNXT_CP_DISABLE_DB(ring) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_MASK)); \ } while (0) #define BNXT_CP_ENABLE_DB(ring) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL)); \ } while (0) #define BNXT_CP_IDX_ENABLE_DB(ring, cons) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_IDX_VALID | \ (cons))); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ } while (0) #define BNXT_CP_IDX_DISABLE_DB(ring, cons) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_IDX_VALID | \ CMPL_DOORBELL_MASK | (cons))); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ } while (0) #define BNXT_TX_DB(ring, idx) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4( \ (ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, \ (ring)->doorbell, htole32(TX_DOORBELL_KEY_TX | (idx))); \ } while (0) #define BNXT_RX_DB(ring, idx) do { \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4( \ (ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, \ (ring)->doorbell, htole32(RX_DOORBELL_KEY_RX | (idx))); \ } while (0) /* Lock macros */ #define BNXT_HWRM_LOCK_INIT(_softc, _name) \ mtx_init(&(_softc)->hwrm_lock, _name, "BNXT HWRM Lock", MTX_DEF) #define BNXT_HWRM_LOCK(_softc) mtx_lock(&(_softc)->hwrm_lock) #define BNXT_HWRM_UNLOCK(_softc) mtx_unlock(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_DESTROY(_softc) mtx_destroy(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_ASSERT(_softc) mtx_assert(&(_softc)->hwrm_lock, \ MA_OWNED) /* Chip info */ #define BNXT_TSO_SIZE UINT16_MAX +#define min_t(type, x, y) ({ \ + type __min1 = (x); \ + type __min2 = (y); \ + __min1 < __min2 ? __min1 : __min2; }) + +#define max_t(type, x, y) ({ \ + type __max1 = (x); \ + type __max2 = (y); \ + __max1 > __max2 ? __max1 : __max2; }) + +#define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) + /* NVRAM access */ enum bnxt_nvm_directory_type { BNX_DIR_TYPE_UNUSED = 0, BNX_DIR_TYPE_PKG_LOG = 1, BNX_DIR_TYPE_UPDATE = 2, BNX_DIR_TYPE_CHIMP_PATCH = 3, BNX_DIR_TYPE_BOOTCODE = 4, BNX_DIR_TYPE_VPD = 5, BNX_DIR_TYPE_EXP_ROM_MBA = 6, BNX_DIR_TYPE_AVS = 7, BNX_DIR_TYPE_PCIE = 8, BNX_DIR_TYPE_PORT_MACRO = 9, BNX_DIR_TYPE_APE_FW = 10, BNX_DIR_TYPE_APE_PATCH = 11, BNX_DIR_TYPE_KONG_FW = 12, BNX_DIR_TYPE_KONG_PATCH = 13, BNX_DIR_TYPE_BONO_FW = 14, BNX_DIR_TYPE_BONO_PATCH = 15, BNX_DIR_TYPE_TANG_FW = 16, BNX_DIR_TYPE_TANG_PATCH = 17, BNX_DIR_TYPE_BOOTCODE_2 = 18, BNX_DIR_TYPE_CCM = 19, BNX_DIR_TYPE_PCI_CFG = 20, BNX_DIR_TYPE_TSCF_UCODE = 21, BNX_DIR_TYPE_ISCSI_BOOT = 22, BNX_DIR_TYPE_ISCSI_BOOT_IPV6 = 24, BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6 = 25, BNX_DIR_TYPE_ISCSI_BOOT_CFG6 = 26, BNX_DIR_TYPE_EXT_PHY = 27, BNX_DIR_TYPE_SHARED_CFG = 40, BNX_DIR_TYPE_PORT_CFG = 41, BNX_DIR_TYPE_FUNC_CFG = 42, BNX_DIR_TYPE_MGMT_CFG = 48, BNX_DIR_TYPE_MGMT_DATA = 49, BNX_DIR_TYPE_MGMT_WEB_DATA = 50, BNX_DIR_TYPE_MGMT_WEB_META = 51, BNX_DIR_TYPE_MGMT_EVENT_LOG = 52, BNX_DIR_TYPE_MGMT_AUDIT_LOG = 53 }; enum bnxnvm_pkglog_field_index { BNX_PKG_LOG_FIELD_IDX_INSTALLED_TIMESTAMP = 0, BNX_PKG_LOG_FIELD_IDX_PKG_DESCRIPTION = 1, BNX_PKG_LOG_FIELD_IDX_PKG_VERSION = 2, BNX_PKG_LOG_FIELD_IDX_PKG_TIMESTAMP = 3, BNX_PKG_LOG_FIELD_IDX_PKG_CHECKSUM = 4, BNX_PKG_LOG_FIELD_IDX_INSTALLED_ITEMS = 5, BNX_PKG_LOG_FIELD_IDX_INSTALLED_MASK = 6 }; #define BNX_DIR_ORDINAL_FIRST 0 #define BNX_DIR_EXT_NONE 0 struct bnxt_bar_info { struct resource *res; bus_space_tag_t tag; bus_space_handle_t handle; bus_size_t size; int rid; }; struct bnxt_link_info { uint8_t media_type; uint8_t transceiver; uint8_t phy_addr; uint8_t phy_link_status; uint8_t wire_speed; uint8_t loop_back; uint8_t link_up; uint8_t last_link_up; uint8_t duplex; uint8_t last_duplex; uint8_t pause; uint8_t last_pause; uint8_t auto_pause; uint8_t force_pause; uint8_t duplex_setting; uint8_t auto_mode; #define PHY_VER_LEN 3 uint8_t phy_ver[PHY_VER_LEN]; uint8_t phy_type; uint16_t link_speed; uint16_t support_speeds; uint16_t auto_link_speeds; uint16_t auto_link_speed; uint16_t force_link_speed; uint32_t preemphasis; /* copy of requested setting */ uint8_t autoneg; #define BNXT_AUTONEG_SPEED 1 #define BNXT_AUTONEG_FLOW_CTRL 2 uint8_t req_duplex; uint8_t req_flow_ctrl; uint16_t req_link_speed; }; enum bnxt_cp_type { BNXT_DEFAULT, BNXT_TX, BNXT_RX, BNXT_SHARED }; struct bnxt_cos_queue { uint8_t id; uint8_t profile; }; struct bnxt_func_info { uint32_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_irqs; uint16_t max_l2_ctxs; uint16_t max_vnics; uint16_t max_stat_ctxs; }; struct bnxt_pf_info { #define BNXT_FIRST_PF_FID 1 #define BNXT_FIRST_VF_FID 128 uint8_t port_id; uint32_t first_vf_id; uint16_t active_vfs; uint16_t max_vfs; uint32_t max_encap_records; uint32_t max_decap_records; uint32_t max_tx_em_flows; uint32_t max_tx_wm_flows; uint32_t max_rx_em_flows; uint32_t max_rx_wm_flows; unsigned long *vf_event_bmap; uint16_t hwrm_cmd_req_pages; void *hwrm_cmd_req_addr[4]; bus_addr_t hwrm_cmd_req_dma_addr[4]; }; struct bnxt_vf_info { uint16_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_l2_ctxs; uint16_t max_irqs; uint16_t max_vnics; uint16_t max_stat_ctxs; uint32_t vlan; #define BNXT_VF_QOS 0x1 #define BNXT_VF_SPOOFCHK 0x2 #define BNXT_VF_LINK_FORCED 0x4 #define BNXT_VF_LINK_UP 0x8 uint32_t flags; uint32_t func_flags; /* func cfg flags */ uint32_t min_tx_rate; uint32_t max_tx_rate; void *hwrm_cmd_req_addr; bus_addr_t hwrm_cmd_req_dma_addr; }; #define BNXT_FLAG_VF (1<<1) #define BNXT_PF(softc) (!((softc)->flags & BNXT_FLAG_VF)) #define BNXT_VF(softc) ((softc)->flags & BNXT_FLAG_VF) struct bnxt_vlan_tag { SLIST_ENTRY(bnxt_vlan_tag) next; uint16_t tpid; uint16_t tag; }; struct bnxt_vnic_info { uint16_t id; uint16_t def_ring_grp; uint16_t cos_rule; uint16_t lb_rule; uint16_t mru; uint32_t rx_mask; bool vlan_only; struct iflib_dma_info mc_list; int mc_list_count; #define BNXT_MAX_MC_ADDRS 16 uint32_t flags; #define BNXT_VNIC_FLAG_DEFAULT 0x01 #define BNXT_VNIC_FLAG_BD_STALL 0x02 #define BNXT_VNIC_FLAG_VLAN_STRIP 0x04 uint64_t filter_id; uint32_t flow_id; uint16_t rss_id; uint32_t rss_hash_type; uint8_t rss_hash_key[HW_HASH_KEY_SIZE]; struct iflib_dma_info rss_hash_key_tbl; struct iflib_dma_info rss_grp_tbl; SLIST_HEAD(vlan_head, bnxt_vlan_tag) vlan_tags; struct iflib_dma_info vlan_tag_list; }; struct bnxt_grp_info { uint16_t stats_ctx; uint16_t grp_id; uint16_t rx_ring_id; uint16_t cp_ring_id; uint16_t ag_ring_id; }; struct bnxt_ring { uint64_t paddr; vm_offset_t doorbell; caddr_t vaddr; struct bnxt_softc *softc; uint32_t ring_size; /* Must be a power of two */ uint16_t id; /* Logical ID */ uint16_t phys_id; struct bnxt_full_tpa_start *tpa_start; }; struct bnxt_cp_ring { struct bnxt_ring ring; struct if_irq irq; uint32_t cons; bool v_bit; /* Value of valid bit */ struct ctx_hw_stats *stats; uint32_t stats_ctx_id; uint32_t last_idx; /* Used by RX rings only * set to the last read pidx */ }; struct bnxt_full_tpa_start { struct rx_tpa_start_cmpl low; struct rx_tpa_start_cmpl_hi high; }; /* All the version information for the part */ #define BNXT_VERSTR_SIZE (3*3+2+1) /* ie: "255.255.255\0" */ #define BNXT_NAME_SIZE 17 struct bnxt_ver_info { uint8_t hwrm_if_major; uint8_t hwrm_if_minor; uint8_t hwrm_if_update; char hwrm_if_ver[BNXT_VERSTR_SIZE]; char driver_hwrm_if_ver[BNXT_VERSTR_SIZE]; char hwrm_fw_ver[BNXT_VERSTR_SIZE]; char mgmt_fw_ver[BNXT_VERSTR_SIZE]; char netctrl_fw_ver[BNXT_VERSTR_SIZE]; char roce_fw_ver[BNXT_VERSTR_SIZE]; char phy_ver[BNXT_VERSTR_SIZE]; char pkg_ver[64]; char hwrm_fw_name[BNXT_NAME_SIZE]; char mgmt_fw_name[BNXT_NAME_SIZE]; char netctrl_fw_name[BNXT_NAME_SIZE]; char roce_fw_name[BNXT_NAME_SIZE]; char phy_vendor[BNXT_NAME_SIZE]; char phy_partnumber[BNXT_NAME_SIZE]; uint16_t chip_num; uint8_t chip_rev; uint8_t chip_metal; uint8_t chip_bond_id; uint8_t chip_type; uint8_t hwrm_min_major; uint8_t hwrm_min_minor; uint8_t hwrm_min_update; struct sysctl_ctx_list ver_ctx; struct sysctl_oid *ver_oid; }; struct bnxt_nvram_info { uint16_t mfg_id; uint16_t device_id; uint32_t sector_size; uint32_t size; uint32_t reserved_size; uint32_t available_size; struct sysctl_ctx_list nvm_ctx; struct sysctl_oid *nvm_oid; }; struct bnxt_softc { device_t dev; if_ctx_t ctx; if_softc_ctx_t scctx; if_shared_ctx_t sctx; struct ifmedia *media; struct bnxt_bar_info hwrm_bar; struct bnxt_bar_info doorbell_bar; struct bnxt_link_info link_info; #define BNXT_FLAG_NPAR 0x1 #define BNXT_FLAG_WOL_CAP 0x2 uint32_t flags; uint32_t total_msix; struct bnxt_func_info func; struct bnxt_pf_info pf; struct bnxt_vf_info vf; uint16_t hwrm_cmd_seq; uint32_t hwrm_cmd_timeo; /* milliseconds */ struct iflib_dma_info hwrm_cmd_resp; /* Interrupt info for HWRM */ struct if_irq irq; struct mtx hwrm_lock; uint16_t hwrm_max_req_len; #define BNXT_MAX_QUEUE 8 uint8_t max_tc; struct bnxt_cos_queue q_info[BNXT_MAX_QUEUE]; struct iflib_dma_info hw_rx_port_stats; struct iflib_dma_info hw_tx_port_stats; struct rx_port_stats *rx_port_stats; struct tx_port_stats *tx_port_stats; int num_cp_rings; struct bnxt_ring *tx_rings; struct bnxt_cp_ring *tx_cp_rings; struct iflib_dma_info tx_stats; int ntxqsets; struct bnxt_vnic_info vnic_info; struct bnxt_ring *ag_rings; struct bnxt_ring *rx_rings; struct bnxt_cp_ring *rx_cp_rings; struct bnxt_grp_info *grp_info; struct iflib_dma_info rx_stats; int nrxqsets; struct bnxt_cp_ring def_cp_ring; struct iflib_dma_info def_cp_ring_mem; struct grouptask def_cp_task; struct sysctl_ctx_list hw_stats; struct sysctl_oid *hw_stats_oid; struct bnxt_ver_info *ver_info; struct bnxt_nvram_info *nvm_info; bool wol; uint8_t wol_filter_id; + uint16_t rx_coal_usecs; + uint16_t rx_coal_usecs_irq; + uint16_t rx_coal_frames; + uint16_t rx_coal_frames_irq; + uint16_t tx_coal_usecs; + uint16_t tx_coal_usecs_irq; + uint16_t tx_coal_frames; + uint16_t tx_coal_frames_irq; + +#define BNXT_USEC_TO_COAL_TIMER(x) ((x) * 25 / 2) +#define BNXT_DEF_STATS_COAL_TICKS 1000000 +#define BNXT_MIN_STATS_COAL_TICKS 250000 +#define BNXT_MAX_STATS_COAL_TICKS 1000000 + }; struct bnxt_filter_info { STAILQ_ENTRY(bnxt_filter_info) next; uint64_t fw_l2_filter_id; #define INVALID_MAC_INDEX ((uint16_t)-1) uint16_t mac_index; /* Filter Characteristics */ uint32_t flags; uint32_t enables; uint8_t l2_addr[ETHER_ADDR_LEN]; uint8_t l2_addr_mask[ETHER_ADDR_LEN]; uint16_t l2_ovlan; uint16_t l2_ovlan_mask; uint16_t l2_ivlan; uint16_t l2_ivlan_mask; uint8_t t_l2_addr[ETHER_ADDR_LEN]; uint8_t t_l2_addr_mask[ETHER_ADDR_LEN]; uint16_t t_l2_ovlan; uint16_t t_l2_ovlan_mask; uint16_t t_l2_ivlan; uint16_t t_l2_ivlan_mask; uint8_t tunnel_type; uint16_t mirror_vnic_id; uint32_t vni; uint8_t pri_hint; uint64_t l2_filter_id_hint; }; /* Function declarations */ void bnxt_report_link(struct bnxt_softc *softc); bool bnxt_check_hwrm_version(struct bnxt_softc *softc); #endif /* _BNXT_H */ Index: projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.c =================================================================== --- projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.c (revision 321584) +++ projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.c (revision 321585) @@ -1,1549 +1,1678 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" #include "hsi_struct_def.h" static int bnxt_hwrm_err_map(uint16_t err); static inline int _is_valid_ether_addr(uint8_t *); static inline void get_random_ether_addr(uint8_t *); static void bnxt_hwrm_set_link_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); static void bnxt_hwrm_set_pause_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); static void bnxt_hwrm_set_eee(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); static int _hwrm_send_message(struct bnxt_softc *, void *, uint32_t); static int hwrm_send_message(struct bnxt_softc *, void *, uint32_t); static void bnxt_hwrm_cmd_hdr_init(struct bnxt_softc *, void *, uint16_t); /* NVRam stuff has a five minute timeout */ #define BNXT_NVM_TIMEO (5 * 60 * 1000) static int bnxt_hwrm_err_map(uint16_t err) { int rc; switch (err) { case HWRM_ERR_CODE_SUCCESS: return 0; case HWRM_ERR_CODE_INVALID_PARAMS: case HWRM_ERR_CODE_INVALID_FLAGS: case HWRM_ERR_CODE_INVALID_ENABLES: return EINVAL; case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: return EACCES; case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: return ENOMEM; case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: return ENOSYS; case HWRM_ERR_CODE_FAIL: return EIO; case HWRM_ERR_CODE_HWRM_ERROR: case HWRM_ERR_CODE_UNKNOWN_ERR: default: return EDOOFUS; } return rc; } int bnxt_alloc_hwrm_dma_mem(struct bnxt_softc *softc) { int rc; rc = iflib_dma_alloc(softc->ctx, PAGE_SIZE, &softc->hwrm_cmd_resp, BUS_DMA_NOWAIT); return rc; } void bnxt_free_hwrm_dma_mem(struct bnxt_softc *softc) { if (softc->hwrm_cmd_resp.idi_vaddr) iflib_dma_free(&softc->hwrm_cmd_resp); softc->hwrm_cmd_resp.idi_vaddr = NULL; return; } static void bnxt_hwrm_cmd_hdr_init(struct bnxt_softc *softc, void *request, uint16_t req_type) { struct input *req = request; req->req_type = htole16(req_type); req->cmpl_ring = 0xffff; req->target_id = 0xffff; req->resp_addr = htole64(softc->hwrm_cmd_resp.idi_paddr); } static int _hwrm_send_message(struct bnxt_softc *softc, void *msg, uint32_t msg_len) { struct input *req = msg; struct hwrm_err_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; uint32_t *data = msg; int i; uint16_t cp_ring_id; uint8_t *valid; uint16_t err; /* TODO: DMASYNC in here. */ req->seq_id = htole16(softc->hwrm_cmd_seq++); memset(resp, 0, PAGE_SIZE); cp_ring_id = le16toh(req->cmpl_ring); /* Write request msg to hwrm channel */ for (i = 0; i < msg_len; i += 4) { bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, i, *data); data++; } /* Clear to the end of the request buffer */ for (i = msg_len; i < HWRM_MAX_REQ_LEN; i += 4) bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, i, 0); /* Ring channel doorbell */ bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, 0x100, htole32(1)); /* Check if response len is updated */ for (i = 0; i < softc->hwrm_cmd_timeo; i++) { if (resp->resp_len && resp->resp_len <= 4096) break; DELAY(1000); } if (i >= softc->hwrm_cmd_timeo) { device_printf(softc->dev, "Timeout sending %s: (timeout: %u) seq: %d\n", GET_HWRM_REQ_TYPE(req->req_type), softc->hwrm_cmd_timeo, le16toh(req->seq_id)); return ETIMEDOUT; } /* Last byte of resp contains the valid key */ valid = (uint8_t *)resp + resp->resp_len - 1; for (i = 0; i < softc->hwrm_cmd_timeo; i++) { if (*valid == HWRM_RESP_VALID_KEY) break; DELAY(1000); } if (i >= softc->hwrm_cmd_timeo) { device_printf(softc->dev, "Timeout sending %s: " "(timeout: %u) msg {0x%x 0x%x} len:%d v: %d\n", GET_HWRM_REQ_TYPE(req->req_type), softc->hwrm_cmd_timeo, le16toh(req->req_type), le16toh(req->seq_id), msg_len, *valid); return ETIMEDOUT; } err = le16toh(resp->error_code); if (err) { /* HWRM_ERR_CODE_FAIL is a "normal" error, don't log */ if (err != HWRM_ERR_CODE_FAIL) { device_printf(softc->dev, "%s command returned %s error.\n", GET_HWRM_REQ_TYPE(req->req_type), GET_HWRM_ERROR_CODE(err)); } return bnxt_hwrm_err_map(err); } return 0; } static int hwrm_send_message(struct bnxt_softc *softc, void *msg, uint32_t msg_len) { int rc; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, msg, msg_len); BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_queue_qportcfg(struct bnxt_softc *softc) { struct hwrm_queue_qportcfg_input req = {0}; struct hwrm_queue_qportcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; uint8_t *qptr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_QUEUE_QPORTCFG); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto qportcfg_exit; if (!resp->max_configurable_queues) { rc = -EINVAL; goto qportcfg_exit; } softc->max_tc = resp->max_configurable_queues; if (softc->max_tc > BNXT_MAX_QUEUE) softc->max_tc = BNXT_MAX_QUEUE; qptr = &resp->queue_id0; for (int i = 0; i < softc->max_tc; i++) { softc->q_info[i].id = *qptr++; softc->q_info[i].profile = *qptr++; } qportcfg_exit: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_ver_get(struct bnxt_softc *softc) { struct hwrm_ver_get_input req = {0}; struct hwrm_ver_get_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; const char nastr[] = ""; const char naver[] = ""; softc->hwrm_max_req_len = HWRM_MAX_REQ_LEN; softc->hwrm_cmd_timeo = 1000; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VER_GET); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; snprintf(softc->ver_info->hwrm_if_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd); softc->ver_info->hwrm_if_major = resp->hwrm_intf_maj; softc->ver_info->hwrm_if_minor = resp->hwrm_intf_min; softc->ver_info->hwrm_if_update = resp->hwrm_intf_upd; snprintf(softc->ver_info->hwrm_fw_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld); strlcpy(softc->ver_info->driver_hwrm_if_ver, HWRM_VERSION_STR, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->hwrm_fw_name, resp->hwrm_fw_name, BNXT_NAME_SIZE); if (resp->mgmt_fw_maj == 0 && resp->mgmt_fw_min == 0 && resp->mgmt_fw_bld == 0) { strlcpy(softc->ver_info->mgmt_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->mgmt_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->mgmt_fw_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->mgmt_fw_maj, resp->mgmt_fw_min, resp->mgmt_fw_bld); strlcpy(softc->ver_info->mgmt_fw_name, resp->mgmt_fw_name, BNXT_NAME_SIZE); } if (resp->netctrl_fw_maj == 0 && resp->netctrl_fw_min == 0 && resp->netctrl_fw_bld == 0) { strlcpy(softc->ver_info->netctrl_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->netctrl_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->netctrl_fw_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->netctrl_fw_maj, resp->netctrl_fw_min, resp->netctrl_fw_bld); strlcpy(softc->ver_info->netctrl_fw_name, resp->netctrl_fw_name, BNXT_NAME_SIZE); } if (resp->roce_fw_maj == 0 && resp->roce_fw_min == 0 && resp->roce_fw_bld == 0) { strlcpy(softc->ver_info->roce_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->roce_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->roce_fw_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->roce_fw_maj, resp->roce_fw_min, resp->roce_fw_bld); strlcpy(softc->ver_info->roce_fw_name, resp->roce_fw_name, BNXT_NAME_SIZE); } softc->ver_info->chip_num = le16toh(resp->chip_num); softc->ver_info->chip_rev = resp->chip_rev; softc->ver_info->chip_metal = resp->chip_metal; softc->ver_info->chip_bond_id = resp->chip_bond_id; softc->ver_info->chip_type = resp->chip_platform_type; if (resp->max_req_win_len) softc->hwrm_max_req_len = le16toh(resp->max_req_win_len); if (resp->def_req_timeout) softc->hwrm_cmd_timeo = le16toh(resp->def_req_timeout); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_drv_rgtr(struct bnxt_softc *softc) { struct hwrm_func_drv_rgtr_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_DRV_RGTR); req.enables = htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_OS_TYPE); req.os_type = htole16(HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_FREEBSD); req.ver_maj = __FreeBSD_version / 100000; req.ver_min = (__FreeBSD_version / 1000) % 100; req.ver_upd = (__FreeBSD_version / 100) % 10; return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_func_drv_unrgtr(struct bnxt_softc *softc, bool shutdown) { struct hwrm_func_drv_unrgtr_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_DRV_UNRGTR); if (shutdown == true) req.flags |= HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN; return hwrm_send_message(softc, &req, sizeof(req)); } static inline int _is_valid_ether_addr(uint8_t *addr) { char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) return (FALSE); return (TRUE); } static inline void get_random_ether_addr(uint8_t *addr) { uint8_t temp[ETHER_ADDR_LEN]; arc4rand(&temp, sizeof(temp), 0); temp[0] &= 0xFE; temp[0] |= 0x02; bcopy(temp, addr, sizeof(temp)); } int bnxt_hwrm_func_qcaps(struct bnxt_softc *softc) { int rc = 0; struct hwrm_func_qcaps_input req = {0}; struct hwrm_func_qcaps_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct bnxt_func_info *func = &softc->func; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS); req.fid = htole16(0xffff); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; if (resp->flags & htole32(HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED)) softc->flags |= BNXT_FLAG_WOL_CAP; func->fw_fid = le16toh(resp->fid); memcpy(func->mac_addr, resp->mac_address, ETHER_ADDR_LEN); func->max_rsscos_ctxs = le16toh(resp->max_rsscos_ctx); func->max_cp_rings = le16toh(resp->max_cmpl_rings); func->max_tx_rings = le16toh(resp->max_tx_rings); func->max_rx_rings = le16toh(resp->max_rx_rings); func->max_hw_ring_grps = le32toh(resp->max_hw_ring_grps); if (!func->max_hw_ring_grps) func->max_hw_ring_grps = func->max_tx_rings; func->max_l2_ctxs = le16toh(resp->max_l2_ctxs); func->max_vnics = le16toh(resp->max_vnics); func->max_stat_ctxs = le16toh(resp->max_stat_ctx); if (BNXT_PF(softc)) { struct bnxt_pf_info *pf = &softc->pf; pf->port_id = le16toh(resp->port_id); pf->first_vf_id = le16toh(resp->first_vf_id); pf->max_vfs = le16toh(resp->max_vfs); pf->max_encap_records = le32toh(resp->max_encap_records); pf->max_decap_records = le32toh(resp->max_decap_records); pf->max_tx_em_flows = le32toh(resp->max_tx_em_flows); pf->max_tx_wm_flows = le32toh(resp->max_tx_wm_flows); pf->max_rx_em_flows = le32toh(resp->max_rx_em_flows); pf->max_rx_wm_flows = le32toh(resp->max_rx_wm_flows); } if (!_is_valid_ether_addr(func->mac_addr)) { device_printf(softc->dev, "Invalid ethernet address, generating random locally administered address\n"); get_random_ether_addr(func->mac_addr); } fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_reset(struct bnxt_softc *softc) { struct hwrm_func_reset_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_RESET); req.enables = 0; return hwrm_send_message(softc, &req, sizeof(req)); } static void bnxt_hwrm_set_link_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { uint8_t autoneg = softc->link_info.autoneg; uint16_t fw_link_speed = softc->link_info.req_link_speed; if (autoneg & BNXT_AUTONEG_SPEED) { req->auto_mode |= HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ALL_SPEEDS; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE); req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG); } else { req->force_link_speed = htole16(fw_link_speed); req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE); } /* tell chimp that the setting takes effect immediately */ req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY); } static void bnxt_hwrm_set_pause_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { if (softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { req->auto_pause = HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE; if (softc->link_info.req_flow_ctrl & HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX) req->auto_pause |= HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX; if (softc->link_info.req_flow_ctrl & HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX) req->auto_pause |= HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE); } else { if (softc->link_info.req_flow_ctrl & HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX) req->force_pause |= HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX; if (softc->link_info.req_flow_ctrl & HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX) req->force_pause |= HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE); req->auto_pause = req->force_pause; req->enables |= htole32( HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE); } } /* JFV this needs interface connection */ static void bnxt_hwrm_set_eee(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { /* struct ethtool_eee *eee = &softc->eee; */ bool eee_enabled = false; if (eee_enabled) { #if 0 uint16_t eee_speeds; uint32_t flags = HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_ENABLE; if (eee->tx_lpi_enabled) flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI; req->flags |= htole32(flags); eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); req->eee_link_speed_mask = htole16(eee_speeds); req->tx_lpi_timer = htole32(eee->tx_lpi_timer); #endif } else { req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_DISABLE); } } int bnxt_hwrm_set_link_setting(struct bnxt_softc *softc, bool set_pause, bool set_eee) { struct hwrm_port_phy_cfg_input req = {0}; if (softc->flags & BNXT_FLAG_NPAR) return ENOTSUP; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_CFG); if (set_pause) bnxt_hwrm_set_pause_common(softc, &req); bnxt_hwrm_set_link_common(softc, &req); if (set_eee) bnxt_hwrm_set_eee(softc, &req); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_set_pause(struct bnxt_softc *softc) { struct hwrm_port_phy_cfg_input req = {0}; int rc; if (softc->flags & BNXT_FLAG_NPAR) return ENOTSUP; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_CFG); bnxt_hwrm_set_pause_common(softc, &req); if (softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) bnxt_hwrm_set_link_common(softc, &req); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (!rc && !(softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { /* since changing of pause setting doesn't trigger any link * change event, the driver needs to update the current pause * result upon successfully return of the phy_cfg command */ softc->link_info.pause = softc->link_info.force_pause = softc->link_info.req_flow_ctrl; softc->link_info.auto_pause = 0; bnxt_report_link(softc); } BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_vnic_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_cfg_input req = {0}; struct hwrm_vnic_cfg_output *resp; resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_CFG); if (vnic->flags & BNXT_VNIC_FLAG_DEFAULT) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT); if (vnic->flags & BNXT_VNIC_FLAG_BD_STALL) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE); if (vnic->flags & BNXT_VNIC_FLAG_VLAN_STRIP) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE); req.enables = htole32(HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP | HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE | HWRM_VNIC_CFG_INPUT_ENABLES_MRU); req.vnic_id = htole16(vnic->id); req.dflt_ring_grp = htole16(vnic->def_ring_grp); req.rss_rule = htole16(vnic->rss_id); req.cos_rule = htole16(vnic->cos_rule); req.lb_rule = htole16(vnic->lb_rule); req.mru = htole16(vnic->mru); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_vnic_alloc(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_alloc_input req = {0}; struct hwrm_vnic_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; if (vnic->id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate vnic %04x\n", vnic->id); return EDOOFUS; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_ALLOC); if (vnic->flags & BNXT_VNIC_FLAG_DEFAULT) req.flags = htole32(HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; vnic->id = le32toh(resp->vnic_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_vnic_ctx_alloc(struct bnxt_softc *softc, uint16_t *ctx_id) { struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; if (*ctx_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate vnic ctx %04x\n", *ctx_id); return EDOOFUS; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; *ctx_id = le32toh(resp->rss_cos_lb_ctx_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_ring_grp_alloc(struct bnxt_softc *softc, struct bnxt_grp_info *grp) { struct hwrm_ring_grp_alloc_input req = {0}; struct hwrm_ring_grp_alloc_output *resp; int rc = 0; if (grp->grp_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate ring group %04x\n", grp->grp_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_GRP_ALLOC); req.cr = htole16(grp->cp_ring_id); req.rr = htole16(grp->rx_ring_id); req.ar = htole16(grp->ag_ring_id); req.sc = htole16(grp->stats_ctx); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; grp->grp_id = le32toh(resp->ring_group_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } /* * Ring allocation message to the firmware */ int bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type, struct bnxt_ring *ring, uint16_t cmpl_ring_id, uint32_t stat_ctx_id, bool irq) { struct hwrm_ring_alloc_input req = {0}; struct hwrm_ring_alloc_output *resp; int rc; if (ring->phys_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate ring %04x\n", ring->phys_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_ALLOC); req.enables = htole32(0); req.fbo = htole32(0); if (stat_ctx_id != HWRM_NA_SIGNATURE) { req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); req.stat_ctx_id = htole32(stat_ctx_id); } req.ring_type = type; req.page_tbl_addr = htole64(ring->paddr); req.length = htole32(ring->ring_size); req.logical_id = htole16(ring->id); req.cmpl_ring_id = htole16(cmpl_ring_id); req.queue_id = htole16(softc->q_info[0].id); #if 0 /* MODE_POLL appears to crash the firmware */ if (irq) req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; else req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_POLL; #else req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; #endif BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; ring->phys_id = le16toh(resp->ring_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_stat_ctx_alloc(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr, uint64_t paddr) { struct hwrm_stat_ctx_alloc_input req = {0}; struct hwrm_stat_ctx_alloc_output *resp; int rc = 0; if (cpr->stats_ctx_id != HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate stats ctx %08x\n", cpr->stats_ctx_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_STAT_CTX_ALLOC); req.update_period_ms = htole32(1000); req.stats_dma_addr = htole64(paddr); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; cpr->stats_ctx_id = le32toh(resp->stat_ctx_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_cfa_l2_set_rx_mask_input req = {0}; struct bnxt_vlan_tag *tag; uint32_t *tags; uint32_t num_vlan_tags = 0;; uint32_t i; uint32_t mask = vnic->rx_mask; int rc; SLIST_FOREACH(tag, &vnic->vlan_tags, next) num_vlan_tags++; if (num_vlan_tags) { if (!(mask & HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN)) { if (!vnic->vlan_only) mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN; else mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY; } if (vnic->vlan_tag_list.idi_vaddr) { iflib_dma_free(&vnic->vlan_tag_list); vnic->vlan_tag_list.idi_vaddr = NULL; } rc = iflib_dma_alloc(softc->ctx, 4 * num_vlan_tags, &vnic->vlan_tag_list, BUS_DMA_NOWAIT); if (rc) return rc; tags = (uint32_t *)vnic->vlan_tag_list.idi_vaddr; i = 0; SLIST_FOREACH(tag, &vnic->vlan_tags, next) { tags[i] = htole32((tag->tpid << 16) | tag->tag); i++; } } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_CFA_L2_SET_RX_MASK); req.vnic_id = htole32(vnic->id); req.mask = htole32(mask); req.mc_tbl_addr = htole64(vnic->mc_list.idi_paddr); req.num_mc_entries = htole32(vnic->mc_list_count); req.vlan_tag_tbl_addr = htole64(vnic->vlan_tag_list.idi_paddr); req.num_vlan_tags = htole32(num_vlan_tags); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_set_filter(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_cfa_l2_filter_alloc_input req = {0}; struct hwrm_cfa_l2_filter_alloc_output *resp; uint32_t enables = 0; int rc = 0; if (vnic->filter_id != -1) { device_printf(softc->dev, "Attempt to re-allocate l2 ctx filter\n"); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_CFA_L2_FILTER_ALLOC); req.flags = htole32(HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX); enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID; req.enables = htole32(enables); req.dst_id = htole16(vnic->id); memcpy(req.l2_addr, if_getlladdr(iflib_get_ifp(softc->ctx)), ETHER_ADDR_LEN); memset(&req.l2_addr_mask, 0xff, sizeof(req.l2_addr_mask)); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; vnic->filter_id = le64toh(resp->l2_filter_id); vnic->flow_id = le64toh(resp->flow_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_rss_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic, uint32_t hash_type) { struct hwrm_vnic_rss_cfg_input req = {0}; struct hwrm_vnic_rss_cfg_output *resp; resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_RSS_CFG); req.hash_type = htole32(hash_type); req.ring_grp_tbl_addr = htole64(vnic->rss_grp_tbl.idi_paddr); req.hash_key_tbl_addr = htole64(vnic->rss_hash_key_tbl.idi_paddr); req.rss_ctx_idx = htole16(vnic->rss_id); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_func_cfg(struct bnxt_softc *softc) { struct hwrm_func_cfg_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG); req.fid = 0xffff; req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); req.async_event_cr = softc->def_cp_ring.ring.phys_id; return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic, uint32_t flags) { struct hwrm_vnic_tpa_cfg_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_TPA_CFG); req.flags = htole32(flags); req.vnic_id = htole16(vnic->id); req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS | HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS | /* HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER | */ HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN); /* TODO: Calculate this based on ring size? */ req.max_agg_segs = htole16(3); /* Base this in the allocated TPA start size... */ req.max_aggs = htole16(7); /* * TODO: max_agg_timer? * req.mag_agg_timer = htole32(XXX); */ req.min_agg_len = htole32(0); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type, uint16_t *ordinal, uint16_t ext, uint16_t *index, bool use_index, uint8_t search_opt, uint32_t *data_length, uint32_t *item_length, uint32_t *fw_ver) { struct hwrm_nvm_find_dir_entry_input req = {0}; struct hwrm_nvm_find_dir_entry_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; uint32_t old_timeo; MPASS(ordinal); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_FIND_DIR_ENTRY); if (use_index) { req.enables = htole32( HWRM_NVM_FIND_DIR_ENTRY_INPUT_ENABLES_DIR_IDX_VALID); req.dir_idx = htole16(*index); } req.dir_type = htole16(type); req.dir_ordinal = htole16(*ordinal); req.dir_ext = htole16(ext); req.opt_ordinal = search_opt; BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (item_length) *item_length = le32toh(resp->dir_item_length); if (data_length) *data_length = le32toh(resp->dir_data_length); if (fw_ver) *fw_ver = le32toh(resp->fw_ver); *ordinal = le16toh(resp->dir_ordinal); if (index) *index = le16toh(resp->dir_idx); exit: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_nvm_read(struct bnxt_softc *softc, uint16_t index, uint32_t offset, uint32_t length, struct iflib_dma_info *data) { struct hwrm_nvm_read_input req = {0}; int rc; uint32_t old_timeo; if (length > data->idi_size) { rc = EINVAL; goto exit; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_READ); req.host_dest_addr = htole64(data->idi_paddr); req.dir_idx = htole16(index); req.offset = htole32(offset); req.len = htole32(length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); if (rc) goto exit; bus_dmamap_sync(data->idi_tag, data->idi_map, BUS_DMASYNC_POSTREAD); goto exit; exit: return rc; } int bnxt_hwrm_nvm_modify(struct bnxt_softc *softc, uint16_t index, uint32_t offset, void *data, bool cpyin, uint32_t length) { struct hwrm_nvm_modify_input req = {0}; struct iflib_dma_info dma_data; int rc; uint32_t old_timeo; if (length == 0 || !data) return EINVAL; rc = iflib_dma_alloc(softc->ctx, length, &dma_data, BUS_DMA_NOWAIT); if (rc) return ENOMEM; if (cpyin) { rc = copyin(data, dma_data.idi_vaddr, length); if (rc) goto exit; } else memcpy(dma_data.idi_vaddr, data, length); bus_dmamap_sync(dma_data.idi_tag, dma_data.idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_MODIFY); req.host_src_addr = htole64(dma_data.idi_paddr); req.dir_idx = htole16(index); req.offset = htole32(offset); req.len = htole32(length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); exit: iflib_dma_free(&dma_data); return rc; } int bnxt_hwrm_fw_reset(struct bnxt_softc *softc, uint8_t processor, uint8_t *selfreset) { struct hwrm_fw_reset_input req = {0}; struct hwrm_fw_reset_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; MPASS(selfreset); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_RESET); req.embedded_proc_type = processor; req.selfrst_status = *selfreset; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; *selfreset = resp->selfrst_status; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_qstatus(struct bnxt_softc *softc, uint8_t type, uint8_t *selfreset) { struct hwrm_fw_qstatus_input req = {0}; struct hwrm_fw_qstatus_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; MPASS(selfreset); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_QSTATUS); req.embedded_proc_type = type; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; *selfreset = resp->selfrst_status; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_write(struct bnxt_softc *softc, void *data, bool cpyin, uint16_t type, uint16_t ordinal, uint16_t ext, uint16_t attr, uint16_t option, uint32_t data_length, bool keep, uint32_t *item_length, uint16_t *index) { struct hwrm_nvm_write_input req = {0}; struct hwrm_nvm_write_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct iflib_dma_info dma_data; int rc; uint32_t old_timeo; if (data_length) { rc = iflib_dma_alloc(softc->ctx, data_length, &dma_data, BUS_DMA_NOWAIT); if (rc) return ENOMEM; if (cpyin) { rc = copyin(data, dma_data.idi_vaddr, data_length); if (rc) goto early_exit; } else memcpy(dma_data.idi_vaddr, data, data_length); bus_dmamap_sync(dma_data.idi_tag, dma_data.idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } else dma_data.idi_paddr = 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_WRITE); req.host_src_addr = htole64(dma_data.idi_paddr); req.dir_type = htole16(type); req.dir_ordinal = htole16(ordinal); req.dir_ext = htole16(ext); req.dir_attr = htole16(attr); req.dir_data_length = htole32(data_length); req.option = htole16(option); if (keep) { req.flags = htole16(HWRM_NVM_WRITE_INPUT_FLAGS_KEEP_ORIG_ACTIVE_IMG); } if (item_length) req.dir_item_length = htole32(*item_length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (item_length) *item_length = le32toh(resp->dir_item_length); if (index) *index = le16toh(resp->dir_idx); exit: BNXT_HWRM_UNLOCK(softc); early_exit: if (data_length) iflib_dma_free(&dma_data); return rc; } int bnxt_hwrm_nvm_erase_dir_entry(struct bnxt_softc *softc, uint16_t index) { struct hwrm_nvm_erase_dir_entry_input req = {0}; uint32_t old_timeo; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_ERASE_DIR_ENTRY); req.dir_idx = htole16(index); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_get_dir_info(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length) { struct hwrm_nvm_get_dir_info_input req = {0}; struct hwrm_nvm_get_dir_info_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DIR_INFO); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (entries) *entries = le32toh(resp->entries); if (entry_length) *entry_length = le32toh(resp->entry_length); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_get_dir_entries(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length, struct iflib_dma_info *dma_data) { struct hwrm_nvm_get_dir_entries_input req = {0}; uint32_t ent; uint32_t ent_len; int rc; uint32_t old_timeo; if (!entries) entries = &ent; if (!entry_length) entry_length = &ent_len; rc = bnxt_hwrm_nvm_get_dir_info(softc, entries, entry_length); if (rc) goto exit; if (*entries * *entry_length > dma_data->idi_size) { rc = EINVAL; goto exit; } /* * TODO: There's a race condition here that could blow up DMA memory... * we need to allocate the max size, not the currently in use * size. The command should totally have a max size here. */ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DIR_ENTRIES); req.host_dest_addr = htole64(dma_data->idi_paddr); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); if (rc) goto exit; bus_dmamap_sync(dma_data->idi_tag, dma_data->idi_map, BUS_DMASYNC_POSTWRITE); exit: return rc; } int bnxt_hwrm_nvm_get_dev_info(struct bnxt_softc *softc, uint16_t *mfg_id, uint16_t *device_id, uint32_t *sector_size, uint32_t *nvram_size, uint32_t *reserved_size, uint32_t *available_size) { struct hwrm_nvm_get_dev_info_input req = {0}; struct hwrm_nvm_get_dev_info_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DEV_INFO); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (mfg_id) *mfg_id = le16toh(resp->manufacturer_id); if (device_id) *device_id = le16toh(resp->device_id); if (sector_size) *sector_size = le32toh(resp->sector_size); if (nvram_size) *nvram_size = le32toh(resp->nvram_size); if (reserved_size) *reserved_size = le32toh(resp->reserved_size); if (available_size) *available_size = le32toh(resp->available_size); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_install_update(struct bnxt_softc *softc, uint32_t install_type, uint64_t *installed_items, uint8_t *result, uint8_t *problem_item, uint8_t *reset_required) { struct hwrm_nvm_install_update_input req = {0}; struct hwrm_nvm_install_update_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_INSTALL_UPDATE); req.install_type = htole32(install_type); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (installed_items) *installed_items = le32toh(resp->installed_items); if (result) *result = resp->result; if (problem_item) *problem_item = resp->problem_item; if (reset_required) *reset_required = resp->reset_required; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_verify_update(struct bnxt_softc *softc, uint16_t type, uint16_t ordinal, uint16_t ext) { struct hwrm_nvm_verify_update_input req = {0}; uint32_t old_timeo; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_VERIFY_UPDATE); req.dir_type = htole16(type); req.dir_ordinal = htole16(ordinal); req.dir_ext = htole16(ext); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_get_time(struct bnxt_softc *softc, uint16_t *year, uint8_t *month, uint8_t *day, uint8_t *hour, uint8_t *minute, uint8_t *second, uint16_t *millisecond, uint16_t *zone) { struct hwrm_fw_get_time_input req = {0}; struct hwrm_fw_get_time_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_GET_TIME); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; if (year) *year = le16toh(resp->year); if (month) *month = resp->month; if (day) *day = resp->day; if (hour) *hour = resp->hour; if (minute) *minute = resp->minute; if (second) *second = resp->second; if (millisecond) *millisecond = le16toh(resp->millisecond); if (zone) *zone = le16toh(resp->zone); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_set_time(struct bnxt_softc *softc, uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second, uint16_t millisecond, uint16_t zone) { struct hwrm_fw_set_time_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_SET_TIME); req.year = htole16(year); req.month = month; req.day = day; req.hour = hour; req.minute = minute; req.second = second; req.millisecond = htole16(millisecond); req.zone = htole16(zone); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_port_phy_qcfg(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; struct hwrm_port_phy_qcfg_input req = {0}; struct hwrm_port_phy_qcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; BNXT_HWRM_LOCK(softc); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_QCFG); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; link_info->phy_link_status = resp->link; link_info->duplex = resp->duplex; link_info->pause = resp->pause; link_info->auto_mode = resp->auto_mode; link_info->auto_pause = resp->auto_pause; link_info->force_pause = resp->force_pause; link_info->duplex_setting = resp->duplex; if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) link_info->link_speed = le16toh(resp->link_speed); else link_info->link_speed = 0; link_info->force_link_speed = le16toh(resp->force_link_speed); link_info->auto_link_speed = le16toh(resp->auto_link_speed); link_info->support_speeds = le16toh(resp->support_speeds); link_info->auto_link_speeds = le16toh(resp->auto_link_speed_mask); link_info->preemphasis = le32toh(resp->preemphasis); link_info->phy_ver[0] = resp->phy_maj; link_info->phy_ver[1] = resp->phy_min; link_info->phy_ver[2] = resp->phy_bld; snprintf(softc->ver_info->phy_ver, sizeof(softc->ver_info->phy_ver), "%d.%d.%d", link_info->phy_ver[0], link_info->phy_ver[1], link_info->phy_ver[2]); strlcpy(softc->ver_info->phy_vendor, resp->phy_vendor_name, BNXT_NAME_SIZE); strlcpy(softc->ver_info->phy_partnumber, resp->phy_vendor_partnumber, BNXT_NAME_SIZE); link_info->media_type = resp->media_type; link_info->phy_type = resp->phy_type; link_info->transceiver = resp->xcvr_pkg_type; link_info->phy_addr = resp->eee_config_phy_addr & HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_MASK; exit: BNXT_HWRM_UNLOCK(softc); return rc; } uint16_t bnxt_hwrm_get_wol_fltrs(struct bnxt_softc *softc, uint16_t handle) { struct hwrm_wol_filter_qcfg_input req = {0}; struct hwrm_wol_filter_qcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; uint16_t next_handle = 0; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_QCFG); req.port_id = htole16(softc->pf.port_id); req.handle = htole16(handle); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { next_handle = le16toh(resp->next_handle); if (next_handle != 0) { if (resp->wol_type == HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT) { softc->wol = 1; softc->wol_filter_id = resp->wol_filter_id; } } } return next_handle; } int bnxt_hwrm_alloc_wol_fltr(struct bnxt_softc *softc) { struct hwrm_wol_filter_alloc_input req = {0}; struct hwrm_wol_filter_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_ALLOC); req.port_id = htole16(softc->pf.port_id); req.wol_type = HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT; req.enables = htole32(HWRM_WOL_FILTER_ALLOC_INPUT_ENABLES_MAC_ADDRESS); memcpy(req.mac_address, softc->func.mac_addr, ETHER_ADDR_LEN); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) softc->wol_filter_id = resp->wol_filter_id; return rc; } int bnxt_hwrm_free_wol_fltr(struct bnxt_softc *softc) { struct hwrm_wol_filter_free_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_FREE); req.port_id = htole16(softc->pf.port_id); req.enables = htole32(HWRM_WOL_FILTER_FREE_INPUT_ENABLES_WOL_FILTER_ID); req.wol_filter_id = softc->wol_filter_id; return hwrm_send_message(softc, &req, sizeof(req)); } + +static void bnxt_hwrm_set_coal_params(struct bnxt_softc *softc, uint32_t max_frames, + uint32_t buf_tmrs, uint16_t flags, + struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) +{ + req->flags = htole16(flags); + req->num_cmpl_dma_aggr = htole16((uint16_t)max_frames); + req->num_cmpl_dma_aggr_during_int = htole16(max_frames >> 16); + req->cmpl_aggr_dma_tmr = htole16((uint16_t)buf_tmrs); + req->cmpl_aggr_dma_tmr_during_int = htole16(buf_tmrs >> 16); + /* Minimum time between 2 interrupts set to buf_tmr x 2 */ + req->int_lat_tmr_min = htole16((uint16_t)buf_tmrs * 2); + req->int_lat_tmr_max = htole16((uint16_t)buf_tmrs * 4); + req->num_cmpl_aggr_int = htole16((uint16_t)max_frames * 4); +} + + +int bnxt_hwrm_set_coal(struct bnxt_softc *softc) +{ + int i, rc = 0; + struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, + req_tx = {0}, *req; + uint16_t max_buf, max_buf_irq; + uint16_t buf_tmr, buf_tmr_irq; + uint32_t flags; + + bnxt_hwrm_cmd_hdr_init(softc, &req_rx, + HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); + bnxt_hwrm_cmd_hdr_init(softc, &req_tx, + HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); + + /* Each rx completion (2 records) should be DMAed immediately. + * DMA 1/4 of the completion buffers at a time. + */ + max_buf = min_t(uint16_t, softc->rx_coal_frames / 4, 2); + /* max_buf must not be zero */ + max_buf = clamp_t(uint16_t, max_buf, 1, 63); + max_buf_irq = clamp_t(uint16_t, softc->rx_coal_frames_irq, 1, 63); + buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs); + /* buf timer set to 1/4 of interrupt timer */ + buf_tmr = max_t(uint16_t, buf_tmr / 4, 1); + buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs_irq); + buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1); + + flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET; + + /* RING_IDLE generates more IRQs for lower latency. Enable it only + * if coal_usecs is less than 25 us. + */ + if (softc->rx_coal_usecs < 25) + flags |= HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_RING_IDLE; + + bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 | max_buf, + buf_tmr_irq << 16 | buf_tmr, flags, &req_rx); + + /* max_buf must not be zero */ + max_buf = clamp_t(uint16_t, softc->tx_coal_frames, 1, 63); + max_buf_irq = clamp_t(uint16_t, softc->tx_coal_frames_irq, 1, 63); + buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs); + /* buf timer set to 1/4 of interrupt timer */ + buf_tmr = max_t(uint16_t, buf_tmr / 4, 1); + buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs_irq); + buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1); + flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET; + bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 | max_buf, + buf_tmr_irq << 16 | buf_tmr, flags, &req_tx); + + for (i = 0; i < softc->nrxqsets; i++) { + + + req = &req_rx; + /* + * TBD: + * Check if Tx also needs to be done + * So far, Tx processing has been done in softirq contest + * + * req = &req_tx; + */ + req->ring_id = htole16(softc->grp_info[i].cp_ring_id); + + rc = hwrm_send_message(softc, req, sizeof(*req)); + if (rc) + break; + } + return rc; +} + + + +int bnxt_hwrm_func_rgtr_async_events(struct bnxt_softc *softc, unsigned long *bmap, + int bmap_size) +{ + struct hwrm_func_drv_rgtr_input req = {0}; + bitstr_t *async_events_bmap; + uint32_t *events; + int i; + + async_events_bmap = bit_alloc(256, M_DEVBUF, M_WAITOK|M_ZERO); + events = (uint32_t *)async_events_bmap; + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_DRV_RGTR); + + req.enables = + htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); + + memset(async_events_bmap, 0, sizeof(256 / 8)); + + bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE); + bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD); + bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED); + bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE); + bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE); + + if (bmap && bmap_size) { + for (i = 0; i < bmap_size; i++) { + if (bit_test(bmap, i)) + bit_set(async_events_bmap, i); + } + } + + for (i = 0; i < 8; i++) + req.async_event_fwd[i] |= htole32(events[i]); + + free(async_events_bmap, M_DEVBUF); + + return hwrm_send_message(softc, &req, sizeof(req)); +} + Index: projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.h =================================================================== --- projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.h (revision 321584) +++ projects/runtime-coverage/sys/dev/bnxt/bnxt_hwrm.h (revision 321585) @@ -1,105 +1,107 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifndef _BNXT_HWRM_H #define _BNXT_HWRM_H /* HWRM Function Prototypes */ int bnxt_alloc_hwrm_dma_mem(struct bnxt_softc *softc); void bnxt_free_hwrm_dma_mem(struct bnxt_softc *softc); int bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type, struct bnxt_ring *ring, uint16_t cmpl_ring_id, uint32_t stat_ctx_id, bool irq); int bnxt_hwrm_ver_get(struct bnxt_softc *softc); int bnxt_hwrm_queue_qportcfg(struct bnxt_softc *softc); int bnxt_hwrm_func_drv_rgtr(struct bnxt_softc *softc); int bnxt_hwrm_func_drv_unrgtr(struct bnxt_softc *softc, bool shutdown); int bnxt_hwrm_func_qcaps(struct bnxt_softc *softc); int bnxt_hwrm_func_reset(struct bnxt_softc *softc); int bnxt_hwrm_set_link_setting(struct bnxt_softc *, bool set_pause, bool set_eee); int bnxt_hwrm_set_pause(struct bnxt_softc *softc); int bnxt_hwrm_vnic_ctx_alloc(struct bnxt_softc *softc, uint16_t *ctx_id); int bnxt_hwrm_vnic_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic); int bnxt_hwrm_stat_ctx_alloc(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr, uint64_t paddr); int bnxt_hwrm_ring_grp_alloc(struct bnxt_softc *softc, struct bnxt_grp_info *grp); int bnxt_hwrm_vnic_alloc(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic); int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic); int bnxt_hwrm_set_filter(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic); int bnxt_hwrm_rss_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic, uint32_t hash_type); int bnxt_hwrm_func_cfg(struct bnxt_softc *softc); int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic, uint32_t flags); int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type, uint16_t *ordinal, uint16_t ext, uint16_t *index, bool use_index, uint8_t search_opt, uint32_t *data_length, uint32_t *item_length, uint32_t *fw_ver); int bnxt_hwrm_nvm_read(struct bnxt_softc *softc, uint16_t index, uint32_t offset, uint32_t length, struct iflib_dma_info *data); int bnxt_hwrm_nvm_modify(struct bnxt_softc *softc, uint16_t index, uint32_t offset, void *data, bool cpyin, uint32_t length); int bnxt_hwrm_fw_reset(struct bnxt_softc *softc, uint8_t processor, uint8_t *selfreset); int bnxt_hwrm_fw_qstatus(struct bnxt_softc *softc, uint8_t type, uint8_t *selfreset); int bnxt_hwrm_nvm_write(struct bnxt_softc *softc, void *data, bool cpyin, uint16_t type, uint16_t ordinal, uint16_t ext, uint16_t attr, uint16_t option, uint32_t data_length, bool keep, uint32_t *item_length, uint16_t *index); int bnxt_hwrm_nvm_erase_dir_entry(struct bnxt_softc *softc, uint16_t index); int bnxt_hwrm_nvm_get_dir_info(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length); int bnxt_hwrm_nvm_get_dir_entries(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length, struct iflib_dma_info *dma_data); int bnxt_hwrm_nvm_get_dev_info(struct bnxt_softc *softc, uint16_t *mfg_id, uint16_t *device_id, uint32_t *sector_size, uint32_t *nvram_size, uint32_t *reserved_size, uint32_t *available_size); int bnxt_hwrm_nvm_install_update(struct bnxt_softc *softc, uint32_t install_type, uint64_t *installed_items, uint8_t *result, uint8_t *problem_item, uint8_t *reset_required); int bnxt_hwrm_nvm_verify_update(struct bnxt_softc *softc, uint16_t type, uint16_t ordinal, uint16_t ext); int bnxt_hwrm_fw_get_time(struct bnxt_softc *softc, uint16_t *year, uint8_t *month, uint8_t *day, uint8_t *hour, uint8_t *minute, uint8_t *second, uint16_t *millisecond, uint16_t *zone); int bnxt_hwrm_fw_set_time(struct bnxt_softc *softc, uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second, uint16_t millisecond, uint16_t zone); int bnxt_hwrm_port_phy_qcfg(struct bnxt_softc *softc); uint16_t bnxt_hwrm_get_wol_fltrs(struct bnxt_softc *softc, uint16_t handle); int bnxt_hwrm_alloc_wol_fltr(struct bnxt_softc *softc); int bnxt_hwrm_free_wol_fltr(struct bnxt_softc *softc); - +int bnxt_hwrm_set_coal(struct bnxt_softc *softc); +int bnxt_hwrm_func_rgtr_async_events(struct bnxt_softc *softc, unsigned long *bmap, + int bmap_size); #endif Index: projects/runtime-coverage/sys/dev/bnxt/bnxt_sysctl.c =================================================================== --- projects/runtime-coverage/sys/dev/bnxt/bnxt_sysctl.c (revision 321584) +++ projects/runtime-coverage/sys/dev/bnxt/bnxt_sysctl.c (revision 321585) @@ -1,618 +1,803 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_sysctl.h" static int bnxt_vlan_only_sysctl(SYSCTL_HANDLER_ARGS); /* * We want to create: * dev.bnxt.0.hwstats.txq0 * dev.bnxt.0.hwstats.txq0.txmbufs * dev.bnxt.0.hwstats.rxq0 * dev.bnxt.0.hwstats.txq0.rxmbufs * so the hwstats ctx list needs to be created in attach_post and populated * during init. * * Then, it needs to be cleaned up in stop. */ int bnxt_init_sysctl_ctx(struct bnxt_softc *softc) { struct sysctl_ctx_list *ctx; sysctl_ctx_init(&softc->hw_stats); ctx = device_get_sysctl_ctx(softc->dev); softc->hw_stats_oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO, "hwstats", CTLFLAG_RD, 0, "hardware statistics"); if (!softc->hw_stats_oid) { sysctl_ctx_free(&softc->hw_stats); return ENOMEM; } sysctl_ctx_init(&softc->ver_info->ver_ctx); ctx = device_get_sysctl_ctx(softc->dev); softc->ver_info->ver_oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO, "ver", CTLFLAG_RD, 0, "hardware/firmware version information"); if (!softc->ver_info->ver_oid) { sysctl_ctx_free(&softc->ver_info->ver_ctx); return ENOMEM; } sysctl_ctx_init(&softc->nvm_info->nvm_ctx); ctx = device_get_sysctl_ctx(softc->dev); softc->nvm_info->nvm_oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO, "nvram", CTLFLAG_RD, 0, "nvram information"); if (!softc->nvm_info->nvm_oid) { sysctl_ctx_free(&softc->nvm_info->nvm_ctx); return ENOMEM; } return 0; } int bnxt_free_sysctl_ctx(struct bnxt_softc *softc) { int orc; int rc = 0; if (softc->hw_stats_oid != NULL) { orc = sysctl_ctx_free(&softc->hw_stats); if (orc) rc = orc; else softc->hw_stats_oid = NULL; } if (softc->ver_info->ver_oid != NULL) { orc = sysctl_ctx_free(&softc->ver_info->ver_ctx); if (orc) rc = orc; else softc->ver_info->ver_oid = NULL; } if (softc->nvm_info->nvm_oid != NULL) { orc = sysctl_ctx_free(&softc->nvm_info->nvm_ctx); if (orc) rc = orc; else softc->nvm_info->nvm_oid = NULL; } return rc; } int bnxt_create_tx_sysctls(struct bnxt_softc *softc, int txr) { struct sysctl_oid *oid; struct ctx_hw_stats *tx_stats = (void *)softc->tx_stats.idi_vaddr; char name[32]; char desc[64]; sprintf(name, "txq%d", txr); sprintf(desc, "transmit queue %d", txr); oid = SYSCTL_ADD_NODE(&softc->hw_stats, SYSCTL_CHILDREN(softc->hw_stats_oid), OID_AUTO, name, CTLFLAG_RD, 0, desc); if (!oid) return ENOMEM; SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "ucast_pkts", CTLFLAG_RD, &tx_stats[txr].tx_ucast_pkts, "unicast packets sent"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "mcast_pkts", CTLFLAG_RD, &tx_stats[txr].tx_mcast_pkts, "multicast packets sent"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "bcast_pkts", CTLFLAG_RD, &tx_stats[txr].tx_bcast_pkts, "broadcast packets sent"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "discard_pkts", CTLFLAG_RD, &tx_stats[txr].tx_discard_pkts, "discarded transmit packets"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "drop_pkts", CTLFLAG_RD, &tx_stats[txr].tx_drop_pkts, "dropped transmit packets"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "ucast_bytes", CTLFLAG_RD, &tx_stats[txr].tx_ucast_bytes, "unicast bytes sent"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "mcast_bytes", CTLFLAG_RD, &tx_stats[txr].tx_mcast_bytes, "multicast bytes sent"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "bcast_bytes", CTLFLAG_RD, &tx_stats[txr].tx_bcast_bytes, "broadcast bytes sent"); return 0; } int bnxt_create_rx_sysctls(struct bnxt_softc *softc, int rxr) { struct sysctl_oid *oid; struct ctx_hw_stats *rx_stats = (void *)softc->rx_stats.idi_vaddr; char name[32]; char desc[64]; sprintf(name, "rxq%d", rxr); sprintf(desc, "receive queue %d", rxr); oid = SYSCTL_ADD_NODE(&softc->hw_stats, SYSCTL_CHILDREN(softc->hw_stats_oid), OID_AUTO, name, CTLFLAG_RD, 0, desc); if (!oid) return ENOMEM; SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "ucast_pkts", CTLFLAG_RD, &rx_stats[rxr].rx_ucast_pkts, "unicast packets received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "mcast_pkts", CTLFLAG_RD, &rx_stats[rxr].rx_mcast_pkts, "multicast packets received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "bcast_pkts", CTLFLAG_RD, &rx_stats[rxr].rx_bcast_pkts, "broadcast packets received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "discard_pkts", CTLFLAG_RD, &rx_stats[rxr].rx_discard_pkts, "discarded receive packets"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "drop_pkts", CTLFLAG_RD, &rx_stats[rxr].rx_drop_pkts, "dropped receive packets"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "ucast_bytes", CTLFLAG_RD, &rx_stats[rxr].rx_ucast_bytes, "unicast bytes received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "mcast_bytes", CTLFLAG_RD, &rx_stats[rxr].rx_mcast_bytes, "multicast bytes received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "bcast_bytes", CTLFLAG_RD, &rx_stats[rxr].rx_bcast_bytes, "broadcast bytes received"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "tpa_pkts", CTLFLAG_RD, &rx_stats[rxr].tpa_pkts, "TPA packets"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "tpa_bytes", CTLFLAG_RD, &rx_stats[rxr].tpa_bytes, "TPA bytes"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "tpa_events", CTLFLAG_RD, &rx_stats[rxr].tpa_events, "TPA events"); SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO, "tpa_aborts", CTLFLAG_RD, &rx_stats[rxr].tpa_aborts, "TPA aborts"); return 0; } static char *bnxt_chip_type[] = { "ASIC", "FPGA", "Palladium", "Unknown" }; #define MAX_CHIP_TYPE 3 static int bnxt_package_ver_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; struct iflib_dma_info dma_data; char *pkglog = NULL; char *p; char *next; char unk[] = ""; char *buf = unk; int rc; int field; uint16_t ordinal = BNX_DIR_ORDINAL_FIRST; uint16_t index; uint32_t data_len; rc = bnxt_hwrm_nvm_find_dir_entry(softc, BNX_DIR_TYPE_PKG_LOG, &ordinal, BNX_DIR_EXT_NONE, &index, false, HWRM_NVM_FIND_DIR_ENTRY_INPUT_OPT_ORDINAL_EQ, &data_len, NULL, NULL); dma_data.idi_vaddr = NULL; if (rc == 0 && data_len) { rc = iflib_dma_alloc(softc->ctx, data_len, &dma_data, BUS_DMA_NOWAIT); if (rc == 0) { rc = bnxt_hwrm_nvm_read(softc, index, 0, data_len, &dma_data); if (rc == 0) { pkglog = dma_data.idi_vaddr; /* NULL terminate (removes last \n) */ pkglog[data_len-1] = 0; /* Set p = start of last line */ p = strrchr(pkglog, '\n'); if (p == NULL) p = pkglog; /* Now find the correct tab delimited field */ for (field = 0, next = p, p = strsep(&next, "\t"); field < BNX_PKG_LOG_FIELD_IDX_PKG_VERSION && p; p = strsep(&next, "\t")) { field++; } if (field == BNX_PKG_LOG_FIELD_IDX_PKG_VERSION) buf = p; } } else dma_data.idi_vaddr = NULL; } rc = sysctl_handle_string(oidp, buf, 0, req); if (dma_data.idi_vaddr) iflib_dma_free(&dma_data); return rc; } static int bnxt_hwrm_min_ver_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; char buf[16]; uint8_t newver[3]; int rc; sprintf(buf, "%hhu.%hhu.%hhu", softc->ver_info->hwrm_min_major, softc->ver_info->hwrm_min_minor, softc->ver_info->hwrm_min_update); rc = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (rc || req->newptr == NULL) return rc; if (sscanf(buf, "%hhu.%hhu.%hhu%*c", &newver[0], &newver[1], &newver[2]) != 3) return EINVAL; softc->ver_info->hwrm_min_major = newver[0]; softc->ver_info->hwrm_min_minor = newver[1]; softc->ver_info->hwrm_min_update = newver[2]; bnxt_check_hwrm_version(softc); return rc; } int bnxt_create_ver_sysctls(struct bnxt_softc *softc) { struct bnxt_ver_info *vi = softc->ver_info; struct sysctl_oid *oid = vi->ver_oid; if (!oid) return ENOMEM; SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "hwrm_if", CTLFLAG_RD, vi->hwrm_if_ver, 0, "HWRM interface version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "driver_hwrm_if", CTLFLAG_RD, vi->driver_hwrm_if_ver, 0, "HWRM firmware version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "hwrm_fw", CTLFLAG_RD, vi->hwrm_fw_ver, 0, "HWRM firmware version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "mgmt_fw", CTLFLAG_RD, vi->mgmt_fw_ver, 0, "management firmware version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "netctrl_fw", CTLFLAG_RD, vi->netctrl_fw_ver, 0, "network control firmware version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "roce_fw", CTLFLAG_RD, vi->roce_fw_ver, 0, "RoCE firmware version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "phy", CTLFLAG_RD, vi->phy_ver, 0, "PHY version"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "hwrm_fw_name", CTLFLAG_RD, vi->hwrm_fw_name, 0, "HWRM firmware name"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "mgmt_fw_name", CTLFLAG_RD, vi->mgmt_fw_name, 0, "management firmware name"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "netctrl_fw_name", CTLFLAG_RD, vi->netctrl_fw_name, 0, "network control firmware name"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "roce_fw_name", CTLFLAG_RD, vi->roce_fw_name, 0, "RoCE firmware name"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "phy_vendor", CTLFLAG_RD, vi->phy_vendor, 0, "PHY vendor name"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "phy_partnumber", CTLFLAG_RD, vi->phy_partnumber, 0, "PHY vendor part number"); SYSCTL_ADD_U16(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "chip_num", CTLFLAG_RD, &vi->chip_num, 0, "chip number"); SYSCTL_ADD_U8(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "chip_rev", CTLFLAG_RD, &vi->chip_rev, 0, "chip revision"); SYSCTL_ADD_U8(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "chip_metal", CTLFLAG_RD, &vi->chip_metal, 0, "chip metal number"); SYSCTL_ADD_U8(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "chip_bond_id", CTLFLAG_RD, &vi->chip_bond_id, 0, "chip bond id"); SYSCTL_ADD_STRING(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "chip_type", CTLFLAG_RD, vi->chip_type > MAX_CHIP_TYPE ? bnxt_chip_type[MAX_CHIP_TYPE] : bnxt_chip_type[vi->chip_type], 0, "RoCE firmware name"); SYSCTL_ADD_PROC(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "package_ver", CTLTYPE_STRING|CTLFLAG_RD, softc, 0, bnxt_package_ver_sysctl, "A", "currently installed package version"); SYSCTL_ADD_PROC(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "hwrm_min_ver", CTLTYPE_STRING|CTLFLAG_RWTUN, softc, 0, bnxt_hwrm_min_ver_sysctl, "A", "minimum hwrm API vesion to support"); return 0; } int bnxt_create_nvram_sysctls(struct bnxt_nvram_info *ni) { struct sysctl_oid *oid = ni->nvm_oid; if (!oid) return ENOMEM; SYSCTL_ADD_U16(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "mfg_id", CTLFLAG_RD, &ni->mfg_id, 0, "manufacturer id"); SYSCTL_ADD_U16(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "device_id", CTLFLAG_RD, &ni->device_id, 0, "device id"); SYSCTL_ADD_U32(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "sector_size", CTLFLAG_RD, &ni->sector_size, 0, "sector size"); SYSCTL_ADD_U32(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "size", CTLFLAG_RD, &ni->size, 0, "nvram total size"); SYSCTL_ADD_U32(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "reserved_size", CTLFLAG_RD, &ni->reserved_size, 0, "total reserved space"); SYSCTL_ADD_U32(&ni->nvm_ctx, SYSCTL_CHILDREN(oid), OID_AUTO, "available_size", CTLFLAG_RD, &ni->available_size, 0, "total available space"); return 0; } static int bnxt_rss_key_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; char buf[HW_HASH_KEY_SIZE*2+1] = {0}; char *p; int i; int rc; for (p = buf, i=0; ivnic_info.rss_hash_key[i]); rc = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (rc || req->newptr == NULL) return rc; if (strspn(buf, "0123456789abcdefABCDEF") != (HW_HASH_KEY_SIZE * 2)) return EINVAL; for (p = buf, i=0; ivnic_info.rss_hash_key[i]) != 1) return EINVAL; p += 2; } if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING) bnxt_hwrm_rss_cfg(softc, &softc->vnic_info, softc->vnic_info.rss_hash_type); return rc; } static const char *bnxt_hash_types[] = {"ipv4", "tcp_ipv4", "udp_ipv4", "ipv6", "tcp_ipv6", "udp_ipv6", NULL}; static int bnxt_get_rss_type_str_bit(char *str) { int i; for (i=0; bnxt_hash_types[i]; i++) if (strcmp(bnxt_hash_types[i], str) == 0) return i; return -1; } static int bnxt_rss_type_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; char buf[256] = {0}; char *p; char *next; int rc; int type; int bit; for (type = softc->vnic_info.rss_hash_type; type; type &= ~(1<= sizeof(bnxt_hash_types) / sizeof(const char *)) continue; if (type != softc->vnic_info.rss_hash_type) strcat(buf, ","); strcat(buf, bnxt_hash_types[bit]); } rc = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (rc || req->newptr == NULL) return rc; for (type = 0, next = buf, p = strsep(&next, " ,"); p; p = strsep(&next, " ,")) { bit = bnxt_get_rss_type_str_bit(p); if (bit == -1) return EINVAL; type |= 1<vnic_info.rss_hash_type) { softc->vnic_info.rss_hash_type = type; if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING) bnxt_hwrm_rss_cfg(softc, &softc->vnic_info, softc->vnic_info.rss_hash_type); } return rc; } static int bnxt_rx_stall_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; int rc; int val; if (softc == NULL) return EBUSY; val = (bool)(softc->vnic_info.flags & BNXT_VNIC_FLAG_BD_STALL); rc = sysctl_handle_int(oidp, &val, 0, req); if (rc || !req->newptr) return rc; if (val) softc->vnic_info.flags |= BNXT_VNIC_FLAG_BD_STALL; else softc->vnic_info.flags &= ~BNXT_VNIC_FLAG_BD_STALL; if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING) rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info); return rc; } static int bnxt_vlan_strip_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; int rc; int val; if (softc == NULL) return EBUSY; val = (bool)(softc->vnic_info.flags & BNXT_VNIC_FLAG_VLAN_STRIP); rc = sysctl_handle_int(oidp, &val, 0, req); if (rc || !req->newptr) return rc; if (val) softc->vnic_info.flags |= BNXT_VNIC_FLAG_VLAN_STRIP; else softc->vnic_info.flags &= ~BNXT_VNIC_FLAG_VLAN_STRIP; if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING) rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info); return rc; } +static int +bnxt_set_coal_rx_usecs(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->rx_coal_usecs; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->rx_coal_usecs = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_rx_frames(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->rx_coal_frames; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->rx_coal_frames = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_rx_usecs_irq(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->rx_coal_usecs_irq; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->rx_coal_usecs_irq = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_rx_frames_irq(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->rx_coal_frames_irq; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->rx_coal_frames_irq = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_tx_usecs(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->tx_coal_usecs; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->tx_coal_usecs = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_tx_frames(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->tx_coal_frames; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->tx_coal_frames = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_tx_usecs_irq(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->tx_coal_usecs_irq; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->tx_coal_usecs_irq = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + +static int +bnxt_set_coal_tx_frames_irq(SYSCTL_HANDLER_ARGS) { + struct bnxt_softc *softc = arg1; + int rc; + int val; + + if (softc == NULL) + return EBUSY; + + val = softc->tx_coal_frames_irq; + rc = sysctl_handle_int(oidp, &val, 0, req); + if (rc || !req->newptr) + return rc; + + softc->tx_coal_frames_irq = val; + rc = bnxt_hwrm_set_coal(softc); + + return rc; +} + int bnxt_create_config_sysctls_pre(struct bnxt_softc *softc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(softc->dev); struct sysctl_oid_list *children; children = SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev));; SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rss_key", CTLTYPE_STRING|CTLFLAG_RWTUN, softc, 0, bnxt_rss_key_sysctl, "A", "RSS key"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rss_type", CTLTYPE_STRING|CTLFLAG_RWTUN, softc, 0, bnxt_rss_type_sysctl, "A", "RSS type bits"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_stall", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_rx_stall_sysctl, "I", "buffer rx packets in hardware until the host posts new buffers"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "vlan_strip", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_vlan_strip_sysctl, "I", "strip VLAN tag in the RX path"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "if_name", CTLFLAG_RD, iflib_get_ifp(softc->ctx)->if_xname, 0, "interface name"); + + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_usecs", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_usecs, + "I", "interrupt coalescing Rx Usecs"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_frames", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_frames, + "I", "interrupt coalescing Rx Frames"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_usecs_irq", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_usecs_irq, + "I", "interrupt coalescing Rx Usecs IRQ"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_frames_irq", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_frames_irq, + "I", "interrupt coalescing Rx Frames IRQ"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_usecs", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_usecs, + "I", "interrupt coalescing Tx Usces"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_frames", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_frames, + "I", "interrupt coalescing Tx Frames"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_usecs_irq", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_usecs_irq, + "I", "interrupt coalescing Tx Usecs IRQ"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_frames_irq", + CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_frames_irq, + "I", "interrupt coalescing Tx Frames IRQ"); return 0; } static int bnxt_vlan_only_sysctl(SYSCTL_HANDLER_ARGS) { struct bnxt_softc *softc = arg1; int rc; int val; if (softc == NULL) return EBUSY; val = softc->vnic_info.vlan_only; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc || !req->newptr) return rc; if (val) val = 1; if (val != softc->vnic_info.vlan_only) { softc->vnic_info.vlan_only = val; if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING) rc = bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); } return rc; } int bnxt_create_config_sysctls_post(struct bnxt_softc *softc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(softc->dev); struct sysctl_oid_list *children; children = SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev));; SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "vlan_only", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0, bnxt_vlan_only_sysctl, "I", "require vlan tag on received packets when vlan is enabled"); return 0; } Index: projects/runtime-coverage/sys/dev/bnxt/if_bnxt.c =================================================================== --- projects/runtime-coverage/sys/dev/bnxt/if_bnxt.c (revision 321584) +++ projects/runtime-coverage/sys/dev/bnxt/if_bnxt.c (revision 321585) @@ -1,2565 +1,2571 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_inet.h" #include "opt_inet6.h" #include "opt_rss.h" #include "ifdi_if.h" #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_ioctl.h" #include "bnxt_sysctl.h" #include "hsi_struct_def.h" /* * PCI Device ID Table */ static pci_vendor_info_t bnxt_vendor_info_array[] = { PVID(BROADCOM_VENDOR_ID, BCM57301, "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57302, "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57304, "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57311, "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57312, "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57314, "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57402, "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57402_NPAR, "Broadcom BCM57402 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57404, "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57404_NPAR, "Broadcom BCM57404 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57406, "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57406_NPAR, "Broadcom BCM57406 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57407, "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57407_NPAR, "Broadcom BCM57407 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57407_SFP, "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57412, "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57412_NPAR1, "Broadcom BCM57412 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57412_NPAR2, "Broadcom BCM57412 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57414, "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57414_NPAR1, "Broadcom BCM57414 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57414_NPAR2, "Broadcom BCM57414 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416, "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57416_NPAR1, "Broadcom BCM57416 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416_NPAR2, "Broadcom BCM57416 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416_SFP, "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57417, "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57417_NPAR1, "Broadcom BCM57417 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57417_NPAR2, "Broadcom BCM57417 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57417_SFP, "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57454, "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM58700, "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF1, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF2, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF3, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF1, "Broadcom NetXtreme-E Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF2, "Broadcom NetXtreme-E Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF3, "Broadcom NetXtreme-E Ethernet Virtual Function"), /* required last entry */ PVID_END }; /* * Function prototypes */ static void *bnxt_register(device_t dev); /* Soft queue setup and teardown */ static int bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); static int bnxt_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets); static void bnxt_queues_free(if_ctx_t ctx); /* Device setup and teardown */ static int bnxt_attach_pre(if_ctx_t ctx); static int bnxt_attach_post(if_ctx_t ctx); static int bnxt_detach(if_ctx_t ctx); /* Device configuration */ static void bnxt_init(if_ctx_t ctx); static void bnxt_stop(if_ctx_t ctx); static void bnxt_multi_set(if_ctx_t ctx); static int bnxt_mtu_set(if_ctx_t ctx, uint32_t mtu); static void bnxt_media_status(if_ctx_t ctx, struct ifmediareq * ifmr); static int bnxt_media_change(if_ctx_t ctx); static int bnxt_promisc_set(if_ctx_t ctx, int flags); static uint64_t bnxt_get_counter(if_ctx_t, ift_counter); static void bnxt_update_admin_status(if_ctx_t ctx); /* Interrupt enable / disable */ static void bnxt_intr_enable(if_ctx_t ctx); static int bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid); static int bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid); static void bnxt_disable_intr(if_ctx_t ctx); static int bnxt_msix_intr_assign(if_ctx_t ctx, int msix); /* vlan support */ static void bnxt_vlan_register(if_ctx_t ctx, uint16_t vtag); static void bnxt_vlan_unregister(if_ctx_t ctx, uint16_t vtag); /* ioctl */ static int bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); static int bnxt_shutdown(if_ctx_t ctx); static int bnxt_suspend(if_ctx_t ctx); static int bnxt_resume(if_ctx_t ctx); /* Internal support functions */ static int bnxt_probe_phy(struct bnxt_softc *softc); static void bnxt_add_media_types(struct bnxt_softc *softc); static int bnxt_pci_mapping(struct bnxt_softc *softc); static void bnxt_pci_mapping_free(struct bnxt_softc *softc); static int bnxt_update_link(struct bnxt_softc *softc, bool chng_link_state); static int bnxt_handle_def_cp(void *arg); static int bnxt_handle_rx_cp(void *arg); static void bnxt_clear_ids(struct bnxt_softc *softc); static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr); static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr); static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr); static void bnxt_def_cp_task(void *context); static void bnxt_handle_async_event(struct bnxt_softc *softc, struct cmpl_base *cmpl); static uint8_t get_phy_type(struct bnxt_softc *softc); static uint64_t bnxt_get_baudrate(struct bnxt_link_info *link); static void bnxt_get_wol_settings(struct bnxt_softc *softc); static int bnxt_wol_config(if_ctx_t ctx); /* * Device Interface Declaration */ static device_method_t bnxt_methods[] = { /* Device interface */ DEVMETHOD(device_register, bnxt_register), DEVMETHOD(device_probe, iflib_device_probe), DEVMETHOD(device_attach, iflib_device_attach), DEVMETHOD(device_detach, iflib_device_detach), DEVMETHOD(device_shutdown, iflib_device_shutdown), DEVMETHOD(device_suspend, iflib_device_suspend), DEVMETHOD(device_resume, iflib_device_resume), DEVMETHOD_END }; static driver_t bnxt_driver = { "bnxt", bnxt_methods, sizeof(struct bnxt_softc), }; devclass_t bnxt_devclass; DRIVER_MODULE(bnxt, pci, bnxt_driver, bnxt_devclass, 0, 0); MODULE_DEPEND(bnxt, pci, 1, 1, 1); MODULE_DEPEND(bnxt, ether, 1, 1, 1); MODULE_DEPEND(bnxt, iflib, 1, 1, 1); static device_method_t bnxt_iflib_methods[] = { DEVMETHOD(ifdi_tx_queues_alloc, bnxt_tx_queues_alloc), DEVMETHOD(ifdi_rx_queues_alloc, bnxt_rx_queues_alloc), DEVMETHOD(ifdi_queues_free, bnxt_queues_free), DEVMETHOD(ifdi_attach_pre, bnxt_attach_pre), DEVMETHOD(ifdi_attach_post, bnxt_attach_post), DEVMETHOD(ifdi_detach, bnxt_detach), DEVMETHOD(ifdi_init, bnxt_init), DEVMETHOD(ifdi_stop, bnxt_stop), DEVMETHOD(ifdi_multi_set, bnxt_multi_set), DEVMETHOD(ifdi_mtu_set, bnxt_mtu_set), DEVMETHOD(ifdi_media_status, bnxt_media_status), DEVMETHOD(ifdi_media_change, bnxt_media_change), DEVMETHOD(ifdi_promisc_set, bnxt_promisc_set), DEVMETHOD(ifdi_get_counter, bnxt_get_counter), DEVMETHOD(ifdi_update_admin_status, bnxt_update_admin_status), DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable), DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_tx_queue_intr_enable), DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_rx_queue_intr_enable), DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr), DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign), DEVMETHOD(ifdi_vlan_register, bnxt_vlan_register), DEVMETHOD(ifdi_vlan_unregister, bnxt_vlan_unregister), DEVMETHOD(ifdi_priv_ioctl, bnxt_priv_ioctl), DEVMETHOD(ifdi_suspend, bnxt_suspend), DEVMETHOD(ifdi_shutdown, bnxt_shutdown), DEVMETHOD(ifdi_resume, bnxt_resume), DEVMETHOD_END }; static driver_t bnxt_iflib_driver = { "bnxt", bnxt_iflib_methods, sizeof(struct bnxt_softc) }; /* * iflib shared context */ #define BNXT_DRIVER_VERSION "1.0.0.1" char bnxt_driver_version[] = BNXT_DRIVER_VERSION; extern struct if_txrx bnxt_txrx; static struct if_shared_ctx bnxt_sctx_init = { .isc_magic = IFLIB_MAGIC, .isc_driver = &bnxt_iflib_driver, .isc_nfl = 2, // Number of Free Lists .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ, .isc_q_align = PAGE_SIZE, .isc_tx_maxsize = BNXT_TSO_SIZE, .isc_tx_maxsegsize = BNXT_TSO_SIZE, .isc_rx_maxsize = BNXT_TSO_SIZE, .isc_rx_maxsegsize = BNXT_TSO_SIZE, // Only use a single segment to avoid page size constraints .isc_rx_nsegments = 1, .isc_ntxqs = 2, .isc_nrxqs = 3, .isc_nrxd_min = {16, 16, 16}, .isc_nrxd_default = {PAGE_SIZE / sizeof(struct cmpl_base) * 8, PAGE_SIZE / sizeof(struct rx_prod_pkt_bd), PAGE_SIZE / sizeof(struct rx_prod_pkt_bd)}, .isc_nrxd_max = {INT32_MAX, INT32_MAX, INT32_MAX}, .isc_ntxd_min = {16, 16, 16}, .isc_ntxd_default = {PAGE_SIZE / sizeof(struct cmpl_base) * 2, PAGE_SIZE / sizeof(struct tx_bd_short)}, .isc_ntxd_max = {INT32_MAX, INT32_MAX, INT32_MAX}, .isc_admin_intrcnt = 1, .isc_vendor_info = bnxt_vendor_info_array, .isc_driver_version = bnxt_driver_version, }; if_shared_ctx_t bnxt_sctx = &bnxt_sctx_init; /* * Device Methods */ static void * bnxt_register(device_t dev) { return bnxt_sctx; } /* * Device Dependent Configuration Functions */ /* Soft queue setup and teardown */ static int bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { struct bnxt_softc *softc; int i; int rc; softc = iflib_get_softc(ctx); softc->tx_cp_rings = malloc(sizeof(struct bnxt_cp_ring) * ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->tx_cp_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate TX completion rings\n"); rc = ENOMEM; goto cp_alloc_fail; } softc->tx_rings = malloc(sizeof(struct bnxt_ring) * ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->tx_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate TX rings\n"); rc = ENOMEM; goto ring_alloc_fail; } rc = iflib_dma_alloc(ctx, sizeof(struct ctx_hw_stats) * ntxqsets, &softc->tx_stats, 0); if (rc) goto dma_alloc_fail; bus_dmamap_sync(softc->tx_stats.idi_tag, softc->tx_stats.idi_map, BUS_DMASYNC_PREREAD); for (i = 0; i < ntxqsets; i++) { /* Set up the completion ring */ softc->tx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.softc = softc; softc->tx_cp_rings[i].ring.id = (softc->scctx->isc_nrxqsets * 2) + 1 + i; softc->tx_cp_rings[i].ring.doorbell = softc->tx_cp_rings[i].ring.id * 0x80; softc->tx_cp_rings[i].ring.ring_size = softc->scctx->isc_ntxd[0]; softc->tx_cp_rings[i].ring.vaddr = vaddrs[i * ntxqs]; softc->tx_cp_rings[i].ring.paddr = paddrs[i * ntxqs]; /* Set up the TX ring */ softc->tx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_rings[i].softc = softc; softc->tx_rings[i].id = (softc->scctx->isc_nrxqsets * 2) + 1 + i; softc->tx_rings[i].doorbell = softc->tx_rings[i].id * 0x80; softc->tx_rings[i].ring_size = softc->scctx->isc_ntxd[1]; softc->tx_rings[i].vaddr = vaddrs[i * ntxqs + 1]; softc->tx_rings[i].paddr = paddrs[i * ntxqs + 1]; bnxt_create_tx_sysctls(softc, i); } softc->ntxqsets = ntxqsets; return rc; dma_alloc_fail: free(softc->tx_rings, M_DEVBUF); ring_alloc_fail: free(softc->tx_cp_rings, M_DEVBUF); cp_alloc_fail: return rc; } static void bnxt_queues_free(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); // Free TX queues iflib_dma_free(&softc->tx_stats); free(softc->tx_rings, M_DEVBUF); softc->tx_rings = NULL; free(softc->tx_cp_rings, M_DEVBUF); softc->tx_cp_rings = NULL; softc->ntxqsets = 0; // Free RX queues iflib_dma_free(&softc->rx_stats); free(softc->grp_info, M_DEVBUF); free(softc->ag_rings, M_DEVBUF); free(softc->rx_rings, M_DEVBUF); free(softc->rx_cp_rings, M_DEVBUF); } static int bnxt_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { struct bnxt_softc *softc; int i; int rc; softc = iflib_get_softc(ctx); softc->rx_cp_rings = malloc(sizeof(struct bnxt_cp_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->rx_cp_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate RX completion rings\n"); rc = ENOMEM; goto cp_alloc_fail; } softc->rx_rings = malloc(sizeof(struct bnxt_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->rx_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate RX rings\n"); rc = ENOMEM; goto ring_alloc_fail; } softc->ag_rings = malloc(sizeof(struct bnxt_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->ag_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate aggregation rings\n"); rc = ENOMEM; goto ag_alloc_fail; } softc->grp_info = malloc(sizeof(struct bnxt_grp_info) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->grp_info) { device_printf(iflib_get_dev(ctx), "unable to allocate ring groups\n"); rc = ENOMEM; goto grp_alloc_fail; } rc = iflib_dma_alloc(ctx, sizeof(struct ctx_hw_stats) * nrxqsets, &softc->rx_stats, 0); if (rc) goto hw_stats_alloc_fail; bus_dmamap_sync(softc->rx_stats.idi_tag, softc->rx_stats.idi_map, BUS_DMASYNC_PREREAD); for (i = 0; i < nrxqsets; i++) { /* Allocation the completion ring */ softc->rx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.softc = softc; softc->rx_cp_rings[i].ring.id = i + 1; softc->rx_cp_rings[i].ring.doorbell = softc->rx_cp_rings[i].ring.id * 0x80; /* * If this ring overflows, RX stops working. */ softc->rx_cp_rings[i].ring.ring_size = softc->scctx->isc_nrxd[0]; softc->rx_cp_rings[i].ring.vaddr = vaddrs[i * nrxqs]; softc->rx_cp_rings[i].ring.paddr = paddrs[i * nrxqs]; /* Allocate the RX ring */ softc->rx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_rings[i].softc = softc; softc->rx_rings[i].id = i + 1; softc->rx_rings[i].doorbell = softc->rx_rings[i].id * 0x80; softc->rx_rings[i].ring_size = softc->scctx->isc_nrxd[1]; softc->rx_rings[i].vaddr = vaddrs[i * nrxqs + 1]; softc->rx_rings[i].paddr = paddrs[i * nrxqs + 1]; /* Allocate the TPA start buffer */ softc->rx_rings[i].tpa_start = malloc(sizeof(struct bnxt_full_tpa_start) * (RX_TPA_START_CMPL_AGG_ID_MASK >> RX_TPA_START_CMPL_AGG_ID_SFT), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->rx_rings[i].tpa_start == NULL) { rc = -ENOMEM; device_printf(softc->dev, "Unable to allocate space for TPA\n"); goto tpa_alloc_fail; } /* Allocate the AG ring */ softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->ag_rings[i].softc = softc; softc->ag_rings[i].id = nrxqsets + i + 1; softc->ag_rings[i].doorbell = softc->ag_rings[i].id * 0x80; softc->ag_rings[i].ring_size = softc->scctx->isc_nrxd[2]; softc->ag_rings[i].vaddr = vaddrs[i * nrxqs + 2]; softc->ag_rings[i].paddr = paddrs[i * nrxqs + 2]; /* Allocate the ring group */ softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE; softc->grp_info[i].stats_ctx = softc->rx_cp_rings[i].stats_ctx_id; softc->grp_info[i].rx_ring_id = softc->rx_rings[i].phys_id; softc->grp_info[i].ag_ring_id = softc->ag_rings[i].phys_id; softc->grp_info[i].cp_ring_id = softc->rx_cp_rings[i].ring.phys_id; bnxt_create_rx_sysctls(softc, i); } /* And finally, the VNIC */ softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.flow_id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.filter_id = -1; softc->vnic_info.def_ring_grp = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.cos_rule = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.lb_rule = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.rx_mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST; softc->vnic_info.mc_list_count = 0; softc->vnic_info.flags = BNXT_VNIC_FLAG_DEFAULT; rc = iflib_dma_alloc(ctx, BNXT_MAX_MC_ADDRS * ETHER_ADDR_LEN, &softc->vnic_info.mc_list, 0); if (rc) goto mc_list_alloc_fail; /* The VNIC RSS Hash Key */ rc = iflib_dma_alloc(ctx, HW_HASH_KEY_SIZE, &softc->vnic_info.rss_hash_key_tbl, 0); if (rc) goto rss_hash_alloc_fail; bus_dmamap_sync(softc->vnic_info.rss_hash_key_tbl.idi_tag, softc->vnic_info.rss_hash_key_tbl.idi_map, BUS_DMASYNC_PREWRITE); memcpy(softc->vnic_info.rss_hash_key_tbl.idi_vaddr, softc->vnic_info.rss_hash_key, HW_HASH_KEY_SIZE); /* Allocate the RSS tables */ rc = iflib_dma_alloc(ctx, HW_HASH_INDEX_SIZE * sizeof(uint16_t), &softc->vnic_info.rss_grp_tbl, 0); if (rc) goto rss_grp_alloc_fail; bus_dmamap_sync(softc->vnic_info.rss_grp_tbl.idi_tag, softc->vnic_info.rss_grp_tbl.idi_map, BUS_DMASYNC_PREWRITE); memset(softc->vnic_info.rss_grp_tbl.idi_vaddr, 0xff, softc->vnic_info.rss_grp_tbl.idi_size); softc->nrxqsets = nrxqsets; return rc; rss_grp_alloc_fail: iflib_dma_free(&softc->vnic_info.rss_hash_key_tbl); rss_hash_alloc_fail: iflib_dma_free(&softc->vnic_info.mc_list); tpa_alloc_fail: mc_list_alloc_fail: for (i = i - 1; i >= 0; i--) free(softc->rx_rings[i].tpa_start, M_DEVBUF); iflib_dma_free(&softc->rx_stats); hw_stats_alloc_fail: free(softc->grp_info, M_DEVBUF); grp_alloc_fail: free(softc->ag_rings, M_DEVBUF); ag_alloc_fail: free(softc->rx_rings, M_DEVBUF); ring_alloc_fail: free(softc->rx_cp_rings, M_DEVBUF); cp_alloc_fail: return rc; } /* Device setup and teardown */ static int bnxt_attach_pre(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_softc_ctx_t scctx; int rc = 0; softc->ctx = ctx; softc->dev = iflib_get_dev(ctx); softc->media = iflib_get_media(ctx); softc->scctx = iflib_get_softc_ctx(ctx); softc->sctx = iflib_get_sctx(ctx); scctx = softc->scctx; /* TODO: Better way of detecting NPAR/VF is needed */ switch (softc->sctx->isc_vendor_info->pvi_device_id) { case BCM57402_NPAR: case BCM57404_NPAR: case BCM57406_NPAR: case BCM57407_NPAR: case BCM57412_NPAR1: case BCM57412_NPAR2: case BCM57414_NPAR1: case BCM57414_NPAR2: case BCM57416_NPAR1: case BCM57416_NPAR2: softc->flags |= BNXT_FLAG_NPAR; break; case NETXTREME_C_VF1: case NETXTREME_C_VF2: case NETXTREME_C_VF3: case NETXTREME_E_VF1: case NETXTREME_E_VF2: case NETXTREME_E_VF3: softc->flags |= BNXT_FLAG_VF; break; } pci_enable_busmaster(softc->dev); if (bnxt_pci_mapping(softc)) return (ENXIO); /* HWRM setup/init */ BNXT_HWRM_LOCK_INIT(softc, device_get_nameunit(softc->dev)); rc = bnxt_alloc_hwrm_dma_mem(softc); if (rc) goto dma_fail; /* Get firmware version and compare with driver */ softc->ver_info = malloc(sizeof(struct bnxt_ver_info), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->ver_info == NULL) { rc = ENOMEM; device_printf(softc->dev, "Unable to allocate space for version info\n"); goto ver_alloc_fail; } /* Default minimum required HWRM version */ softc->ver_info->hwrm_min_major = 1; softc->ver_info->hwrm_min_minor = 2; softc->ver_info->hwrm_min_update = 2; rc = bnxt_hwrm_ver_get(softc); if (rc) { device_printf(softc->dev, "attach: hwrm ver get failed\n"); goto ver_fail; } /* Get NVRAM info */ softc->nvm_info = malloc(sizeof(struct bnxt_nvram_info), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->nvm_info == NULL) { rc = ENOMEM; device_printf(softc->dev, "Unable to allocate space for NVRAM info\n"); goto nvm_alloc_fail; } rc = bnxt_hwrm_nvm_get_dev_info(softc, &softc->nvm_info->mfg_id, &softc->nvm_info->device_id, &softc->nvm_info->sector_size, &softc->nvm_info->size, &softc->nvm_info->reserved_size, &softc->nvm_info->available_size); /* Register the driver with the FW */ rc = bnxt_hwrm_func_drv_rgtr(softc); if (rc) { device_printf(softc->dev, "attach: hwrm drv rgtr failed\n"); goto drv_rgtr_fail; } + rc = bnxt_hwrm_func_rgtr_async_events(softc, NULL, 0); + if (rc) { + device_printf(softc->dev, "attach: hwrm rgtr async evts failed\n"); + goto drv_rgtr_fail; + } + /* Get the HW capabilities */ rc = bnxt_hwrm_func_qcaps(softc); if (rc) goto failed; iflib_set_mac(ctx, softc->func.mac_addr); scctx->isc_txrx = &bnxt_txrx; scctx->isc_tx_csum_flags = (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | CSUM_TSO); scctx->isc_capenable = /* These are translated to hwassit bits */ IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_TSO4 | IFCAP_TSO6 | /* These are checked by iflib */ IFCAP_LRO | IFCAP_VLAN_HWFILTER | /* These are part of the iflib mask */ IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | /* These likely get lost... */ IFCAP_VLAN_HWCSUM | IFCAP_JUMBO_MTU; if (bnxt_wol_supported(softc)) scctx->isc_capenable |= IFCAP_WOL_MAGIC; /* Get the queue config */ rc = bnxt_hwrm_queue_qportcfg(softc); if (rc) { device_printf(softc->dev, "attach: hwrm qportcfg failed\n"); goto failed; } bnxt_get_wol_settings(softc); /* Now perform a function reset */ rc = bnxt_hwrm_func_reset(softc); bnxt_clear_ids(softc); if (rc) goto failed; /* Now set up iflib sc */ scctx->isc_tx_nsegments = 31, scctx->isc_tx_tso_segments_max = 31; scctx->isc_tx_tso_size_max = BNXT_TSO_SIZE; scctx->isc_tx_tso_segsize_max = BNXT_TSO_SIZE; scctx->isc_vectors = softc->func.max_cp_rings; scctx->isc_txrx = &bnxt_txrx; if (scctx->isc_nrxd[0] < ((scctx->isc_nrxd[1] * 4) + scctx->isc_nrxd[2])) device_printf(softc->dev, "WARNING: nrxd0 (%d) should be at least 4 * nrxd1 (%d) + nrxd2 (%d). Driver may be unstable\n", scctx->isc_nrxd[0], scctx->isc_nrxd[1], scctx->isc_nrxd[2]); if (scctx->isc_ntxd[0] < scctx->isc_ntxd[1] * 2) device_printf(softc->dev, "WARNING: ntxd0 (%d) should be at least 2 * ntxd1 (%d). Driver may be unstable\n", scctx->isc_ntxd[0], scctx->isc_ntxd[1]); scctx->isc_txqsizes[0] = sizeof(struct cmpl_base) * scctx->isc_ntxd[0]; scctx->isc_txqsizes[1] = sizeof(struct tx_bd_short) * scctx->isc_ntxd[1]; scctx->isc_rxqsizes[0] = sizeof(struct cmpl_base) * scctx->isc_nrxd[0]; scctx->isc_rxqsizes[1] = sizeof(struct rx_prod_pkt_bd) * scctx->isc_nrxd[1]; scctx->isc_rxqsizes[2] = sizeof(struct rx_prod_pkt_bd) * scctx->isc_nrxd[2]; scctx->isc_max_rxqsets = min(pci_msix_count(softc->dev)-1, softc->func.max_cp_rings - 1); scctx->isc_max_rxqsets = min(scctx->isc_max_rxqsets, softc->func.max_rx_rings); scctx->isc_max_txqsets = min(softc->func.max_rx_rings, softc->func.max_cp_rings - scctx->isc_max_rxqsets - 1); scctx->isc_rss_table_size = HW_HASH_INDEX_SIZE; scctx->isc_rss_table_mask = scctx->isc_rss_table_size - 1; /* iflib will map and release this bar */ scctx->isc_msix_bar = pci_msix_table_bar(softc->dev); /* Allocate the default completion ring */ softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.softc = softc; softc->def_cp_ring.ring.id = 0; softc->def_cp_ring.ring.doorbell = softc->def_cp_ring.ring.id * 0x80; softc->def_cp_ring.ring.ring_size = PAGE_SIZE / sizeof(struct cmpl_base); rc = iflib_dma_alloc(ctx, sizeof(struct cmpl_base) * softc->def_cp_ring.ring.ring_size, &softc->def_cp_ring_mem, 0); softc->def_cp_ring.ring.vaddr = softc->def_cp_ring_mem.idi_vaddr; softc->def_cp_ring.ring.paddr = softc->def_cp_ring_mem.idi_paddr; iflib_config_gtask_init(ctx, &softc->def_cp_task, bnxt_def_cp_task, "dflt_cp"); rc = bnxt_init_sysctl_ctx(softc); if (rc) goto init_sysctl_failed; rc = bnxt_create_nvram_sysctls(softc->nvm_info); if (rc) goto failed; arc4rand(softc->vnic_info.rss_hash_key, HW_HASH_KEY_SIZE, 0); softc->vnic_info.rss_hash_type = HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6; rc = bnxt_create_config_sysctls_pre(softc); if (rc) goto failed; /* Initialize the vlan list */ SLIST_INIT(&softc->vnic_info.vlan_tags); softc->vnic_info.vlan_tag_list.idi_vaddr = NULL; return (rc); failed: bnxt_free_sysctl_ctx(softc); init_sysctl_failed: bnxt_hwrm_func_drv_unrgtr(softc, false); drv_rgtr_fail: free(softc->nvm_info, M_DEVBUF); nvm_alloc_fail: ver_fail: free(softc->ver_info, M_DEVBUF); ver_alloc_fail: bnxt_free_hwrm_dma_mem(softc); dma_fail: BNXT_HWRM_LOCK_DESTROY(softc); bnxt_pci_mapping_free(softc); pci_disable_busmaster(softc->dev); return (rc); } static int bnxt_attach_post(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); int rc; bnxt_create_config_sysctls_post(softc); /* Update link state etc... */ rc = bnxt_probe_phy(softc); if (rc) goto failed; /* Needs to be done after probing the phy */ bnxt_create_ver_sysctls(softc); bnxt_add_media_types(softc); ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); softc->scctx->isc_max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; failed: return rc; } static int bnxt_detach(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *tag; struct bnxt_vlan_tag *tmp; int i; bnxt_wol_config(ctx); bnxt_do_disable_intr(&softc->def_cp_ring); bnxt_free_sysctl_ctx(softc); bnxt_hwrm_func_reset(softc); bnxt_clear_ids(softc); iflib_irq_free(ctx, &softc->def_cp_ring.irq); iflib_config_gtask_deinit(&softc->def_cp_task); /* We need to free() these here... */ for (i = softc->nrxqsets-1; i>=0; i--) { iflib_irq_free(ctx, &softc->rx_cp_rings[i].irq); } iflib_dma_free(&softc->vnic_info.mc_list); iflib_dma_free(&softc->vnic_info.rss_hash_key_tbl); iflib_dma_free(&softc->vnic_info.rss_grp_tbl); if (softc->vnic_info.vlan_tag_list.idi_vaddr) iflib_dma_free(&softc->vnic_info.vlan_tag_list); SLIST_FOREACH_SAFE(tag, &softc->vnic_info.vlan_tags, next, tmp) free(tag, M_DEVBUF); iflib_dma_free(&softc->def_cp_ring_mem); for (i = 0; i < softc->nrxqsets; i++) free(softc->rx_rings[i].tpa_start, M_DEVBUF); free(softc->ver_info, M_DEVBUF); free(softc->nvm_info, M_DEVBUF); bnxt_hwrm_func_drv_unrgtr(softc, false); bnxt_free_hwrm_dma_mem(softc); BNXT_HWRM_LOCK_DESTROY(softc); pci_disable_busmaster(softc->dev); bnxt_pci_mapping_free(softc); return 0; } /* Device configuration */ static void bnxt_init(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifmediareq ifmr; int i, j; int rc; rc = bnxt_hwrm_func_reset(softc); if (rc) return; bnxt_clear_ids(softc); /* Allocate the default completion ring */ softc->def_cp_ring.cons = UINT32_MAX; softc->def_cp_ring.v_bit = 1; bnxt_mark_cpr_invalid(&softc->def_cp_ring); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, &softc->def_cp_ring.ring, (uint16_t)HWRM_NA_SIGNATURE, HWRM_NA_SIGNATURE, true); if (rc) goto fail; /* And now set the default CP ring as the async CP ring */ rc = bnxt_hwrm_func_cfg(softc); if (rc) goto fail; for (i = 0; i < softc->nrxqsets; i++) { /* Allocate the statistics context */ rc = bnxt_hwrm_stat_ctx_alloc(softc, &softc->rx_cp_rings[i], softc->rx_stats.idi_paddr + (sizeof(struct ctx_hw_stats) * i)); if (rc) goto fail; /* Allocate the completion ring */ softc->rx_cp_rings[i].cons = UINT32_MAX; softc->rx_cp_rings[i].v_bit = 1; softc->rx_cp_rings[i].last_idx = UINT32_MAX; bnxt_mark_cpr_invalid(&softc->rx_cp_rings[i]); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, &softc->rx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE, HWRM_NA_SIGNATURE, true); if (rc) goto fail; /* Allocate the RX ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, &softc->rx_rings[i], (uint16_t)HWRM_NA_SIGNATURE, HWRM_NA_SIGNATURE, false); if (rc) goto fail; BNXT_RX_DB(&softc->rx_rings[i], 0); /* TODO: Cumulus+ doesn't need the double doorbell */ BNXT_RX_DB(&softc->rx_rings[i], 0); /* Allocate the AG ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, &softc->ag_rings[i], (uint16_t)HWRM_NA_SIGNATURE, HWRM_NA_SIGNATURE, false); if (rc) goto fail; BNXT_RX_DB(&softc->rx_rings[i], 0); /* TODO: Cumulus+ doesn't need the double doorbell */ BNXT_RX_DB(&softc->ag_rings[i], 0); /* Allocate the ring group */ softc->grp_info[i].stats_ctx = softc->rx_cp_rings[i].stats_ctx_id; softc->grp_info[i].rx_ring_id = softc->rx_rings[i].phys_id; softc->grp_info[i].ag_ring_id = softc->ag_rings[i].phys_id; softc->grp_info[i].cp_ring_id = softc->rx_cp_rings[i].ring.phys_id; rc = bnxt_hwrm_ring_grp_alloc(softc, &softc->grp_info[i]); if (rc) goto fail; } /* Allocate the VNIC RSS context */ rc = bnxt_hwrm_vnic_ctx_alloc(softc, &softc->vnic_info.rss_id); if (rc) goto fail; /* Allocate the vnic */ softc->vnic_info.def_ring_grp = softc->grp_info[0].grp_id; softc->vnic_info.mru = softc->scctx->isc_max_frame_size; rc = bnxt_hwrm_vnic_alloc(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_set_filter(softc, &softc->vnic_info); if (rc) goto fail; /* Enable RSS on the VNICs */ for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { ((uint16_t *) softc->vnic_info.rss_grp_tbl.idi_vaddr)[i] = htole16(softc->grp_info[j].grp_id); if (++j == softc->nrxqsets) j = 0; } rc = bnxt_hwrm_rss_cfg(softc, &softc->vnic_info, softc->vnic_info.rss_hash_type); if (rc) goto fail; /* * Enable LRO/TPA/GRO * TBD: * Enable / Disable HW_LRO based on * ifconfig lro / ifconfig -lro setting */ rc = bnxt_hwrm_vnic_tpa_cfg(softc, &softc->vnic_info, (if_getcapenable(iflib_get_ifp(ctx)) & IFCAP_LRO) ? HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA : 0); if (rc) goto fail; for (i = 0; i < softc->ntxqsets; i++) { /* Allocate the statistics context */ rc = bnxt_hwrm_stat_ctx_alloc(softc, &softc->tx_cp_rings[i], softc->tx_stats.idi_paddr + (sizeof(struct ctx_hw_stats) * i)); if (rc) goto fail; /* Allocate the completion ring */ softc->tx_cp_rings[i].cons = UINT32_MAX; softc->tx_cp_rings[i].v_bit = 1; bnxt_mark_cpr_invalid(&softc->tx_cp_rings[i]); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, &softc->tx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE, HWRM_NA_SIGNATURE, false); if (rc) goto fail; /* Allocate the TX ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, &softc->tx_rings[i], softc->tx_cp_rings[i].ring.phys_id, softc->tx_cp_rings[i].stats_ctx_id, false); if (rc) goto fail; BNXT_TX_DB(&softc->tx_rings[i], 0); /* TODO: Cumulus+ doesn't need the double doorbell */ BNXT_TX_DB(&softc->tx_rings[i], 0); } bnxt_do_enable_intr(&softc->def_cp_ring); bnxt_media_status(softc->ctx, &ifmr); return; fail: bnxt_hwrm_func_reset(softc); bnxt_clear_ids(softc); return; } static void bnxt_stop(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); bnxt_do_disable_intr(&softc->def_cp_ring); bnxt_hwrm_func_reset(softc); bnxt_clear_ids(softc); return; } static void bnxt_multi_set(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); uint8_t *mta; int cnt, mcnt; mcnt = if_multiaddr_count(ifp, -1); if (mcnt > BNXT_MAX_MC_ADDRS) { softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); } else { softc->vnic_info.rx_mask &= ~HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; mta = softc->vnic_info.mc_list.idi_vaddr; bzero(mta, softc->vnic_info.mc_list.idi_size); if_multiaddr_array(ifp, mta, &cnt, mcnt); bus_dmamap_sync(softc->vnic_info.mc_list.idi_tag, softc->vnic_info.mc_list.idi_map, BUS_DMASYNC_PREWRITE); softc->vnic_info.mc_list_count = cnt; softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; if (bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info)) device_printf(softc->dev, "set_multi: rx_mask set failed\n"); } } static int bnxt_mtu_set(if_ctx_t ctx, uint32_t mtu) { struct bnxt_softc *softc = iflib_get_softc(ctx); if (mtu > BNXT_MAX_MTU) return EINVAL; softc->scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; return 0; } static void bnxt_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_link_info *link_info = &softc->link_info; uint8_t phy_type = get_phy_type(softc); bnxt_update_link(softc, true); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (link_info->link_up) ifmr->ifm_status |= IFM_ACTIVE; else ifmr->ifm_status &= ~IFM_ACTIVE; if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; switch (link_info->link_speed) { case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB: ifmr->ifm_active |= IFM_100_T; break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX: ifmr->ifm_active |= IFM_1000_KX; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET: ifmr->ifm_active |= IFM_1000_T; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY: ifmr->ifm_active |= IFM_1000_SGMII; break; default: /* * Workaround: * Don't return IFM_UNKNOWN until * Stratus return proper media_type */ ifmr->ifm_active |= IFM_1000_KX; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX: ifmr->ifm_active |= IFM_2500_KX; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET: ifmr->ifm_active |= IFM_2500_T; break; default: ifmr->ifm_active |= IFM_UNKNOWN; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: ifmr->ifm_active |= IFM_10G_CR1; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: ifmr->ifm_active |= IFM_10G_KR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR: ifmr->ifm_active |= IFM_10G_LR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: ifmr->ifm_active |= IFM_10G_SR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX: ifmr->ifm_active |= IFM_10G_KX4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET: ifmr->ifm_active |= IFM_10G_T; break; default: /* * Workaround: * Don't return IFM_UNKNOWN until * Stratus return proper media_type */ ifmr->ifm_active |= IFM_10G_CR1; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB: ifmr->ifm_active |= IFM_20G_KR2; break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: ifmr->ifm_active |= IFM_25G_CR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: ifmr->ifm_active |= IFM_25G_KR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: ifmr->ifm_active |= IFM_25G_SR; break; default: /* * Workaround: * Don't return IFM_UNKNOWN until * Stratus return proper media_type */ ifmr->ifm_active |= IFM_25G_CR; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: ifmr->ifm_active |= IFM_40G_CR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: ifmr->ifm_active |= IFM_40G_KR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR: ifmr->ifm_active |= IFM_40G_LR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: ifmr->ifm_active |= IFM_40G_SR4; break; default: ifmr->ifm_active |= IFM_UNKNOWN; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: ifmr->ifm_active |= IFM_50G_CR2; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: ifmr->ifm_active |= IFM_50G_KR2; break; default: /* * Workaround: * Don't return IFM_UNKNOWN until * Stratus return proper media_type */ ifmr->ifm_active |= IFM_50G_CR2; break; } break; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB: switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: ifmr->ifm_active |= IFM_100G_CR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: ifmr->ifm_active |= IFM_100G_KR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR: ifmr->ifm_active |= IFM_100G_LR4; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: ifmr->ifm_active |= IFM_100G_SR4; break; default: /* * Workaround: * Don't return IFM_UNKNOWN until * Stratus return proper media_type */ ifmr->ifm_active |= IFM_100G_CR4; break; } default: return; } if (link_info->pause == (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX | HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) ifmr->ifm_active |= (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); else if (link_info->pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX) ifmr->ifm_active |= IFM_ETH_TXPAUSE; else if (link_info->pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX) ifmr->ifm_active |= IFM_ETH_RXPAUSE; bnxt_report_link(softc); return; } static int bnxt_media_change(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifmedia *ifm = iflib_get_media(ctx); struct ifmediareq ifmr; int rc; if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return EINVAL; switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_100_T: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100MB; break; case IFM_1000_KX: case IFM_1000_T: case IFM_1000_SGMII: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_1GB; break; case IFM_2500_KX: case IFM_2500_T: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2_5GB; break; case IFM_10G_CR1: case IFM_10G_KR: case IFM_10G_LR: case IFM_10G_SR: case IFM_10G_T: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB; break; case IFM_20G_KR2: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_20GB; break; case IFM_25G_CR: case IFM_25G_KR: case IFM_25G_SR: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_25GB; break; case IFM_40G_CR4: case IFM_40G_KR4: case IFM_40G_LR4: case IFM_40G_SR4: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB; break; case IFM_50G_CR2: case IFM_50G_KR2: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB; break; case IFM_100G_CR4: case IFM_100G_KR4: case IFM_100G_LR4: case IFM_100G_SR4: softc->link_info.autoneg &= ~BNXT_AUTONEG_SPEED; softc->link_info.req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100GB; break; default: device_printf(softc->dev, "Unsupported media type! Using auto\n"); /* Fall-through */ case IFM_AUTO: // Auto softc->link_info.autoneg |= BNXT_AUTONEG_SPEED; break; } rc = bnxt_hwrm_set_link_setting(softc, true, true); bnxt_media_status(softc->ctx, &ifmr); return rc; } static int bnxt_promisc_set(if_ctx_t ctx, int flags) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); int rc; if (ifp->if_flags & IFF_ALLMULTI || if_multiaddr_count(ifp, -1) > BNXT_MAX_MC_ADDRS) softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; else softc->vnic_info.rx_mask &= ~HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; if (ifp->if_flags & IFF_PROMISC) softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS | HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN; else softc->vnic_info.rx_mask &= ~(HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS | HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN); rc = bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); return rc; } static uint64_t bnxt_get_counter(if_ctx_t ctx, ift_counter cnt) { if_t ifp = iflib_get_ifp(ctx); if (cnt < IFCOUNTERS) return if_get_counter_default(ifp, cnt); return 0; } static void bnxt_update_admin_status(if_ctx_t ctx) { /* TODO: do we need to do anything here? */ return; } static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr) { if (cpr->ring.phys_id != (uint16_t)HWRM_NA_SIGNATURE) { /* First time enabling, do not set index */ if (cpr->cons == UINT32_MAX) BNXT_CP_ENABLE_DB(&cpr->ring); else BNXT_CP_IDX_ENABLE_DB(&cpr->ring, cpr->cons); } } static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr) { if (cpr->ring.phys_id != (uint16_t)HWRM_NA_SIGNATURE) BNXT_CP_DISABLE_DB(&cpr->ring); } /* Enable all interrupts */ static void bnxt_intr_enable(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); int i; bnxt_do_enable_intr(&softc->def_cp_ring); for (i = 0; i < softc->nrxqsets; i++) bnxt_do_enable_intr(&softc->rx_cp_rings[i]); return; } /* Enable interrupt for a single queue */ static int bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) { struct bnxt_softc *softc = iflib_get_softc(ctx); bnxt_do_enable_intr(&softc->tx_cp_rings[qid]); return 0; } static int bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) { struct bnxt_softc *softc = iflib_get_softc(ctx); bnxt_do_enable_intr(&softc->rx_cp_rings[qid]); return 0; } /* Disable all interrupts */ static void bnxt_disable_intr(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); int i; /* * NOTE: These TX interrupts should never get enabled, so don't * update the index */ for (i = 0; i < softc->ntxqsets; i++) bnxt_do_disable_intr(&softc->tx_cp_rings[i]); for (i = 0; i < softc->nrxqsets; i++) bnxt_do_disable_intr(&softc->rx_cp_rings[i]); return; } static int bnxt_msix_intr_assign(if_ctx_t ctx, int msix) { struct bnxt_softc *softc = iflib_get_softc(ctx); int rc; int i; char irq_name[16]; rc = iflib_irq_alloc_generic(ctx, &softc->def_cp_ring.irq, softc->def_cp_ring.ring.id + 1, IFLIB_INTR_ADMIN, bnxt_handle_def_cp, softc, 0, "def_cp"); if (rc) { device_printf(iflib_get_dev(ctx), "Failed to register default completion ring handler\n"); return rc; } for (i=0; iscctx->isc_nrxqsets; i++) { snprintf(irq_name, sizeof(irq_name), "rxq%d", i); rc = iflib_irq_alloc_generic(ctx, &softc->rx_cp_rings[i].irq, softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RX, bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, irq_name); if (rc) { device_printf(iflib_get_dev(ctx), "Failed to register RX completion ring handler\n"); i--; goto fail; } } for (i=0; iscctx->isc_ntxqsets; i++) iflib_softirq_alloc_generic(ctx, i + 1, IFLIB_INTR_TX, NULL, i, "tx_cp"); return rc; fail: for (; i>=0; i--) iflib_irq_free(ctx, &softc->rx_cp_rings[i].irq); iflib_irq_free(ctx, &softc->def_cp_ring.irq); return rc; } /* * We're explicitly allowing duplicates here. They will need to be * removed as many times as they are added. */ static void bnxt_vlan_register(if_ctx_t ctx, uint16_t vtag) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *new_tag; new_tag = malloc(sizeof(struct bnxt_vlan_tag), M_DEVBUF, M_NOWAIT); if (new_tag == NULL) return; new_tag->tag = vtag; new_tag->tpid = 8100; SLIST_INSERT_HEAD(&softc->vnic_info.vlan_tags, new_tag, next); }; static void bnxt_vlan_unregister(if_ctx_t ctx, uint16_t vtag) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *vlan_tag; SLIST_FOREACH(vlan_tag, &softc->vnic_info.vlan_tags, next) { if (vlan_tag->tag == vtag) { SLIST_REMOVE(&softc->vnic_info.vlan_tags, vlan_tag, bnxt_vlan_tag, next); free(vlan_tag, M_DEVBUF); break; } } } static int bnxt_wol_config(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); if (!softc) return -EBUSY; if (!bnxt_wol_supported(softc)) return -ENOTSUP; if (if_getcapabilities(ifp) & IFCAP_WOL_MAGIC) { if (!softc->wol) { if (bnxt_hwrm_alloc_wol_fltr(softc)) return -EBUSY; softc->wol = 1; } } else { if (softc->wol) { if (bnxt_hwrm_free_wol_fltr(softc)) return -EBUSY; softc->wol = 0; } } return 0; } static int bnxt_shutdown(if_ctx_t ctx) { bnxt_wol_config(ctx); return 0; } static int bnxt_suspend(if_ctx_t ctx) { bnxt_wol_config(ctx); return 0; } static int bnxt_resume(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); bnxt_get_wol_settings(softc); return 0; } static int bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifreq *ifr = (struct ifreq *)data; struct ifreq_buffer *ifbuf = &ifr->ifr_ifru.ifru_buffer; struct bnxt_ioctl_header *ioh = (struct bnxt_ioctl_header *)(ifbuf->buffer); int rc = ENOTSUP; struct bnxt_ioctl_data *iod = NULL; switch (command) { case SIOCGPRIVATE_0: if ((rc = priv_check(curthread, PRIV_DRIVER)) != 0) goto exit; iod = malloc(ifbuf->length, M_DEVBUF, M_NOWAIT | M_ZERO); if (!iod) { rc = ENOMEM; goto exit; } copyin(ioh, iod, ifbuf->length); switch (ioh->type) { case BNXT_HWRM_NVM_FIND_DIR_ENTRY: { struct bnxt_ioctl_hwrm_nvm_find_dir_entry *find = &iod->find; rc = bnxt_hwrm_nvm_find_dir_entry(softc, find->type, &find->ordinal, find->ext, &find->index, find->use_index, find->search_opt, &find->data_length, &find->item_length, &find->fw_ver); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_READ: { struct bnxt_ioctl_hwrm_nvm_read *rd = &iod->read; struct iflib_dma_info dma_data; size_t offset; size_t remain; size_t csize; /* * Some HWRM versions can't read more than 0x8000 bytes */ rc = iflib_dma_alloc(softc->ctx, min(rd->length, 0x8000), &dma_data, BUS_DMA_NOWAIT); if (rc) break; for (remain = rd->length, offset = 0; remain && offset < rd->length; offset += 0x8000) { csize = min(remain, 0x8000); rc = bnxt_hwrm_nvm_read(softc, rd->index, rd->offset + offset, csize, &dma_data); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); break; } else { copyout(dma_data.idi_vaddr, rd->data + offset, csize); iod->hdr.rc = 0; } remain -= csize; } if (iod->hdr.rc == 0) copyout(iod, ioh, ifbuf->length); iflib_dma_free(&dma_data); rc = 0; goto exit; } case BNXT_HWRM_FW_RESET: { struct bnxt_ioctl_hwrm_fw_reset *rst = &iod->reset; rc = bnxt_hwrm_fw_reset(softc, rst->processor, &rst->selfreset); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_FW_QSTATUS: { struct bnxt_ioctl_hwrm_fw_qstatus *qstat = &iod->status; rc = bnxt_hwrm_fw_qstatus(softc, qstat->processor, &qstat->selfreset); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_WRITE: { struct bnxt_ioctl_hwrm_nvm_write *wr = &iod->write; rc = bnxt_hwrm_nvm_write(softc, wr->data, true, wr->type, wr->ordinal, wr->ext, wr->attr, wr->option, wr->data_length, wr->keep, &wr->item_length, &wr->index); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_ERASE_DIR_ENTRY: { struct bnxt_ioctl_hwrm_nvm_erase_dir_entry *erase = &iod->erase; rc = bnxt_hwrm_nvm_erase_dir_entry(softc, erase->index); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_GET_DIR_INFO: { struct bnxt_ioctl_hwrm_nvm_get_dir_info *info = &iod->dir_info; rc = bnxt_hwrm_nvm_get_dir_info(softc, &info->entries, &info->entry_length); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_GET_DIR_ENTRIES: { struct bnxt_ioctl_hwrm_nvm_get_dir_entries *get = &iod->dir_entries; struct iflib_dma_info dma_data; rc = iflib_dma_alloc(softc->ctx, get->max_size, &dma_data, BUS_DMA_NOWAIT); if (rc) break; rc = bnxt_hwrm_nvm_get_dir_entries(softc, &get->entries, &get->entry_length, &dma_data); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { copyout(dma_data.idi_vaddr, get->data, get->entry_length * get->entries); iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } iflib_dma_free(&dma_data); rc = 0; goto exit; } case BNXT_HWRM_NVM_VERIFY_UPDATE: { struct bnxt_ioctl_hwrm_nvm_verify_update *vrfy = &iod->verify; rc = bnxt_hwrm_nvm_verify_update(softc, vrfy->type, vrfy->ordinal, vrfy->ext); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_INSTALL_UPDATE: { struct bnxt_ioctl_hwrm_nvm_install_update *inst = &iod->install; rc = bnxt_hwrm_nvm_install_update(softc, inst->install_type, &inst->installed_items, &inst->result, &inst->problem_item, &inst->reset_required); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_NVM_MODIFY: { struct bnxt_ioctl_hwrm_nvm_modify *mod = &iod->modify; rc = bnxt_hwrm_nvm_modify(softc, mod->index, mod->offset, mod->data, true, mod->length); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_FW_GET_TIME: { struct bnxt_ioctl_hwrm_fw_get_time *gtm = &iod->get_time; rc = bnxt_hwrm_fw_get_time(softc, >m->year, >m->month, >m->day, >m->hour, >m->minute, >m->second, >m->millisecond, >m->zone); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } case BNXT_HWRM_FW_SET_TIME: { struct bnxt_ioctl_hwrm_fw_set_time *stm = &iod->set_time; rc = bnxt_hwrm_fw_set_time(softc, stm->year, stm->month, stm->day, stm->hour, stm->minute, stm->second, stm->millisecond, stm->zone); if (rc) { iod->hdr.rc = rc; copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; copyout(iod, ioh, ifbuf->length); } rc = 0; goto exit; } } break; } exit: if (iod) free(iod, M_DEVBUF); return rc; } /* * Support functions */ static int bnxt_probe_phy(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; int rc = 0; rc = bnxt_update_link(softc, false); if (rc) { device_printf(softc->dev, "Probe phy can't update link (rc: %x)\n", rc); return (rc); } /*initialize the ethool setting copy with NVM settings */ if (link_info->auto_mode != HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE) link_info->autoneg |= BNXT_AUTONEG_SPEED; if (link_info->auto_pause & (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX | HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) { if (link_info->auto_pause == ( HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX | HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; link_info->req_flow_ctrl = link_info->auto_pause; } else if (link_info->force_pause & ( HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX | HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) { link_info->req_flow_ctrl = link_info->force_pause; } link_info->req_duplex = link_info->duplex_setting; if (link_info->autoneg & BNXT_AUTONEG_SPEED) link_info->req_link_speed = link_info->auto_link_speed; else link_info->req_link_speed = link_info->force_link_speed; return (rc); } static void bnxt_add_media_types(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; uint16_t supported; uint8_t phy_type = get_phy_type(softc); supported = link_info->support_speeds; /* Auto is always supported */ ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); if (softc->flags & BNXT_FLAG_NPAR) return; switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) ifmedia_add(softc->media, IFM_ETHER | IFM_100G_CR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB) ifmedia_add(softc->media, IFM_ETHER | IFM_50G_CR2, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB) ifmedia_add(softc->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB) ifmedia_add(softc->media, IFM_ETHER | IFM_25G_CR, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) ifmedia_add(softc->media, IFM_ETHER | IFM_100G_KR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB) ifmedia_add(softc->media, IFM_ETHER | IFM_50G_KR2, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB) ifmedia_add(softc->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB) ifmedia_add(softc->media, IFM_ETHER | IFM_25G_KR, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB) ifmedia_add(softc->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) ifmedia_add(softc->media, IFM_ETHER | IFM_100G_LR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB) ifmedia_add(softc->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) ifmedia_add(softc->media, IFM_ETHER | IFM_100G_SR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB) ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB) ifmedia_add(softc->media, IFM_ETHER | IFM_25G_SR, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB) ifmedia_add(softc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB) ifmedia_add(softc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB) ifmedia_add(softc->media, IFM_ETHER | IFM_10_T, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB) ifmedia_add(softc->media, IFM_ETHER | IFM_100_T, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB) ifmedia_add(softc->media, IFM_ETHER | IFM_1000_T, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB) ifmedia_add(softc->media, IFM_ETHER | IFM_2500_T, 0, NULL); if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB) ifmedia_add(softc->media, IFM_ETHER | IFM_10G_T, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY: if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB) ifmedia_add(softc->media, IFM_ETHER | IFM_1000_SGMII, 0, NULL); break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN: default: /* * Workaround for Cumulus & Stratus * For Stratus: * media_type is being returned as 0x0 * Return support speeds as 10G, 25G, 50G & 100G * * For Cumulus: * phy_type is being returned as 0x14 (PHY_TYPE_40G_BASECR4) * Return support speeds as 1G, 10G, 25G & 50G */ if (pci_get_device(softc->dev) == BCM57454) { /* For Stratus: 10G, 25G, 50G & 100G */ ifmedia_add(softc->media, IFM_ETHER | IFM_100G_CR4, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_50G_CR2, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_25G_CR, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); } else if (pci_get_device(softc->dev) == BCM57414) { /* For Cumulus: 1G, 10G, 25G & 50G */ ifmedia_add(softc->media, IFM_ETHER | IFM_50G_CR2, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_25G_CR, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); ifmedia_add(softc->media, IFM_ETHER | IFM_1000_T, 0, NULL); } break; } return; } static int bnxt_map_bar(struct bnxt_softc *softc, struct bnxt_bar_info *bar, int bar_num, bool shareable) { uint32_t flag; if (bar->res != NULL) { device_printf(softc->dev, "Bar %d already mapped\n", bar_num); return EDOOFUS; } bar->rid = PCIR_BAR(bar_num); flag = RF_ACTIVE; if (shareable) flag |= RF_SHAREABLE; if ((bar->res = bus_alloc_resource_any(softc->dev, SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { device_printf(softc->dev, "PCI BAR%d mapping failure\n", bar_num); return (ENXIO); } bar->tag = rman_get_bustag(bar->res); bar->handle = rman_get_bushandle(bar->res); bar->size = rman_get_size(bar->res); return 0; } static int bnxt_pci_mapping(struct bnxt_softc *softc) { int rc; rc = bnxt_map_bar(softc, &softc->hwrm_bar, 0, true); if (rc) return rc; rc = bnxt_map_bar(softc, &softc->doorbell_bar, 2, false); return rc; } static void bnxt_pci_mapping_free(struct bnxt_softc *softc) { if (softc->hwrm_bar.res != NULL) bus_release_resource(softc->dev, SYS_RES_MEMORY, softc->hwrm_bar.rid, softc->hwrm_bar.res); softc->hwrm_bar.res = NULL; if (softc->doorbell_bar.res != NULL) bus_release_resource(softc->dev, SYS_RES_MEMORY, softc->doorbell_bar.rid, softc->doorbell_bar.res); softc->doorbell_bar.res = NULL; } static int bnxt_update_link(struct bnxt_softc *softc, bool chng_link_state) { struct bnxt_link_info *link_info = &softc->link_info; uint8_t link_up = link_info->link_up; int rc = 0; rc = bnxt_hwrm_port_phy_qcfg(softc); if (rc) goto exit; /* TODO: need to add more logic to report VF link */ if (chng_link_state) { if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) link_info->link_up = 1; else link_info->link_up = 0; if (link_up != link_info->link_up) bnxt_report_link(softc); } else { /* always link down if not require to update link state */ link_info->link_up = 0; } exit: return rc; } void bnxt_report_link(struct bnxt_softc *softc) { const char *duplex = NULL, *flow_ctrl = NULL; if (softc->link_info.link_up == softc->link_info.last_link_up) { if (!softc->link_info.link_up) return; if (softc->link_info.pause == softc->link_info.last_pause && softc->link_info.duplex == softc->link_info.last_duplex) return; } if (softc->link_info.link_up) { if (softc->link_info.duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL) duplex = "full duplex"; else duplex = "half duplex"; if (softc->link_info.pause == ( HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX | HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) flow_ctrl = "FC - receive & transmit"; else if (softc->link_info.pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX) flow_ctrl = "FC - transmit"; else if (softc->link_info.pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX) flow_ctrl = "FC - receive"; else - flow_ctrl = "none"; + flow_ctrl = "FC - none"; iflib_link_state_change(softc->ctx, LINK_STATE_UP, IF_Gbps(100)); - device_printf(softc->dev, "Link is UP %s, %s\n", duplex, - flow_ctrl); + device_printf(softc->dev, "Link is UP %s, %s - %d Mbps \n", duplex, + flow_ctrl, (softc->link_info.link_speed * 100)); } else { iflib_link_state_change(softc->ctx, LINK_STATE_DOWN, bnxt_get_baudrate(&softc->link_info)); device_printf(softc->dev, "Link is Down\n"); } softc->link_info.last_link_up = softc->link_info.link_up; softc->link_info.last_pause = softc->link_info.pause; softc->link_info.last_duplex = softc->link_info.duplex; } static int bnxt_handle_rx_cp(void *arg) { struct bnxt_cp_ring *cpr = arg; /* Disable further interrupts for this queue */ BNXT_CP_DISABLE_DB(&cpr->ring); return FILTER_SCHEDULE_THREAD; } static int bnxt_handle_def_cp(void *arg) { struct bnxt_softc *softc = arg; BNXT_CP_DISABLE_DB(&softc->def_cp_ring.ring); GROUPTASK_ENQUEUE(&softc->def_cp_task); return FILTER_HANDLED; } static void bnxt_clear_ids(struct bnxt_softc *softc) { int i; softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; for (i = 0; i < softc->ntxqsets; i++) { softc->tx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; } for (i = 0; i < softc->nrxqsets; i++) { softc->rx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE; } softc->vnic_info.filter_id = -1; softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.rss_id = (uint16_t)HWRM_NA_SIGNATURE; memset(softc->vnic_info.rss_grp_tbl.idi_vaddr, 0xff, softc->vnic_info.rss_grp_tbl.idi_size); } static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr) { struct cmpl_base *cmp = (void *)cpr->ring.vaddr; int i; for (i = 0; i < cpr->ring.ring_size; i++) cmp[i].info3_v = !cpr->v_bit; } static void bnxt_handle_async_event(struct bnxt_softc *softc, struct cmpl_base *cmpl) { struct hwrm_async_event_cmpl *ae = (void *)cmpl; uint16_t async_id = le16toh(ae->event_id); struct ifmediareq ifmr; switch (async_id) { case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: bnxt_media_status(softc->ctx, &ifmr); break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR: device_printf(softc->dev, "Unhandled async completion type %u\n", async_id); break; default: device_printf(softc->dev, "Unknown async completion type %u\n", async_id); break; } } static void bnxt_def_cp_task(void *context) { if_ctx_t ctx = context; struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_cp_ring *cpr = &softc->def_cp_ring; /* Handle completions on the default completion ring */ struct cmpl_base *cmpl; uint32_t cons = cpr->cons; bool v_bit = cpr->v_bit; bool last_v_bit; uint32_t last_cons; uint16_t type; for (;;) { last_cons = cons; last_v_bit = v_bit; NEXT_CP_CONS_V(&cpr->ring, cons, v_bit); cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons]; if (!CMP_VALID(cmpl, v_bit)) break; type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK; switch (type) { case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: bnxt_handle_async_event(softc, cmpl); break; case CMPL_BASE_TYPE_TX_L2: case CMPL_BASE_TYPE_RX_L2: case CMPL_BASE_TYPE_RX_AGG: case CMPL_BASE_TYPE_RX_TPA_START: case CMPL_BASE_TYPE_RX_TPA_END: case CMPL_BASE_TYPE_STAT_EJECT: case CMPL_BASE_TYPE_HWRM_DONE: case CMPL_BASE_TYPE_HWRM_FWD_REQ: case CMPL_BASE_TYPE_HWRM_FWD_RESP: case CMPL_BASE_TYPE_CQ_NOTIFICATION: case CMPL_BASE_TYPE_SRQ_EVENT: case CMPL_BASE_TYPE_DBQ_EVENT: case CMPL_BASE_TYPE_QP_EVENT: case CMPL_BASE_TYPE_FUNC_EVENT: device_printf(softc->dev, "Unhandled completion type %u\n", type); break; default: device_printf(softc->dev, "Unknown completion type %u\n", type); break; } } cpr->cons = last_cons; cpr->v_bit = last_v_bit; BNXT_CP_IDX_ENABLE_DB(&cpr->ring, cpr->cons); } static uint8_t get_phy_type(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; uint8_t phy_type = link_info->phy_type; uint16_t supported; if (phy_type != HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN) return phy_type; /* Deduce the phy type from the media type and supported speeds */ supported = link_info->support_speeds; if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_TP) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET; if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_DAC) { if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX; if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR; return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR; } if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_FIBRE) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR; return phy_type; } bool bnxt_check_hwrm_version(struct bnxt_softc *softc) { char buf[16]; sprintf(buf, "%hhu.%hhu.%hhu", softc->ver_info->hwrm_min_major, softc->ver_info->hwrm_min_minor, softc->ver_info->hwrm_min_update); if (softc->ver_info->hwrm_min_major > softc->ver_info->hwrm_if_major) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } else if(softc->ver_info->hwrm_min_major == softc->ver_info->hwrm_if_major) { if (softc->ver_info->hwrm_min_minor > softc->ver_info->hwrm_if_minor) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } else if (softc->ver_info->hwrm_min_minor == softc->ver_info->hwrm_if_minor) { if (softc->ver_info->hwrm_min_update > softc->ver_info->hwrm_if_update) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } } } return true; } static uint64_t bnxt_get_baudrate(struct bnxt_link_info *link) { switch (link->link_speed) { case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB: return IF_Mbps(100); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB: return IF_Gbps(1); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2GB: return IF_Gbps(2); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB: return IF_Mbps(2500); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB: return IF_Gbps(10); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB: return IF_Gbps(20); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB: return IF_Gbps(25); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB: return IF_Gbps(40); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB: return IF_Gbps(50); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB: return IF_Gbps(100); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10MB: return IF_Mbps(10); } return IF_Gbps(100); } static void bnxt_get_wol_settings(struct bnxt_softc *softc) { uint16_t wol_handle = 0; if (!bnxt_wol_supported(softc)) return; do { wol_handle = bnxt_hwrm_get_wol_fltrs(softc, wol_handle); } while (wol_handle && wol_handle != BNXT_NO_MORE_WOL_FILTERS); } Index: projects/runtime-coverage/sys/dev/cxgbe/common/common.h =================================================================== --- projects/runtime-coverage/sys/dev/cxgbe/common/common.h (revision 321584) +++ projects/runtime-coverage/sys/dev/cxgbe/common/common.h (revision 321585) @@ -1,832 +1,848 @@ /*- * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __CHELSIO_COMMON_H #define __CHELSIO_COMMON_H #include "t4_hw.h" #define GLBL_INTR_MASK (F_CIM | F_MPS | F_PL | F_PCIE | F_MC0 | F_EDC0 | \ F_EDC1 | F_LE | F_TP | F_MA | F_PM_TX | F_PM_RX | F_ULP_RX | \ F_CPL_SWITCH | F_SGE | F_ULP_TX) enum { MAX_NPORTS = 4, /* max # of ports */ SERNUM_LEN = 24, /* Serial # length */ EC_LEN = 16, /* E/C length */ ID_LEN = 16, /* ID length */ PN_LEN = 16, /* Part Number length */ MACADDR_LEN = 12, /* MAC Address length */ }; enum { T4_REGMAP_SIZE = (160 * 1024), T5_REGMAP_SIZE = (332 * 1024), }; enum { MEM_EDC0, MEM_EDC1, MEM_MC, MEM_MC0 = MEM_MC, MEM_MC1 }; enum dev_master { MASTER_CANT, MASTER_MAY, MASTER_MUST }; enum dev_state { DEV_STATE_UNINIT, DEV_STATE_INIT, DEV_STATE_ERR }; enum { PAUSE_RX = 1 << 0, PAUSE_TX = 1 << 1, PAUSE_AUTONEG = 1 << 2 }; enum { FEC_RS = 1 << 0, FEC_BASER_RS = 1 << 1, FEC_RESERVED = 1 << 2, }; struct port_stats { u64 tx_octets; /* total # of octets in good frames */ u64 tx_frames; /* all good frames */ u64 tx_bcast_frames; /* all broadcast frames */ u64 tx_mcast_frames; /* all multicast frames */ u64 tx_ucast_frames; /* all unicast frames */ u64 tx_error_frames; /* all error frames */ u64 tx_frames_64; /* # of Tx frames in a particular range */ u64 tx_frames_65_127; u64 tx_frames_128_255; u64 tx_frames_256_511; u64 tx_frames_512_1023; u64 tx_frames_1024_1518; u64 tx_frames_1519_max; u64 tx_drop; /* # of dropped Tx frames */ u64 tx_pause; /* # of transmitted pause frames */ u64 tx_ppp0; /* # of transmitted PPP prio 0 frames */ u64 tx_ppp1; /* # of transmitted PPP prio 1 frames */ u64 tx_ppp2; /* # of transmitted PPP prio 2 frames */ u64 tx_ppp3; /* # of transmitted PPP prio 3 frames */ u64 tx_ppp4; /* # of transmitted PPP prio 4 frames */ u64 tx_ppp5; /* # of transmitted PPP prio 5 frames */ u64 tx_ppp6; /* # of transmitted PPP prio 6 frames */ u64 tx_ppp7; /* # of transmitted PPP prio 7 frames */ u64 rx_octets; /* total # of octets in good frames */ u64 rx_frames; /* all good frames */ u64 rx_bcast_frames; /* all broadcast frames */ u64 rx_mcast_frames; /* all multicast frames */ u64 rx_ucast_frames; /* all unicast frames */ u64 rx_too_long; /* # of frames exceeding MTU */ u64 rx_jabber; /* # of jabber frames */ u64 rx_fcs_err; /* # of received frames with bad FCS */ u64 rx_len_err; /* # of received frames with length error */ u64 rx_symbol_err; /* symbol errors */ u64 rx_runt; /* # of short frames */ u64 rx_frames_64; /* # of Rx frames in a particular range */ u64 rx_frames_65_127; u64 rx_frames_128_255; u64 rx_frames_256_511; u64 rx_frames_512_1023; u64 rx_frames_1024_1518; u64 rx_frames_1519_max; u64 rx_pause; /* # of received pause frames */ u64 rx_ppp0; /* # of received PPP prio 0 frames */ u64 rx_ppp1; /* # of received PPP prio 1 frames */ u64 rx_ppp2; /* # of received PPP prio 2 frames */ u64 rx_ppp3; /* # of received PPP prio 3 frames */ u64 rx_ppp4; /* # of received PPP prio 4 frames */ u64 rx_ppp5; /* # of received PPP prio 5 frames */ u64 rx_ppp6; /* # of received PPP prio 6 frames */ u64 rx_ppp7; /* # of received PPP prio 7 frames */ u64 rx_ovflow0; /* drops due to buffer-group 0 overflows */ u64 rx_ovflow1; /* drops due to buffer-group 1 overflows */ u64 rx_ovflow2; /* drops due to buffer-group 2 overflows */ u64 rx_ovflow3; /* drops due to buffer-group 3 overflows */ u64 rx_trunc0; /* buffer-group 0 truncated packets */ u64 rx_trunc1; /* buffer-group 1 truncated packets */ u64 rx_trunc2; /* buffer-group 2 truncated packets */ u64 rx_trunc3; /* buffer-group 3 truncated packets */ }; struct lb_port_stats { u64 octets; u64 frames; u64 bcast_frames; u64 mcast_frames; u64 ucast_frames; u64 error_frames; u64 frames_64; u64 frames_65_127; u64 frames_128_255; u64 frames_256_511; u64 frames_512_1023; u64 frames_1024_1518; u64 frames_1519_max; u64 drop; u64 ovflow0; u64 ovflow1; u64 ovflow2; u64 ovflow3; u64 trunc0; u64 trunc1; u64 trunc2; u64 trunc3; }; struct tp_tcp_stats { u32 tcp_out_rsts; u64 tcp_in_segs; u64 tcp_out_segs; u64 tcp_retrans_segs; }; struct tp_usm_stats { u32 frames; u32 drops; u64 octets; }; struct tp_fcoe_stats { u32 frames_ddp; u32 frames_drop; u64 octets_ddp; }; struct tp_err_stats { u32 mac_in_errs[MAX_NCHAN]; u32 hdr_in_errs[MAX_NCHAN]; u32 tcp_in_errs[MAX_NCHAN]; u32 tnl_cong_drops[MAX_NCHAN]; u32 ofld_chan_drops[MAX_NCHAN]; u32 tnl_tx_drops[MAX_NCHAN]; u32 ofld_vlan_drops[MAX_NCHAN]; u32 tcp6_in_errs[MAX_NCHAN]; u32 ofld_no_neigh; u32 ofld_cong_defer; }; struct tp_proxy_stats { u32 proxy[MAX_NCHAN]; }; struct tp_cpl_stats { u32 req[MAX_NCHAN]; u32 rsp[MAX_NCHAN]; }; struct tp_rdma_stats { u32 rqe_dfr_pkt; u32 rqe_dfr_mod; }; struct sge_params { int timer_val[SGE_NTIMERS]; /* final, scaled values */ int counter_val[SGE_NCOUNTERS]; int fl_starve_threshold; int fl_starve_threshold2; int page_shift; int eq_s_qpp; int iq_s_qpp; int spg_len; int pad_boundary; int pack_boundary; int fl_pktshift; u32 sge_control; u32 sge_fl_buffer_size[SGE_FLBUF_SIZES]; }; struct tp_params { unsigned int tre; /* log2 of core clocks per TP tick */ unsigned int dack_re; /* DACK timer resolution */ unsigned int la_mask; /* what events are recorded by TP LA */ unsigned short tx_modq[MAX_NCHAN]; /* channel to modulation queue map */ uint32_t vlan_pri_map; uint32_t ingress_config; __be16 err_vec_mask; int8_t fcoe_shift; int8_t port_shift; int8_t vnic_shift; int8_t vlan_shift; int8_t tos_shift; int8_t protocol_shift; int8_t ethertype_shift; int8_t macmatch_shift; int8_t matchtype_shift; int8_t frag_shift; }; struct vpd_params { unsigned int cclk; u8 ec[EC_LEN + 1]; u8 sn[SERNUM_LEN + 1]; u8 id[ID_LEN + 1]; u8 pn[PN_LEN + 1]; u8 na[MACADDR_LEN + 1]; }; struct pci_params { unsigned int vpd_cap_addr; unsigned int mps; unsigned short speed; unsigned short width; }; /* * Firmware device log. */ struct devlog_params { u32 memtype; /* which memory (FW_MEMTYPE_* ) */ u32 start; /* start of log in firmware memory */ u32 size; /* size of log */ u32 addr; /* start address in flat addr space */ }; /* Stores chip specific parameters */ struct chip_params { u8 nchan; u8 pm_stats_cnt; u8 cng_ch_bits_log; /* congestion channel map bits width */ u8 nsched_cls; u8 cim_num_obq; u16 mps_rplc_size; u16 vfcount; u32 sge_fl_db; u16 mps_tcam_size; }; /* VF-only parameters. */ /* * Global Receive Side Scaling (RSS) parameters in host-native format. */ struct rss_params { unsigned int mode; /* RSS mode */ union { struct { u_int synmapen:1; /* SYN Map Enable */ u_int syn4tupenipv6:1; /* enable hashing 4-tuple IPv6 SYNs */ u_int syn2tupenipv6:1; /* enable hashing 2-tuple IPv6 SYNs */ u_int syn4tupenipv4:1; /* enable hashing 4-tuple IPv4 SYNs */ u_int syn2tupenipv4:1; /* enable hashing 2-tuple IPv4 SYNs */ u_int ofdmapen:1; /* Offload Map Enable */ u_int tnlmapen:1; /* Tunnel Map Enable */ u_int tnlalllookup:1; /* Tunnel All Lookup */ u_int hashtoeplitz:1; /* use Toeplitz hash */ } basicvirtual; } u; }; /* * Maximum resources provisioned for a PCI VF. */ struct vf_resources { unsigned int nvi; /* N virtual interfaces */ unsigned int neq; /* N egress Qs */ unsigned int nethctrl; /* N egress ETH or CTRL Qs */ unsigned int niqflint; /* N ingress Qs/w free list(s) & intr */ unsigned int niq; /* N ingress Qs */ unsigned int tc; /* PCI-E traffic class */ unsigned int pmask; /* port access rights mask */ unsigned int nexactf; /* N exact MPS filters */ unsigned int r_caps; /* read capabilities */ unsigned int wx_caps; /* write/execute capabilities */ }; struct adapter_params { struct sge_params sge; struct tp_params tp; /* PF-only */ struct vpd_params vpd; struct pci_params pci; struct devlog_params devlog; /* PF-only */ struct rss_params rss; /* VF-only */ struct vf_resources vfres; /* VF-only */ unsigned int sf_size; /* serial flash size in bytes */ unsigned int sf_nsec; /* # of flash sectors */ unsigned int fw_vers; /* firmware version */ unsigned int bs_vers; /* bootstrap version */ unsigned int tp_vers; /* TP microcode version */ unsigned int er_vers; /* expansion ROM version */ unsigned int scfg_vers; /* Serial Configuration version */ unsigned int vpd_vers; /* VPD version */ unsigned short mtus[NMTUS]; unsigned short a_wnd[NCCTRL_WIN]; unsigned short b_wnd[NCCTRL_WIN]; u_int ftid_min; u_int ftid_max; u_int etid_min; u_int netids; unsigned int cim_la_size; uint8_t nports; /* # of ethernet ports */ uint8_t portvec; unsigned int chipid:4; /* chip ID. T4 = 4, T5 = 5, ... */ unsigned int rev:4; /* chip revision */ unsigned int fpga:1; /* this is an FPGA */ unsigned int offload:1; /* hw is TOE capable, fw has divvied up card resources for TOE operation. */ unsigned int bypass:1; /* this is a bypass card */ unsigned int ethoffload:1; unsigned int ofldq_wr_cred; unsigned int eo_wr_cred; unsigned int max_ordird_qp; unsigned int max_ird_adapter; }; #define CHELSIO_T4 0x4 #define CHELSIO_T5 0x5 #define CHELSIO_T6 0x6 /* * State needed to monitor the forward progress of SGE Ingress DMA activities * and possible hangs. */ struct sge_idma_monitor_state { unsigned int idma_1s_thresh; /* 1s threshold in Core Clock ticks */ unsigned int idma_stalled[2]; /* synthesized stalled timers in HZ */ unsigned int idma_state[2]; /* IDMA Hang detect state */ unsigned int idma_qid[2]; /* IDMA Hung Ingress Queue ID */ unsigned int idma_warn[2]; /* time to warning in HZ */ }; struct trace_params { u32 data[TRACE_LEN / 4]; u32 mask[TRACE_LEN / 4]; unsigned short snap_len; unsigned short min_len; unsigned char skip_ofst; unsigned char skip_len; unsigned char invert; unsigned char port; }; struct link_config { /* OS-specific code owns all the requested_* fields */ unsigned char requested_aneg; /* link aneg user has requested */ unsigned char requested_fc; /* flow control user has requested */ unsigned char requested_fec; /* FEC user has requested */ unsigned int requested_speed; /* speed user has requested */ unsigned short supported; /* link capabilities */ unsigned short advertising; /* advertised capabilities */ unsigned short lp_advertising; /* peer advertised capabilities */ unsigned int speed; /* actual link speed */ unsigned char fc; /* actual link flow control */ unsigned char fec; /* actual FEC */ unsigned char link_ok; /* link up? */ unsigned char link_down_rc; /* link down reason */ }; #include "adapter.h" #ifndef PCI_VENDOR_ID_CHELSIO # define PCI_VENDOR_ID_CHELSIO 0x1425 #endif #define for_each_port(adapter, iter) \ for (iter = 0; iter < (adapter)->params.nports; ++iter) static inline int is_ftid(const struct adapter *sc, u_int tid) { return (tid >= sc->params.ftid_min && tid <= sc->params.ftid_max); } static inline int is_etid(const struct adapter *sc, u_int tid) { return (tid >= sc->params.etid_min); } static inline int is_offload(const struct adapter *adap) { return adap->params.offload; } static inline int is_ethoffload(const struct adapter *adap) { return adap->params.ethoffload; } static inline int chip_id(struct adapter *adap) { return adap->params.chipid; } static inline int chip_rev(struct adapter *adap) { return adap->params.rev; } static inline int is_t4(struct adapter *adap) { return adap->params.chipid == CHELSIO_T4; } static inline int is_t5(struct adapter *adap) { return adap->params.chipid == CHELSIO_T5; } static inline int is_t6(struct adapter *adap) { return adap->params.chipid == CHELSIO_T6; } static inline int is_fpga(struct adapter *adap) { return adap->params.fpga; } static inline unsigned int core_ticks_per_usec(const struct adapter *adap) { return adap->params.vpd.cclk / 1000; } static inline unsigned int us_to_core_ticks(const struct adapter *adap, unsigned int us) { return (us * adap->params.vpd.cclk) / 1000; } static inline unsigned int core_ticks_to_us(const struct adapter *adapter, unsigned int ticks) { /* add Core Clock / 2 to round ticks to nearest uS */ return ((ticks * 1000 + adapter->params.vpd.cclk/2) / adapter->params.vpd.cclk); } static inline unsigned int dack_ticks_to_usec(const struct adapter *adap, unsigned int ticks) { return (ticks << adap->params.tp.dack_re) / core_ticks_per_usec(adap); } static inline u_int ms_to_tcp_ticks(const struct adapter *adap, u_int ms) { u_long l; l = (u_long)ms * adap->params.vpd.cclk >> adap->params.tp.tre; return (l); } void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, u32 val); int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl, bool sleep_ok, int timeout); int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl, bool sleep_ok); static inline int t4_wr_mbox_timeout(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl, int timeout) { return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, true, timeout); } static inline int t4_wr_mbox(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl) { return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, true); } static inline int t4_wr_mbox_ns(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl) { return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, false); } void t4_read_indirect(struct adapter *adap, unsigned int addr_reg, unsigned int data_reg, u32 *vals, unsigned int nregs, unsigned int start_idx); void t4_write_indirect(struct adapter *adap, unsigned int addr_reg, unsigned int data_reg, const u32 *vals, unsigned int nregs, unsigned int start_idx); u32 t4_hw_pci_read_cfg4(adapter_t *adapter, int reg); struct fw_filter_wr; void t4_intr_enable(struct adapter *adapter); void t4_intr_disable(struct adapter *adapter); void t4_intr_clear(struct adapter *adapter); int t4_slow_intr_handler(struct adapter *adapter); int t4_hash_mac_addr(const u8 *addr); int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, struct link_config *lc); int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port); int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data); int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data); int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz); int t4_seeprom_wp(struct adapter *adapter, int enable); int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented); int t4_write_flash(struct adapter *adapter, unsigned int addr, unsigned int n, const u8 *data, int byte_oriented); int t4_load_fw(struct adapter *adapter, const u8 *fw_data, unsigned int size); int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op); int t5_fw_init_extern_mem(struct adapter *adap); int t4_load_bootcfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size); int t4_load_boot(struct adapter *adap, u8 *boot_data, unsigned int boot_addr, unsigned int size); int t4_flash_erase_sectors(struct adapter *adapter, int start, int end); int t4_flash_cfg_addr(struct adapter *adapter); int t4_load_cfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size); int t4_get_fw_version(struct adapter *adapter, u32 *vers); int t4_get_bs_version(struct adapter *adapter, u32 *vers); int t4_get_tp_version(struct adapter *adapter, u32 *vers); int t4_get_exprom_version(struct adapter *adapter, u32 *vers); int t4_get_scfg_version(struct adapter *adapter, u32 *vers); int t4_get_vpd_version(struct adapter *adapter, u32 *vers); int t4_get_version_info(struct adapter *adapter); int t4_init_hw(struct adapter *adapter, u32 fw_params); const struct chip_params *t4_get_chip_params(int chipid); int t4_prep_adapter(struct adapter *adapter, u8 *buf); int t4_shutdown_adapter(struct adapter *adapter); int t4_init_devlog_params(struct adapter *adapter, int fw_attach); int t4_init_sge_params(struct adapter *adapter); -int t4_init_tp_params(struct adapter *adap); +int t4_init_tp_params(struct adapter *adap, bool sleep_ok); int t4_filter_field_shift(const struct adapter *adap, int filter_sel); int t4_port_init(struct adapter *adap, int mbox, int pf, int vf, int port_id); void t4_fatal_err(struct adapter *adapter); void t4_db_full(struct adapter *adapter); void t4_db_dropped(struct adapter *adapter); int t4_set_trace_filter(struct adapter *adapter, const struct trace_params *tp, int filter_index, int enable); void t4_get_trace_filter(struct adapter *adapter, struct trace_params *tp, int filter_index, int *enabled); int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid, int start, int n, const u16 *rspq, unsigned int nrspq); int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode, unsigned int flags); int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid, unsigned int flags, unsigned int defq, unsigned int skeyidx, unsigned int skey); int t4_read_rss(struct adapter *adapter, u16 *entries); -void t4_fw_tp_pio_rw(struct adapter *adap, u32 *vals, unsigned int nregs, - unsigned int start_index, unsigned int rw); -void t4_read_rss_key(struct adapter *adapter, u32 *key); -void t4_write_rss_key(struct adapter *adap, u32 *key, int idx); -void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *valp); -void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val); +void t4_read_rss_key(struct adapter *adapter, u32 *key, bool sleep_ok); +void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx, + bool sleep_ok); +void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, + u32 *valp, bool sleep_ok); +void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, + u32 val, bool sleep_ok); void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index, - u32 *vfl, u32 *vfh); + u32 *vfl, u32 *vfh, bool sleep_ok); void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index, - u32 vfl, u32 vfh); -u32 t4_read_rss_pf_map(struct adapter *adapter); -void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap); -u32 t4_read_rss_pf_mask(struct adapter *adapter); -void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask); + u32 vfl, u32 vfh, bool sleep_ok); +u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok); +void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap, bool sleep_ok); +u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok); +void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask, bool sleep_ok); int t4_mps_set_active_ports(struct adapter *adap, unsigned int port_mask); void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]); void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]); void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres); int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n); int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n); int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n, unsigned int *valp); int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n, const unsigned int *valp); int t4_cim_ctl_read(struct adapter *adap, unsigned int addr, unsigned int n, unsigned int *valp); int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr); void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp, unsigned int *pif_req_wrptr, unsigned int *pif_rsp_wrptr); void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp); int t4_get_flash_params(struct adapter *adapter); u32 t4_read_pcie_cfg4(struct adapter *adap, int reg, int drv_fw_attach); int t4_mc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *parity); int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *parity); int t4_mem_read(struct adapter *adap, int mtype, u32 addr, u32 size, __be32 *data); void t4_idma_monitor_init(struct adapter *adapter, struct sge_idma_monitor_state *idma); void t4_idma_monitor(struct adapter *adapter, struct sge_idma_monitor_state *idma, int hz, int ticks); unsigned int t4_get_regs_len(struct adapter *adapter); void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size); const char *t4_get_port_type_description(enum fw_port_type port_type); void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p); void t4_get_port_stats_offset(struct adapter *adap, int idx, struct port_stats *stats, struct port_stats *offset); void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p); void t4_clr_port_stats(struct adapter *adap, int idx); void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log); void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN]); void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED]); void t4_get_tx_sched(struct adapter *adap, unsigned int sched, unsigned int *kbps, - unsigned int *ipg); + unsigned int *ipg, bool sleep_ok); void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr, unsigned int mask, unsigned int val); void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr); -void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st); -void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st); -void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st); -void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st); -void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st); +void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st, + bool sleep_ok); +void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st, + bool sleep_ok); +void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st, + bool sleep_ok); +void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st, + bool sleep_ok); +void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st, + bool sleep_ok); void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4, - struct tp_tcp_stats *v6); + struct tp_tcp_stats *v6, bool sleep_ok); void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx, - struct tp_fcoe_stats *st); + struct tp_fcoe_stats *st, bool sleep_ok); void t4_load_mtus(struct adapter *adap, const unsigned short *mtus, const unsigned short *alpha, const unsigned short *beta); void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf); int t4_set_sched_bps(struct adapter *adap, int sched, unsigned int kbps); int t4_set_sched_ipg(struct adapter *adap, int sched, unsigned int ipg); int t4_set_pace_tbl(struct adapter *adap, const unsigned int *pace_vals, unsigned int start, unsigned int n); void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate); -int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map); +int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map, + bool sleep_ok); void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid); void t4_wol_magic_enable(struct adapter *adap, unsigned int port, const u8 *addr); int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map, u64 mask0, u64 mask1, unsigned int crc, bool enable); int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox, enum dev_master master, enum dev_state *state); int t4_fw_bye(struct adapter *adap, unsigned int mbox); int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force); int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_upgrade(struct adapter *adap, unsigned int mbox, const u8 *fw_data, unsigned int size, int force); int t4_fw_forceinstall(struct adapter *adap, const u8 *fw_data, unsigned int size); int t4_fw_initialize(struct adapter *adap, unsigned int mbox); int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, u32 *val); int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, u32 *val, int rw); int t4_set_params_timeout(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val, int timeout); int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val); int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl, unsigned int rxqi, unsigned int rxq, unsigned int tc, unsigned int vi, unsigned int cmask, unsigned int pmask, unsigned int exactf, unsigned int rcaps, unsigned int wxcaps); int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox, unsigned int port, unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, u16 *rss_size, unsigned int portfunc, unsigned int idstype); int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port, unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, u16 *rss_size); int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int viid); int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid, int mtu, int promisc, int all_multi, int bcast, int vlanex, bool sleep_ok); int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox, unsigned int viid, bool free, unsigned int naddr, const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok); int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid, int idx, const u8 *addr, bool persist, bool add_smt); int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid, bool ucast, u64 vec, bool sleep_ok); int t4_enable_vi_params(struct adapter *adap, unsigned int mbox, unsigned int viid, bool rx_en, bool tx_en, bool dcb_en); int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid, bool rx_en, bool tx_en); int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid, unsigned int nblinks); int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, unsigned int mmd, unsigned int reg, unsigned int *valp); int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, unsigned int mmd, unsigned int reg, unsigned int val); int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port, unsigned int devid, unsigned int offset, unsigned int len, u8 *buf); int t4_i2c_wr(struct adapter *adap, unsigned int mbox, int port, unsigned int devid, unsigned int offset, unsigned int len, u8 *buf); int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id); int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id); int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid); int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid); int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid); int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid, enum ctxt_type ctype, u32 *data); int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, enum ctxt_type ctype, u32 *data); int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox); const char *t4_link_down_rc_str(unsigned char link_down_rc); int t4_update_port_info(struct port_info *pi); int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl); int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, u32 addr, u32 val); int t4_sched_config(struct adapter *adapter, int type, int minmaxen, int sleep_ok); int t4_sched_params(struct adapter *adapter, int type, int level, int mode, int rateunit, int ratemode, int channel, int cl, int minrate, int maxrate, int weight, int pktsize, int sleep_ok); int t4_sched_params_ch_rl(struct adapter *adapter, int channel, int ratemode, unsigned int maxrate, int sleep_ok); int t4_sched_params_cl_wrr(struct adapter *adapter, int channel, int cl, int weight, int sleep_ok); int t4_sched_params_cl_rl_kbps(struct adapter *adapter, int channel, int cl, int mode, unsigned int maxrate, int pktsize, int sleep_ok); int t4_config_watchdog(struct adapter *adapter, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int timeout, unsigned int action); int t4_get_devlog_level(struct adapter *adapter, unsigned int *level); int t4_set_devlog_level(struct adapter *adapter, unsigned int level); void t4_sge_decode_idma_state(struct adapter *adapter, int state); + +void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); +void t4_tp_pio_write(struct adapter *adap, const u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); +void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); +void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); static inline int t4vf_query_params(struct adapter *adapter, unsigned int nparams, const u32 *params, u32 *vals) { return t4_query_params(adapter, 0, 0, 0, nparams, params, vals); } static inline int t4vf_set_params(struct adapter *adapter, unsigned int nparams, const u32 *params, const u32 *vals) { return t4_set_params(adapter, 0, 0, 0, nparams, params, vals); } static inline int t4vf_wr_mbox(struct adapter *adap, const void *cmd, int size, void *rpl) { return t4_wr_mbox(adap, adap->mbox, cmd, size, rpl); } int t4vf_wait_dev_ready(struct adapter *adapter); int t4vf_fw_reset(struct adapter *adapter); int t4vf_get_sge_params(struct adapter *adapter); int t4vf_get_rss_glb_config(struct adapter *adapter); int t4vf_get_vfres(struct adapter *adapter); int t4vf_prep_adapter(struct adapter *adapter); #endif /* __CHELSIO_COMMON_H */ Index: projects/runtime-coverage/sys/dev/cxgbe/common/t4_hw.c =================================================================== --- projects/runtime-coverage/sys/dev/cxgbe/common/t4_hw.c (revision 321584) +++ projects/runtime-coverage/sys/dev/cxgbe/common/t4_hw.c (revision 321585) @@ -1,9589 +1,9746 @@ /*- * Copyright (c) 2012, 2016 Chelsio Communications, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include #include #include "common.h" #include "t4_regs.h" #include "t4_regs_values.h" #include "firmware/t4fw_interface.h" #undef msleep #define msleep(x) do { \ if (cold) \ DELAY((x) * 1000); \ else \ pause("t4hw", (x) * hz / 1000); \ } while (0) /** * t4_wait_op_done_val - wait until an operation is completed * @adapter: the adapter performing the operation * @reg: the register to check for completion * @mask: a single-bit field within @reg that indicates completion * @polarity: the value of the field when the operation is completed * @attempts: number of check iterations * @delay: delay in usecs between iterations * @valp: where to store the value of the register at completion time * * Wait until an operation is completed by checking a bit in a register * up to @attempts times. If @valp is not NULL the value of the register * at the time it indicated completion is stored there. Returns 0 if the * operation completes and -EAGAIN otherwise. */ static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask, int polarity, int attempts, int delay, u32 *valp) { while (1) { u32 val = t4_read_reg(adapter, reg); if (!!(val & mask) == polarity) { if (valp) *valp = val; return 0; } if (--attempts == 0) return -EAGAIN; if (delay) udelay(delay); } } static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask, int polarity, int attempts, int delay) { return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts, delay, NULL); } /** * t4_set_reg_field - set a register field to a value * @adapter: the adapter to program * @addr: the register address * @mask: specifies the portion of the register to modify * @val: the new value for the register field * * Sets a register field specified by the supplied mask to the * given value. */ void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask, u32 val) { u32 v = t4_read_reg(adapter, addr) & ~mask; t4_write_reg(adapter, addr, v | val); (void) t4_read_reg(adapter, addr); /* flush */ } /** * t4_read_indirect - read indirectly addressed registers * @adap: the adapter * @addr_reg: register holding the indirect address * @data_reg: register holding the value of the indirect register * @vals: where the read register values are stored * @nregs: how many indirect registers to read * @start_idx: index of first indirect register to read * * Reads registers that are accessed indirectly through an address/data * register pair. */ void t4_read_indirect(struct adapter *adap, unsigned int addr_reg, unsigned int data_reg, u32 *vals, unsigned int nregs, unsigned int start_idx) { while (nregs--) { t4_write_reg(adap, addr_reg, start_idx); *vals++ = t4_read_reg(adap, data_reg); start_idx++; } } /** * t4_write_indirect - write indirectly addressed registers * @adap: the adapter * @addr_reg: register holding the indirect addresses * @data_reg: register holding the value for the indirect registers * @vals: values to write * @nregs: how many indirect registers to write * @start_idx: address of first indirect register to write * * Writes a sequential block of registers that are accessed indirectly * through an address/data register pair. */ void t4_write_indirect(struct adapter *adap, unsigned int addr_reg, unsigned int data_reg, const u32 *vals, unsigned int nregs, unsigned int start_idx) { while (nregs--) { t4_write_reg(adap, addr_reg, start_idx++); t4_write_reg(adap, data_reg, *vals++); } } /* * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor * mechanism. This guarantees that we get the real value even if we're * operating within a Virtual Machine and the Hypervisor is trapping our * Configuration Space accesses. * * N.B. This routine should only be used as a last resort: the firmware uses * the backdoor registers on a regular basis and we can end up * conflicting with it's uses! */ u32 t4_hw_pci_read_cfg4(adapter_t *adap, int reg) { u32 req = V_FUNCTION(adap->pf) | V_REGISTER(reg); u32 val; if (chip_id(adap) <= CHELSIO_T5) req |= F_ENABLE; else req |= F_T6_ENABLE; if (is_t4(adap)) req |= F_LOCALCFG; t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, req); val = t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA); /* * Reset F_ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a * Configuration Space read. (None of the other fields matter when * F_ENABLE is 0 so a simple register write is easier than a * read-modify-write via t4_set_reg_field().) */ t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, 0); return val; } /* * t4_report_fw_error - report firmware error * @adap: the adapter * * The adapter firmware can indicate error conditions to the host. * If the firmware has indicated an error, print out the reason for * the firmware error. */ static void t4_report_fw_error(struct adapter *adap) { static const char *const reason[] = { "Crash", /* PCIE_FW_EVAL_CRASH */ "During Device Preparation", /* PCIE_FW_EVAL_PREP */ "During Device Configuration", /* PCIE_FW_EVAL_CONF */ "During Device Initialization", /* PCIE_FW_EVAL_INIT */ "Unexpected Event", /* PCIE_FW_EVAL_UNEXPECTEDEVENT */ "Insufficient Airflow", /* PCIE_FW_EVAL_OVERHEAT */ "Device Shutdown", /* PCIE_FW_EVAL_DEVICESHUTDOWN */ "Reserved", /* reserved */ }; u32 pcie_fw; pcie_fw = t4_read_reg(adap, A_PCIE_FW); if (pcie_fw & F_PCIE_FW_ERR) CH_ERR(adap, "Firmware reports adapter error: %s\n", reason[G_PCIE_FW_EVAL(pcie_fw)]); } /* * Get the reply to a mailbox command and store it in @rpl in big-endian order. */ static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit, u32 mbox_addr) { for ( ; nflit; nflit--, mbox_addr += 8) *rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr)); } /* * Handle a FW assertion reported in a mailbox. */ static void fw_asrt(struct adapter *adap, struct fw_debug_cmd *asrt) { CH_ALERT(adap, "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n", asrt->u.assert.filename_0_7, be32_to_cpu(asrt->u.assert.line), be32_to_cpu(asrt->u.assert.x), be32_to_cpu(asrt->u.assert.y)); } #define X_CIM_PF_NOACCESS 0xeeeeeeee /** * t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox * @adap: the adapter * @mbox: index of the mailbox to use * @cmd: the command to write * @size: command length in bytes * @rpl: where to optionally store the reply * @sleep_ok: if true we may sleep while awaiting command completion * @timeout: time to wait for command to finish before timing out * (negative implies @sleep_ok=false) * * Sends the given command to FW through the selected mailbox and waits * for the FW to execute the command. If @rpl is not %NULL it is used to * store the FW's reply to the command. The command and its optional * reply are of the same length. Some FW commands like RESET and * INITIALIZE can take a considerable amount of time to execute. * @sleep_ok determines whether we may sleep while awaiting the response. * If sleeping is allowed we use progressive backoff otherwise we spin. * Note that passing in a negative @timeout is an alternate mechanism * for specifying @sleep_ok=false. This is useful when a higher level * interface allows for specification of @timeout but not @sleep_ok ... * * The return value is 0 on success or a negative errno on failure. A * failure can happen either because we are not able to execute the * command or FW executes it but signals an error. In the latter case * the return value is the error code indicated by FW (negated). */ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl, bool sleep_ok, int timeout) { /* * We delay in small increments at first in an effort to maintain * responsiveness for simple, fast executing commands but then back * off to larger delays to a maximum retry delay. */ static const int delay[] = { 1, 1, 3, 5, 10, 10, 20, 50, 100 }; u32 v; u64 res; int i, ms, delay_idx, ret; const __be64 *p = cmd; u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA); u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL); u32 ctl; __be64 cmd_rpl[MBOX_LEN/8]; u32 pcie_fw; if ((size & 15) || size > MBOX_LEN) return -EINVAL; if (adap->flags & IS_VF) { if (is_t6(adap)) data_reg = FW_T6VF_MBDATA_BASE_ADDR; else data_reg = FW_T4VF_MBDATA_BASE_ADDR; ctl_reg = VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL); } /* * If we have a negative timeout, that implies that we can't sleep. */ if (timeout < 0) { sleep_ok = false; timeout = -timeout; } /* * Attempt to gain access to the mailbox. */ for (i = 0; i < 4; i++) { ctl = t4_read_reg(adap, ctl_reg); v = G_MBOWNER(ctl); if (v != X_MBOWNER_NONE) break; } /* * If we were unable to gain access, dequeue ourselves from the * mailbox atomic access list and report the error to our caller. */ if (v != X_MBOWNER_PL) { t4_report_fw_error(adap); ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT; return ret; } /* * If we gain ownership of the mailbox and there's a "valid" message * in it, this is likely an asynchronous error message from the * firmware. So we'll report that and then proceed on with attempting * to issue our own command ... which may well fail if the error * presaged the firmware crashing ... */ if (ctl & F_MBMSGVALID) { CH_ERR(adap, "found VALID command in mbox %u: " "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox, (unsigned long long)t4_read_reg64(adap, data_reg), (unsigned long long)t4_read_reg64(adap, data_reg + 8), (unsigned long long)t4_read_reg64(adap, data_reg + 16), (unsigned long long)t4_read_reg64(adap, data_reg + 24), (unsigned long long)t4_read_reg64(adap, data_reg + 32), (unsigned long long)t4_read_reg64(adap, data_reg + 40), (unsigned long long)t4_read_reg64(adap, data_reg + 48), (unsigned long long)t4_read_reg64(adap, data_reg + 56)); } /* * Copy in the new mailbox command and send it on its way ... */ for (i = 0; i < size; i += 8, p++) t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p)); if (adap->flags & IS_VF) { /* * For the VFs, the Mailbox Data "registers" are * actually backed by T4's "MA" interface rather than * PL Registers (as is the case for the PFs). Because * these are in different coherency domains, the write * to the VF's PL-register-backed Mailbox Control can * race in front of the writes to the MA-backed VF * Mailbox Data "registers". So we need to do a * read-back on at least one byte of the VF Mailbox * Data registers before doing the write to the VF * Mailbox Control register. */ t4_read_reg(adap, data_reg); } CH_DUMP_MBOX(adap, mbox, data_reg); t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW)); t4_read_reg(adap, ctl_reg); /* flush write */ delay_idx = 0; ms = delay[0]; /* * Loop waiting for the reply; bail out if we time out or the firmware * reports an error. */ pcie_fw = 0; for (i = 0; i < timeout; i += ms) { if (!(adap->flags & IS_VF)) { pcie_fw = t4_read_reg(adap, A_PCIE_FW); if (pcie_fw & F_PCIE_FW_ERR) break; } if (sleep_ok) { ms = delay[delay_idx]; /* last element may repeat */ if (delay_idx < ARRAY_SIZE(delay) - 1) delay_idx++; msleep(ms); } else { mdelay(ms); } v = t4_read_reg(adap, ctl_reg); if (v == X_CIM_PF_NOACCESS) continue; if (G_MBOWNER(v) == X_MBOWNER_PL) { if (!(v & F_MBMSGVALID)) { t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE)); continue; } /* * Retrieve the command reply and release the mailbox. */ get_mbox_rpl(adap, cmd_rpl, MBOX_LEN/8, data_reg); t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE)); CH_DUMP_MBOX(adap, mbox, data_reg); res = be64_to_cpu(cmd_rpl[0]); if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) { fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl); res = V_FW_CMD_RETVAL(EIO); } else if (rpl) memcpy(rpl, cmd_rpl, size); return -G_FW_CMD_RETVAL((int)res); } } /* * We timed out waiting for a reply to our mailbox command. Report * the error and also check to see if the firmware reported any * errors ... */ ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT; CH_ERR(adap, "command %#x in mailbox %d timed out\n", *(const u8 *)cmd, mbox); /* If DUMP_MBOX is set the mbox has already been dumped */ if ((adap->debug_flags & DF_DUMP_MBOX) == 0) { p = cmd; CH_ERR(adap, "mbox: %016llx %016llx %016llx %016llx " "%016llx %016llx %016llx %016llx\n", (unsigned long long)be64_to_cpu(p[0]), (unsigned long long)be64_to_cpu(p[1]), (unsigned long long)be64_to_cpu(p[2]), (unsigned long long)be64_to_cpu(p[3]), (unsigned long long)be64_to_cpu(p[4]), (unsigned long long)be64_to_cpu(p[5]), (unsigned long long)be64_to_cpu(p[6]), (unsigned long long)be64_to_cpu(p[7])); } t4_report_fw_error(adap); t4_fatal_err(adap); return ret; } int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl, bool sleep_ok) { return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, sleep_ok, FW_CMD_MAX_TIMEOUT); } static int t4_edc_err_read(struct adapter *adap, int idx) { u32 edc_ecc_err_addr_reg; u32 edc_bist_status_rdata_reg; if (is_t4(adap)) { CH_WARN(adap, "%s: T4 NOT supported.\n", __func__); return 0; } - if (idx != 0 && idx != 1) { + if (idx != MEM_EDC0 && idx != MEM_EDC1) { CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx); return 0; } edc_ecc_err_addr_reg = EDC_T5_REG(A_EDC_H_ECC_ERR_ADDR, idx); edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx); CH_WARN(adap, "edc%d err addr 0x%x: 0x%x.\n", idx, edc_ecc_err_addr_reg, t4_read_reg(adap, edc_ecc_err_addr_reg)); CH_WARN(adap, "bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n", edc_bist_status_rdata_reg, (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 16), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 24), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 32), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 40), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 48), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 56), (unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 64)); return 0; } /** * t4_mc_read - read from MC through backdoor accesses * @adap: the adapter * @idx: which MC to access * @addr: address of first byte requested * @data: 64 bytes of data containing the requested address * @ecc: where to store the corresponding 64-bit ECC word * * Read 64 bytes of data from MC starting at a 64-byte-aligned address * that covers the requested address @addr. If @parity is not %NULL it * is assigned the 64-bit ECC word for the read data. */ int t4_mc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc) { int i; u32 mc_bist_cmd_reg, mc_bist_cmd_addr_reg, mc_bist_cmd_len_reg; u32 mc_bist_status_rdata_reg, mc_bist_data_pattern_reg; if (is_t4(adap)) { mc_bist_cmd_reg = A_MC_BIST_CMD; mc_bist_cmd_addr_reg = A_MC_BIST_CMD_ADDR; mc_bist_cmd_len_reg = A_MC_BIST_CMD_LEN; mc_bist_status_rdata_reg = A_MC_BIST_STATUS_RDATA; mc_bist_data_pattern_reg = A_MC_BIST_DATA_PATTERN; } else { mc_bist_cmd_reg = MC_REG(A_MC_P_BIST_CMD, idx); mc_bist_cmd_addr_reg = MC_REG(A_MC_P_BIST_CMD_ADDR, idx); mc_bist_cmd_len_reg = MC_REG(A_MC_P_BIST_CMD_LEN, idx); mc_bist_status_rdata_reg = MC_REG(A_MC_P_BIST_STATUS_RDATA, idx); mc_bist_data_pattern_reg = MC_REG(A_MC_P_BIST_DATA_PATTERN, idx); } if (t4_read_reg(adap, mc_bist_cmd_reg) & F_START_BIST) return -EBUSY; t4_write_reg(adap, mc_bist_cmd_addr_reg, addr & ~0x3fU); t4_write_reg(adap, mc_bist_cmd_len_reg, 64); t4_write_reg(adap, mc_bist_data_pattern_reg, 0xc); t4_write_reg(adap, mc_bist_cmd_reg, V_BIST_OPCODE(1) | F_START_BIST | V_BIST_CMD_GAP(1)); i = t4_wait_op_done(adap, mc_bist_cmd_reg, F_START_BIST, 0, 10, 1); if (i) return i; #define MC_DATA(i) MC_BIST_STATUS_REG(mc_bist_status_rdata_reg, i) for (i = 15; i >= 0; i--) *data++ = ntohl(t4_read_reg(adap, MC_DATA(i))); if (ecc) *ecc = t4_read_reg64(adap, MC_DATA(16)); #undef MC_DATA return 0; } /** * t4_edc_read - read from EDC through backdoor accesses * @adap: the adapter * @idx: which EDC to access * @addr: address of first byte requested * @data: 64 bytes of data containing the requested address * @ecc: where to store the corresponding 64-bit ECC word * * Read 64 bytes of data from EDC starting at a 64-byte-aligned address * that covers the requested address @addr. If @parity is not %NULL it * is assigned the 64-bit ECC word for the read data. */ int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc) { int i; u32 edc_bist_cmd_reg, edc_bist_cmd_addr_reg, edc_bist_cmd_len_reg; u32 edc_bist_cmd_data_pattern, edc_bist_status_rdata_reg; if (is_t4(adap)) { edc_bist_cmd_reg = EDC_REG(A_EDC_BIST_CMD, idx); edc_bist_cmd_addr_reg = EDC_REG(A_EDC_BIST_CMD_ADDR, idx); edc_bist_cmd_len_reg = EDC_REG(A_EDC_BIST_CMD_LEN, idx); edc_bist_cmd_data_pattern = EDC_REG(A_EDC_BIST_DATA_PATTERN, idx); edc_bist_status_rdata_reg = EDC_REG(A_EDC_BIST_STATUS_RDATA, idx); } else { /* * These macro are missing in t4_regs.h file. * Added temporarily for testing. */ #define EDC_STRIDE_T5 (EDC_T51_BASE_ADDR - EDC_T50_BASE_ADDR) #define EDC_REG_T5(reg, idx) (reg + EDC_STRIDE_T5 * idx) edc_bist_cmd_reg = EDC_REG_T5(A_EDC_H_BIST_CMD, idx); edc_bist_cmd_addr_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_ADDR, idx); edc_bist_cmd_len_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_LEN, idx); edc_bist_cmd_data_pattern = EDC_REG_T5(A_EDC_H_BIST_DATA_PATTERN, idx); edc_bist_status_rdata_reg = EDC_REG_T5(A_EDC_H_BIST_STATUS_RDATA, idx); #undef EDC_REG_T5 #undef EDC_STRIDE_T5 } if (t4_read_reg(adap, edc_bist_cmd_reg) & F_START_BIST) return -EBUSY; t4_write_reg(adap, edc_bist_cmd_addr_reg, addr & ~0x3fU); t4_write_reg(adap, edc_bist_cmd_len_reg, 64); t4_write_reg(adap, edc_bist_cmd_data_pattern, 0xc); t4_write_reg(adap, edc_bist_cmd_reg, V_BIST_OPCODE(1) | V_BIST_CMD_GAP(1) | F_START_BIST); i = t4_wait_op_done(adap, edc_bist_cmd_reg, F_START_BIST, 0, 10, 1); if (i) return i; #define EDC_DATA(i) EDC_BIST_STATUS_REG(edc_bist_status_rdata_reg, i) for (i = 15; i >= 0; i--) *data++ = ntohl(t4_read_reg(adap, EDC_DATA(i))); if (ecc) *ecc = t4_read_reg64(adap, EDC_DATA(16)); #undef EDC_DATA return 0; } /** * t4_mem_read - read EDC 0, EDC 1 or MC into buffer * @adap: the adapter * @mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC * @addr: address within indicated memory type * @len: amount of memory to read * @buf: host memory buffer * * Reads an [almost] arbitrary memory region in the firmware: the * firmware memory address, length and host buffer must be aligned on * 32-bit boudaries. The memory is returned as a raw byte sequence from * the firmware's memory. If this memory contains data structures which * contain multi-byte integers, it's the callers responsibility to * perform appropriate byte order conversions. */ int t4_mem_read(struct adapter *adap, int mtype, u32 addr, u32 len, __be32 *buf) { u32 pos, start, end, offset; int ret; /* * Argument sanity checks ... */ if ((addr & 0x3) || (len & 0x3)) return -EINVAL; /* * The underlaying EDC/MC read routines read 64 bytes at a time so we * need to round down the start and round up the end. We'll start * copying out of the first line at (addr - start) a word at a time. */ start = rounddown2(addr, 64); end = roundup2(addr + len, 64); offset = (addr - start)/sizeof(__be32); for (pos = start; pos < end; pos += 64, offset = 0) { __be32 data[16]; /* * Read the chip's memory block and bail if there's an error. */ if ((mtype == MEM_MC) || (mtype == MEM_MC1)) ret = t4_mc_read(adap, mtype - MEM_MC, pos, data, NULL); else ret = t4_edc_read(adap, mtype, pos, data, NULL); if (ret) return ret; /* * Copy the data into the caller's memory buffer. */ while (offset < 16 && len > 0) { *buf++ = data[offset++]; len -= sizeof(__be32); } } return 0; } /* * Return the specified PCI-E Configuration Space register from our Physical * Function. We try first via a Firmware LDST Command (if fw_attach != 0) * since we prefer to let the firmware own all of these registers, but if that * fails we go for it directly ourselves. */ u32 t4_read_pcie_cfg4(struct adapter *adap, int reg, int drv_fw_attach) { /* * If fw_attach != 0, construct and send the Firmware LDST Command to * retrieve the specified PCI-E Configuration Space register. */ if (drv_fw_attach != 0) { struct fw_ldst_cmd ldst_cmd; int ret; memset(&ldst_cmd, 0, sizeof(ldst_cmd)); ldst_cmd.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FUNC_PCIE)); ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd)); ldst_cmd.u.pcie.select_naccess = V_FW_LDST_CMD_NACCESS(1); ldst_cmd.u.pcie.ctrl_to_fn = (F_FW_LDST_CMD_LC | V_FW_LDST_CMD_FN(adap->pf)); ldst_cmd.u.pcie.r = reg; /* * If the LDST Command succeeds, return the result, otherwise * fall through to reading it directly ourselves ... */ ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd), &ldst_cmd); if (ret == 0) return be32_to_cpu(ldst_cmd.u.pcie.data[0]); CH_WARN(adap, "Firmware failed to return " "Configuration Space register %d, err = %d\n", reg, -ret); } /* * Read the desired Configuration Space register via the PCI-E * Backdoor mechanism. */ return t4_hw_pci_read_cfg4(adap, reg); } /** * t4_get_regs_len - return the size of the chips register set * @adapter: the adapter * * Returns the size of the chip's BAR0 register space. */ unsigned int t4_get_regs_len(struct adapter *adapter) { unsigned int chip_version = chip_id(adapter); switch (chip_version) { case CHELSIO_T4: if (adapter->flags & IS_VF) return FW_T4VF_REGMAP_SIZE; return T4_REGMAP_SIZE; case CHELSIO_T5: case CHELSIO_T6: if (adapter->flags & IS_VF) return FW_T4VF_REGMAP_SIZE; return T5_REGMAP_SIZE; } CH_ERR(adapter, "Unsupported chip version %d\n", chip_version); return 0; } /** * t4_get_regs - read chip registers into provided buffer * @adap: the adapter * @buf: register buffer * @buf_size: size (in bytes) of register buffer * * If the provided register buffer isn't large enough for the chip's * full register range, the register dump will be truncated to the * register buffer's size. */ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size) { static const unsigned int t4_reg_ranges[] = { 0x1008, 0x1108, 0x1180, 0x1184, 0x1190, 0x1194, 0x11a0, 0x11a4, 0x11b0, 0x11b4, 0x11fc, 0x123c, 0x1300, 0x173c, 0x1800, 0x18fc, 0x3000, 0x30d8, 0x30e0, 0x30e4, 0x30ec, 0x5910, 0x5920, 0x5924, 0x5960, 0x5960, 0x5968, 0x5968, 0x5970, 0x5970, 0x5978, 0x5978, 0x5980, 0x5980, 0x5988, 0x5988, 0x5990, 0x5990, 0x5998, 0x5998, 0x59a0, 0x59d4, 0x5a00, 0x5ae0, 0x5ae8, 0x5ae8, 0x5af0, 0x5af0, 0x5af8, 0x5af8, 0x6000, 0x6098, 0x6100, 0x6150, 0x6200, 0x6208, 0x6240, 0x6248, 0x6280, 0x62b0, 0x62c0, 0x6338, 0x6370, 0x638c, 0x6400, 0x643c, 0x6500, 0x6524, 0x6a00, 0x6a04, 0x6a14, 0x6a38, 0x6a60, 0x6a70, 0x6a78, 0x6a78, 0x6b00, 0x6b0c, 0x6b1c, 0x6b84, 0x6bf0, 0x6bf8, 0x6c00, 0x6c0c, 0x6c1c, 0x6c84, 0x6cf0, 0x6cf8, 0x6d00, 0x6d0c, 0x6d1c, 0x6d84, 0x6df0, 0x6df8, 0x6e00, 0x6e0c, 0x6e1c, 0x6e84, 0x6ef0, 0x6ef8, 0x6f00, 0x6f0c, 0x6f1c, 0x6f84, 0x6ff0, 0x6ff8, 0x7000, 0x700c, 0x701c, 0x7084, 0x70f0, 0x70f8, 0x7100, 0x710c, 0x711c, 0x7184, 0x71f0, 0x71f8, 0x7200, 0x720c, 0x721c, 0x7284, 0x72f0, 0x72f8, 0x7300, 0x730c, 0x731c, 0x7384, 0x73f0, 0x73f8, 0x7400, 0x7450, 0x7500, 0x7530, 0x7600, 0x760c, 0x7614, 0x761c, 0x7680, 0x76cc, 0x7700, 0x7798, 0x77c0, 0x77fc, 0x7900, 0x79fc, 0x7b00, 0x7b58, 0x7b60, 0x7b84, 0x7b8c, 0x7c38, 0x7d00, 0x7d38, 0x7d40, 0x7d80, 0x7d8c, 0x7ddc, 0x7de4, 0x7e04, 0x7e10, 0x7e1c, 0x7e24, 0x7e38, 0x7e40, 0x7e44, 0x7e4c, 0x7e78, 0x7e80, 0x7ea4, 0x7eac, 0x7edc, 0x7ee8, 0x7efc, 0x8dc0, 0x8e04, 0x8e10, 0x8e1c, 0x8e30, 0x8e78, 0x8ea0, 0x8eb8, 0x8ec0, 0x8f6c, 0x8fc0, 0x9008, 0x9010, 0x9058, 0x9060, 0x9060, 0x9068, 0x9074, 0x90fc, 0x90fc, 0x9400, 0x9408, 0x9410, 0x9458, 0x9600, 0x9600, 0x9608, 0x9638, 0x9640, 0x96bc, 0x9800, 0x9808, 0x9820, 0x983c, 0x9850, 0x9864, 0x9c00, 0x9c6c, 0x9c80, 0x9cec, 0x9d00, 0x9d6c, 0x9d80, 0x9dec, 0x9e00, 0x9e6c, 0x9e80, 0x9eec, 0x9f00, 0x9f6c, 0x9f80, 0x9fec, 0xd004, 0xd004, 0xd010, 0xd03c, 0xdfc0, 0xdfe0, 0xe000, 0xea7c, - 0xf000, 0x11190, + 0xf000, 0x11110, + 0x11118, 0x11190, 0x19040, 0x1906c, 0x19078, 0x19080, 0x1908c, 0x190e4, 0x190f0, 0x190f8, 0x19100, 0x19110, 0x19120, 0x19124, 0x19150, 0x19194, 0x1919c, 0x191b0, 0x191d0, 0x191e8, 0x19238, 0x1924c, 0x193f8, 0x1943c, 0x1944c, 0x19474, 0x19490, 0x194e0, 0x194f0, 0x194f8, 0x19800, 0x19c08, 0x19c10, 0x19c90, 0x19ca0, 0x19ce4, 0x19cf0, 0x19d40, 0x19d50, 0x19d94, 0x19da0, 0x19de8, 0x19df0, 0x19e40, 0x19e50, 0x19e90, 0x19ea0, 0x19f4c, 0x1a000, 0x1a004, 0x1a010, 0x1a06c, 0x1a0b0, 0x1a0e4, 0x1a0ec, 0x1a0f4, 0x1a100, 0x1a108, 0x1a114, 0x1a120, 0x1a128, 0x1a130, 0x1a138, 0x1a138, 0x1a190, 0x1a1c4, 0x1a1fc, 0x1a1fc, 0x1e040, 0x1e04c, 0x1e284, 0x1e28c, 0x1e2c0, 0x1e2c0, 0x1e2e0, 0x1e2e0, 0x1e300, 0x1e384, 0x1e3c0, 0x1e3c8, 0x1e440, 0x1e44c, 0x1e684, 0x1e68c, 0x1e6c0, 0x1e6c0, 0x1e6e0, 0x1e6e0, 0x1e700, 0x1e784, 0x1e7c0, 0x1e7c8, 0x1e840, 0x1e84c, 0x1ea84, 0x1ea8c, 0x1eac0, 0x1eac0, 0x1eae0, 0x1eae0, 0x1eb00, 0x1eb84, 0x1ebc0, 0x1ebc8, 0x1ec40, 0x1ec4c, 0x1ee84, 0x1ee8c, 0x1eec0, 0x1eec0, 0x1eee0, 0x1eee0, 0x1ef00, 0x1ef84, 0x1efc0, 0x1efc8, 0x1f040, 0x1f04c, 0x1f284, 0x1f28c, 0x1f2c0, 0x1f2c0, 0x1f2e0, 0x1f2e0, 0x1f300, 0x1f384, 0x1f3c0, 0x1f3c8, 0x1f440, 0x1f44c, 0x1f684, 0x1f68c, 0x1f6c0, 0x1f6c0, 0x1f6e0, 0x1f6e0, 0x1f700, 0x1f784, 0x1f7c0, 0x1f7c8, 0x1f840, 0x1f84c, 0x1fa84, 0x1fa8c, 0x1fac0, 0x1fac0, 0x1fae0, 0x1fae0, 0x1fb00, 0x1fb84, 0x1fbc0, 0x1fbc8, 0x1fc40, 0x1fc4c, 0x1fe84, 0x1fe8c, 0x1fec0, 0x1fec0, 0x1fee0, 0x1fee0, 0x1ff00, 0x1ff84, 0x1ffc0, 0x1ffc8, 0x20000, 0x2002c, 0x20100, 0x2013c, 0x20190, 0x201a0, 0x201a8, 0x201b8, 0x201c4, 0x201c8, 0x20200, 0x20318, 0x20400, 0x204b4, 0x204c0, 0x20528, 0x20540, 0x20614, 0x21000, 0x21040, 0x2104c, 0x21060, 0x210c0, 0x210ec, 0x21200, 0x21268, 0x21270, 0x21284, 0x212fc, 0x21388, 0x21400, 0x21404, 0x21500, 0x21500, 0x21510, 0x21518, 0x2152c, 0x21530, 0x2153c, 0x2153c, 0x21550, 0x21554, 0x21600, 0x21600, 0x21608, 0x2161c, 0x21624, 0x21628, 0x21630, 0x21634, 0x2163c, 0x2163c, 0x21700, 0x2171c, 0x21780, 0x2178c, 0x21800, 0x21818, 0x21820, 0x21828, 0x21830, 0x21848, 0x21850, 0x21854, 0x21860, 0x21868, 0x21870, 0x21870, 0x21878, 0x21898, 0x218a0, 0x218a8, 0x218b0, 0x218c8, 0x218d0, 0x218d4, 0x218e0, 0x218e8, 0x218f0, 0x218f0, 0x218f8, 0x21a18, 0x21a20, 0x21a28, 0x21a30, 0x21a48, 0x21a50, 0x21a54, 0x21a60, 0x21a68, 0x21a70, 0x21a70, 0x21a78, 0x21a98, 0x21aa0, 0x21aa8, 0x21ab0, 0x21ac8, 0x21ad0, 0x21ad4, 0x21ae0, 0x21ae8, 0x21af0, 0x21af0, 0x21af8, 0x21c18, 0x21c20, 0x21c20, 0x21c28, 0x21c30, 0x21c38, 0x21c38, 0x21c80, 0x21c98, 0x21ca0, 0x21ca8, 0x21cb0, 0x21cc8, 0x21cd0, 0x21cd4, 0x21ce0, 0x21ce8, 0x21cf0, 0x21cf0, 0x21cf8, 0x21d7c, 0x21e00, 0x21e04, 0x22000, 0x2202c, 0x22100, 0x2213c, 0x22190, 0x221a0, 0x221a8, 0x221b8, 0x221c4, 0x221c8, 0x22200, 0x22318, 0x22400, 0x224b4, 0x224c0, 0x22528, 0x22540, 0x22614, 0x23000, 0x23040, 0x2304c, 0x23060, 0x230c0, 0x230ec, 0x23200, 0x23268, 0x23270, 0x23284, 0x232fc, 0x23388, 0x23400, 0x23404, 0x23500, 0x23500, 0x23510, 0x23518, 0x2352c, 0x23530, 0x2353c, 0x2353c, 0x23550, 0x23554, 0x23600, 0x23600, 0x23608, 0x2361c, 0x23624, 0x23628, 0x23630, 0x23634, 0x2363c, 0x2363c, 0x23700, 0x2371c, 0x23780, 0x2378c, 0x23800, 0x23818, 0x23820, 0x23828, 0x23830, 0x23848, 0x23850, 0x23854, 0x23860, 0x23868, 0x23870, 0x23870, 0x23878, 0x23898, 0x238a0, 0x238a8, 0x238b0, 0x238c8, 0x238d0, 0x238d4, 0x238e0, 0x238e8, 0x238f0, 0x238f0, 0x238f8, 0x23a18, 0x23a20, 0x23a28, 0x23a30, 0x23a48, 0x23a50, 0x23a54, 0x23a60, 0x23a68, 0x23a70, 0x23a70, 0x23a78, 0x23a98, 0x23aa0, 0x23aa8, 0x23ab0, 0x23ac8, 0x23ad0, 0x23ad4, 0x23ae0, 0x23ae8, 0x23af0, 0x23af0, 0x23af8, 0x23c18, 0x23c20, 0x23c20, 0x23c28, 0x23c30, 0x23c38, 0x23c38, 0x23c80, 0x23c98, 0x23ca0, 0x23ca8, 0x23cb0, 0x23cc8, 0x23cd0, 0x23cd4, 0x23ce0, 0x23ce8, 0x23cf0, 0x23cf0, 0x23cf8, 0x23d7c, 0x23e00, 0x23e04, 0x24000, 0x2402c, 0x24100, 0x2413c, 0x24190, 0x241a0, 0x241a8, 0x241b8, 0x241c4, 0x241c8, 0x24200, 0x24318, 0x24400, 0x244b4, 0x244c0, 0x24528, 0x24540, 0x24614, 0x25000, 0x25040, 0x2504c, 0x25060, 0x250c0, 0x250ec, 0x25200, 0x25268, 0x25270, 0x25284, 0x252fc, 0x25388, 0x25400, 0x25404, 0x25500, 0x25500, 0x25510, 0x25518, 0x2552c, 0x25530, 0x2553c, 0x2553c, 0x25550, 0x25554, 0x25600, 0x25600, 0x25608, 0x2561c, 0x25624, 0x25628, 0x25630, 0x25634, 0x2563c, 0x2563c, 0x25700, 0x2571c, 0x25780, 0x2578c, 0x25800, 0x25818, 0x25820, 0x25828, 0x25830, 0x25848, 0x25850, 0x25854, 0x25860, 0x25868, 0x25870, 0x25870, 0x25878, 0x25898, 0x258a0, 0x258a8, 0x258b0, 0x258c8, 0x258d0, 0x258d4, 0x258e0, 0x258e8, 0x258f0, 0x258f0, 0x258f8, 0x25a18, 0x25a20, 0x25a28, 0x25a30, 0x25a48, 0x25a50, 0x25a54, 0x25a60, 0x25a68, 0x25a70, 0x25a70, 0x25a78, 0x25a98, 0x25aa0, 0x25aa8, 0x25ab0, 0x25ac8, 0x25ad0, 0x25ad4, 0x25ae0, 0x25ae8, 0x25af0, 0x25af0, 0x25af8, 0x25c18, 0x25c20, 0x25c20, 0x25c28, 0x25c30, 0x25c38, 0x25c38, 0x25c80, 0x25c98, 0x25ca0, 0x25ca8, 0x25cb0, 0x25cc8, 0x25cd0, 0x25cd4, 0x25ce0, 0x25ce8, 0x25cf0, 0x25cf0, 0x25cf8, 0x25d7c, 0x25e00, 0x25e04, 0x26000, 0x2602c, 0x26100, 0x2613c, 0x26190, 0x261a0, 0x261a8, 0x261b8, 0x261c4, 0x261c8, 0x26200, 0x26318, 0x26400, 0x264b4, 0x264c0, 0x26528, 0x26540, 0x26614, 0x27000, 0x27040, 0x2704c, 0x27060, 0x270c0, 0x270ec, 0x27200, 0x27268, 0x27270, 0x27284, 0x272fc, 0x27388, 0x27400, 0x27404, 0x27500, 0x27500, 0x27510, 0x27518, 0x2752c, 0x27530, 0x2753c, 0x2753c, 0x27550, 0x27554, 0x27600, 0x27600, 0x27608, 0x2761c, 0x27624, 0x27628, 0x27630, 0x27634, 0x2763c, 0x2763c, 0x27700, 0x2771c, 0x27780, 0x2778c, 0x27800, 0x27818, 0x27820, 0x27828, 0x27830, 0x27848, 0x27850, 0x27854, 0x27860, 0x27868, 0x27870, 0x27870, 0x27878, 0x27898, 0x278a0, 0x278a8, 0x278b0, 0x278c8, 0x278d0, 0x278d4, 0x278e0, 0x278e8, 0x278f0, 0x278f0, 0x278f8, 0x27a18, 0x27a20, 0x27a28, 0x27a30, 0x27a48, 0x27a50, 0x27a54, 0x27a60, 0x27a68, 0x27a70, 0x27a70, 0x27a78, 0x27a98, 0x27aa0, 0x27aa8, 0x27ab0, 0x27ac8, 0x27ad0, 0x27ad4, 0x27ae0, 0x27ae8, 0x27af0, 0x27af0, 0x27af8, 0x27c18, 0x27c20, 0x27c20, 0x27c28, 0x27c30, 0x27c38, 0x27c38, 0x27c80, 0x27c98, 0x27ca0, 0x27ca8, 0x27cb0, 0x27cc8, 0x27cd0, 0x27cd4, 0x27ce0, 0x27ce8, 0x27cf0, 0x27cf0, 0x27cf8, 0x27d7c, 0x27e00, 0x27e04, }; static const unsigned int t4vf_reg_ranges[] = { VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), VF_MPS_REG(A_MPS_VF_CTL), VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_WHOAMI), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), FW_T4VF_MBDATA_BASE_ADDR, FW_T4VF_MBDATA_BASE_ADDR + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), }; static const unsigned int t5_reg_ranges[] = { 0x1008, 0x10c0, 0x10cc, 0x10f8, 0x1100, 0x1100, 0x110c, 0x1148, 0x1180, 0x1184, 0x1190, 0x1194, 0x11a0, 0x11a4, 0x11b0, 0x11b4, 0x11fc, 0x123c, 0x1280, 0x173c, 0x1800, 0x18fc, 0x3000, 0x3028, 0x3060, 0x30b0, 0x30b8, 0x30d8, 0x30e0, 0x30fc, 0x3140, 0x357c, 0x35a8, 0x35cc, 0x35ec, 0x35ec, 0x3600, 0x5624, 0x56cc, 0x56ec, 0x56f4, 0x5720, 0x5728, 0x575c, 0x580c, 0x5814, 0x5890, 0x589c, 0x58a4, 0x58ac, 0x58b8, 0x58bc, 0x5940, 0x59c8, 0x59d0, 0x59dc, 0x59fc, 0x5a18, 0x5a60, 0x5a70, 0x5a80, 0x5a9c, 0x5b94, 0x5bfc, 0x6000, 0x6020, 0x6028, 0x6040, 0x6058, 0x609c, 0x60a8, 0x614c, 0x7700, 0x7798, 0x77c0, 0x78fc, 0x7b00, 0x7b58, 0x7b60, 0x7b84, 0x7b8c, 0x7c54, 0x7d00, 0x7d38, 0x7d40, 0x7d80, 0x7d8c, 0x7ddc, 0x7de4, 0x7e04, 0x7e10, 0x7e1c, 0x7e24, 0x7e38, 0x7e40, 0x7e44, 0x7e4c, 0x7e78, 0x7e80, 0x7edc, 0x7ee8, 0x7efc, 0x8dc0, 0x8de0, 0x8df8, 0x8e04, 0x8e10, 0x8e84, 0x8ea0, 0x8f84, 0x8fc0, 0x9058, 0x9060, 0x9060, 0x9068, 0x90f8, 0x9400, 0x9408, 0x9410, 0x9470, 0x9600, 0x9600, 0x9608, 0x9638, 0x9640, 0x96f4, 0x9800, 0x9808, 0x9820, 0x983c, 0x9850, 0x9864, 0x9c00, 0x9c6c, 0x9c80, 0x9cec, 0x9d00, 0x9d6c, 0x9d80, 0x9dec, 0x9e00, 0x9e6c, 0x9e80, 0x9eec, 0x9f00, 0x9f6c, 0x9f80, 0xa020, 0xd004, 0xd004, 0xd010, 0xd03c, 0xdfc0, 0xdfe0, 0xe000, 0x1106c, 0x11074, 0x11088, 0x1109c, 0x1117c, 0x11190, 0x11204, 0x19040, 0x1906c, 0x19078, 0x19080, 0x1908c, 0x190e8, 0x190f0, 0x190f8, 0x19100, 0x19110, 0x19120, 0x19124, 0x19150, 0x19194, 0x1919c, 0x191b0, 0x191d0, 0x191e8, 0x19238, 0x19290, 0x193f8, 0x19428, 0x19430, 0x19444, 0x1944c, 0x1946c, 0x19474, 0x19474, 0x19490, 0x194cc, 0x194f0, 0x194f8, 0x19c00, 0x19c08, 0x19c10, 0x19c60, 0x19c94, 0x19ce4, 0x19cf0, 0x19d40, 0x19d50, 0x19d94, 0x19da0, 0x19de8, 0x19df0, 0x19e10, 0x19e50, 0x19e90, 0x19ea0, 0x19f24, 0x19f34, 0x19f34, 0x19f40, 0x19f50, 0x19f90, 0x19fb4, 0x19fc4, 0x19fe4, 0x1a000, 0x1a004, 0x1a010, 0x1a06c, 0x1a0b0, 0x1a0e4, 0x1a0ec, 0x1a0f8, 0x1a100, 0x1a108, 0x1a114, 0x1a120, 0x1a128, 0x1a130, 0x1a138, 0x1a138, 0x1a190, 0x1a1c4, 0x1a1fc, 0x1a1fc, 0x1e008, 0x1e00c, 0x1e040, 0x1e044, 0x1e04c, 0x1e04c, 0x1e284, 0x1e290, 0x1e2c0, 0x1e2c0, 0x1e2e0, 0x1e2e0, 0x1e300, 0x1e384, 0x1e3c0, 0x1e3c8, 0x1e408, 0x1e40c, 0x1e440, 0x1e444, 0x1e44c, 0x1e44c, 0x1e684, 0x1e690, 0x1e6c0, 0x1e6c0, 0x1e6e0, 0x1e6e0, 0x1e700, 0x1e784, 0x1e7c0, 0x1e7c8, 0x1e808, 0x1e80c, 0x1e840, 0x1e844, 0x1e84c, 0x1e84c, 0x1ea84, 0x1ea90, 0x1eac0, 0x1eac0, 0x1eae0, 0x1eae0, 0x1eb00, 0x1eb84, 0x1ebc0, 0x1ebc8, 0x1ec08, 0x1ec0c, 0x1ec40, 0x1ec44, 0x1ec4c, 0x1ec4c, 0x1ee84, 0x1ee90, 0x1eec0, 0x1eec0, 0x1eee0, 0x1eee0, 0x1ef00, 0x1ef84, 0x1efc0, 0x1efc8, 0x1f008, 0x1f00c, 0x1f040, 0x1f044, 0x1f04c, 0x1f04c, 0x1f284, 0x1f290, 0x1f2c0, 0x1f2c0, 0x1f2e0, 0x1f2e0, 0x1f300, 0x1f384, 0x1f3c0, 0x1f3c8, 0x1f408, 0x1f40c, 0x1f440, 0x1f444, 0x1f44c, 0x1f44c, 0x1f684, 0x1f690, 0x1f6c0, 0x1f6c0, 0x1f6e0, 0x1f6e0, 0x1f700, 0x1f784, 0x1f7c0, 0x1f7c8, 0x1f808, 0x1f80c, 0x1f840, 0x1f844, 0x1f84c, 0x1f84c, 0x1fa84, 0x1fa90, 0x1fac0, 0x1fac0, 0x1fae0, 0x1fae0, 0x1fb00, 0x1fb84, 0x1fbc0, 0x1fbc8, 0x1fc08, 0x1fc0c, 0x1fc40, 0x1fc44, 0x1fc4c, 0x1fc4c, 0x1fe84, 0x1fe90, 0x1fec0, 0x1fec0, 0x1fee0, 0x1fee0, 0x1ff00, 0x1ff84, 0x1ffc0, 0x1ffc8, 0x30000, 0x30030, - 0x30038, 0x30038, - 0x30040, 0x30040, 0x30100, 0x30144, 0x30190, 0x301a0, 0x301a8, 0x301b8, 0x301c4, 0x301c8, 0x301d0, 0x301d0, 0x30200, 0x30318, 0x30400, 0x304b4, 0x304c0, 0x3052c, 0x30540, 0x3061c, 0x30800, 0x30828, 0x30834, 0x30834, 0x308c0, 0x30908, 0x30910, 0x309ac, 0x30a00, 0x30a14, 0x30a1c, 0x30a2c, 0x30a44, 0x30a50, 0x30a74, 0x30a74, 0x30a7c, 0x30afc, 0x30b08, 0x30c24, 0x30d00, 0x30d00, 0x30d08, 0x30d14, 0x30d1c, 0x30d20, 0x30d3c, 0x30d3c, 0x30d48, 0x30d50, 0x31200, 0x3120c, 0x31220, 0x31220, 0x31240, 0x31240, 0x31600, 0x3160c, 0x31a00, 0x31a1c, 0x31e00, 0x31e20, 0x31e38, 0x31e3c, 0x31e80, 0x31e80, 0x31e88, 0x31ea8, 0x31eb0, 0x31eb4, 0x31ec8, 0x31ed4, 0x31fb8, 0x32004, 0x32200, 0x32200, 0x32208, 0x32240, 0x32248, 0x32280, 0x32288, 0x322c0, 0x322c8, 0x322fc, 0x32600, 0x32630, 0x32a00, 0x32abc, 0x32b00, 0x32b10, 0x32b20, 0x32b30, 0x32b40, 0x32b50, 0x32b60, 0x32b70, 0x33000, 0x33028, 0x33030, 0x33048, 0x33060, 0x33068, 0x33070, 0x3309c, 0x330f0, 0x33128, 0x33130, 0x33148, 0x33160, 0x33168, 0x33170, 0x3319c, 0x331f0, 0x33238, 0x33240, 0x33240, 0x33248, 0x33250, 0x3325c, 0x33264, 0x33270, 0x332b8, 0x332c0, 0x332e4, 0x332f8, 0x33338, 0x33340, 0x33340, 0x33348, 0x33350, 0x3335c, 0x33364, 0x33370, 0x333b8, 0x333c0, 0x333e4, 0x333f8, 0x33428, 0x33430, 0x33448, 0x33460, 0x33468, 0x33470, 0x3349c, 0x334f0, 0x33528, 0x33530, 0x33548, 0x33560, 0x33568, 0x33570, 0x3359c, 0x335f0, 0x33638, 0x33640, 0x33640, 0x33648, 0x33650, 0x3365c, 0x33664, 0x33670, 0x336b8, 0x336c0, 0x336e4, 0x336f8, 0x33738, 0x33740, 0x33740, 0x33748, 0x33750, 0x3375c, 0x33764, 0x33770, 0x337b8, 0x337c0, 0x337e4, 0x337f8, 0x337fc, 0x33814, 0x33814, 0x3382c, 0x3382c, 0x33880, 0x3388c, 0x338e8, 0x338ec, 0x33900, 0x33928, 0x33930, 0x33948, 0x33960, 0x33968, 0x33970, 0x3399c, 0x339f0, 0x33a38, 0x33a40, 0x33a40, 0x33a48, 0x33a50, 0x33a5c, 0x33a64, 0x33a70, 0x33ab8, 0x33ac0, 0x33ae4, 0x33af8, 0x33b10, 0x33b28, 0x33b28, 0x33b3c, 0x33b50, 0x33bf0, 0x33c10, 0x33c28, 0x33c28, 0x33c3c, 0x33c50, 0x33cf0, 0x33cfc, 0x34000, 0x34030, - 0x34038, 0x34038, - 0x34040, 0x34040, 0x34100, 0x34144, 0x34190, 0x341a0, 0x341a8, 0x341b8, 0x341c4, 0x341c8, 0x341d0, 0x341d0, 0x34200, 0x34318, 0x34400, 0x344b4, 0x344c0, 0x3452c, 0x34540, 0x3461c, 0x34800, 0x34828, 0x34834, 0x34834, 0x348c0, 0x34908, 0x34910, 0x349ac, 0x34a00, 0x34a14, 0x34a1c, 0x34a2c, 0x34a44, 0x34a50, 0x34a74, 0x34a74, 0x34a7c, 0x34afc, 0x34b08, 0x34c24, 0x34d00, 0x34d00, 0x34d08, 0x34d14, 0x34d1c, 0x34d20, 0x34d3c, 0x34d3c, 0x34d48, 0x34d50, 0x35200, 0x3520c, 0x35220, 0x35220, 0x35240, 0x35240, 0x35600, 0x3560c, 0x35a00, 0x35a1c, 0x35e00, 0x35e20, 0x35e38, 0x35e3c, 0x35e80, 0x35e80, 0x35e88, 0x35ea8, 0x35eb0, 0x35eb4, 0x35ec8, 0x35ed4, 0x35fb8, 0x36004, 0x36200, 0x36200, 0x36208, 0x36240, 0x36248, 0x36280, 0x36288, 0x362c0, 0x362c8, 0x362fc, 0x36600, 0x36630, 0x36a00, 0x36abc, 0x36b00, 0x36b10, 0x36b20, 0x36b30, 0x36b40, 0x36b50, 0x36b60, 0x36b70, 0x37000, 0x37028, 0x37030, 0x37048, 0x37060, 0x37068, 0x37070, 0x3709c, 0x370f0, 0x37128, 0x37130, 0x37148, 0x37160, 0x37168, 0x37170, 0x3719c, 0x371f0, 0x37238, 0x37240, 0x37240, 0x37248, 0x37250, 0x3725c, 0x37264, 0x37270, 0x372b8, 0x372c0, 0x372e4, 0x372f8, 0x37338, 0x37340, 0x37340, 0x37348, 0x37350, 0x3735c, 0x37364, 0x37370, 0x373b8, 0x373c0, 0x373e4, 0x373f8, 0x37428, 0x37430, 0x37448, 0x37460, 0x37468, 0x37470, 0x3749c, 0x374f0, 0x37528, 0x37530, 0x37548, 0x37560, 0x37568, 0x37570, 0x3759c, 0x375f0, 0x37638, 0x37640, 0x37640, 0x37648, 0x37650, 0x3765c, 0x37664, 0x37670, 0x376b8, 0x376c0, 0x376e4, 0x376f8, 0x37738, 0x37740, 0x37740, 0x37748, 0x37750, 0x3775c, 0x37764, 0x37770, 0x377b8, 0x377c0, 0x377e4, 0x377f8, 0x377fc, 0x37814, 0x37814, 0x3782c, 0x3782c, 0x37880, 0x3788c, 0x378e8, 0x378ec, 0x37900, 0x37928, 0x37930, 0x37948, 0x37960, 0x37968, 0x37970, 0x3799c, 0x379f0, 0x37a38, 0x37a40, 0x37a40, 0x37a48, 0x37a50, 0x37a5c, 0x37a64, 0x37a70, 0x37ab8, 0x37ac0, 0x37ae4, 0x37af8, 0x37b10, 0x37b28, 0x37b28, 0x37b3c, 0x37b50, 0x37bf0, 0x37c10, 0x37c28, 0x37c28, 0x37c3c, 0x37c50, 0x37cf0, 0x37cfc, 0x38000, 0x38030, - 0x38038, 0x38038, - 0x38040, 0x38040, 0x38100, 0x38144, 0x38190, 0x381a0, 0x381a8, 0x381b8, 0x381c4, 0x381c8, 0x381d0, 0x381d0, 0x38200, 0x38318, 0x38400, 0x384b4, 0x384c0, 0x3852c, 0x38540, 0x3861c, 0x38800, 0x38828, 0x38834, 0x38834, 0x388c0, 0x38908, 0x38910, 0x389ac, 0x38a00, 0x38a14, 0x38a1c, 0x38a2c, 0x38a44, 0x38a50, 0x38a74, 0x38a74, 0x38a7c, 0x38afc, 0x38b08, 0x38c24, 0x38d00, 0x38d00, 0x38d08, 0x38d14, 0x38d1c, 0x38d20, 0x38d3c, 0x38d3c, 0x38d48, 0x38d50, 0x39200, 0x3920c, 0x39220, 0x39220, 0x39240, 0x39240, 0x39600, 0x3960c, 0x39a00, 0x39a1c, 0x39e00, 0x39e20, 0x39e38, 0x39e3c, 0x39e80, 0x39e80, 0x39e88, 0x39ea8, 0x39eb0, 0x39eb4, 0x39ec8, 0x39ed4, 0x39fb8, 0x3a004, 0x3a200, 0x3a200, 0x3a208, 0x3a240, 0x3a248, 0x3a280, 0x3a288, 0x3a2c0, 0x3a2c8, 0x3a2fc, 0x3a600, 0x3a630, 0x3aa00, 0x3aabc, 0x3ab00, 0x3ab10, 0x3ab20, 0x3ab30, 0x3ab40, 0x3ab50, 0x3ab60, 0x3ab70, 0x3b000, 0x3b028, 0x3b030, 0x3b048, 0x3b060, 0x3b068, 0x3b070, 0x3b09c, 0x3b0f0, 0x3b128, 0x3b130, 0x3b148, 0x3b160, 0x3b168, 0x3b170, 0x3b19c, 0x3b1f0, 0x3b238, 0x3b240, 0x3b240, 0x3b248, 0x3b250, 0x3b25c, 0x3b264, 0x3b270, 0x3b2b8, 0x3b2c0, 0x3b2e4, 0x3b2f8, 0x3b338, 0x3b340, 0x3b340, 0x3b348, 0x3b350, 0x3b35c, 0x3b364, 0x3b370, 0x3b3b8, 0x3b3c0, 0x3b3e4, 0x3b3f8, 0x3b428, 0x3b430, 0x3b448, 0x3b460, 0x3b468, 0x3b470, 0x3b49c, 0x3b4f0, 0x3b528, 0x3b530, 0x3b548, 0x3b560, 0x3b568, 0x3b570, 0x3b59c, 0x3b5f0, 0x3b638, 0x3b640, 0x3b640, 0x3b648, 0x3b650, 0x3b65c, 0x3b664, 0x3b670, 0x3b6b8, 0x3b6c0, 0x3b6e4, 0x3b6f8, 0x3b738, 0x3b740, 0x3b740, 0x3b748, 0x3b750, 0x3b75c, 0x3b764, 0x3b770, 0x3b7b8, 0x3b7c0, 0x3b7e4, 0x3b7f8, 0x3b7fc, 0x3b814, 0x3b814, 0x3b82c, 0x3b82c, 0x3b880, 0x3b88c, 0x3b8e8, 0x3b8ec, 0x3b900, 0x3b928, 0x3b930, 0x3b948, 0x3b960, 0x3b968, 0x3b970, 0x3b99c, 0x3b9f0, 0x3ba38, 0x3ba40, 0x3ba40, 0x3ba48, 0x3ba50, 0x3ba5c, 0x3ba64, 0x3ba70, 0x3bab8, 0x3bac0, 0x3bae4, 0x3baf8, 0x3bb10, 0x3bb28, 0x3bb28, 0x3bb3c, 0x3bb50, 0x3bbf0, 0x3bc10, 0x3bc28, 0x3bc28, 0x3bc3c, 0x3bc50, 0x3bcf0, 0x3bcfc, 0x3c000, 0x3c030, - 0x3c038, 0x3c038, - 0x3c040, 0x3c040, 0x3c100, 0x3c144, 0x3c190, 0x3c1a0, 0x3c1a8, 0x3c1b8, 0x3c1c4, 0x3c1c8, 0x3c1d0, 0x3c1d0, 0x3c200, 0x3c318, 0x3c400, 0x3c4b4, 0x3c4c0, 0x3c52c, 0x3c540, 0x3c61c, 0x3c800, 0x3c828, 0x3c834, 0x3c834, 0x3c8c0, 0x3c908, 0x3c910, 0x3c9ac, 0x3ca00, 0x3ca14, 0x3ca1c, 0x3ca2c, 0x3ca44, 0x3ca50, 0x3ca74, 0x3ca74, 0x3ca7c, 0x3cafc, 0x3cb08, 0x3cc24, 0x3cd00, 0x3cd00, 0x3cd08, 0x3cd14, 0x3cd1c, 0x3cd20, 0x3cd3c, 0x3cd3c, 0x3cd48, 0x3cd50, 0x3d200, 0x3d20c, 0x3d220, 0x3d220, 0x3d240, 0x3d240, 0x3d600, 0x3d60c, 0x3da00, 0x3da1c, 0x3de00, 0x3de20, 0x3de38, 0x3de3c, 0x3de80, 0x3de80, 0x3de88, 0x3dea8, 0x3deb0, 0x3deb4, 0x3dec8, 0x3ded4, 0x3dfb8, 0x3e004, 0x3e200, 0x3e200, 0x3e208, 0x3e240, 0x3e248, 0x3e280, 0x3e288, 0x3e2c0, 0x3e2c8, 0x3e2fc, 0x3e600, 0x3e630, 0x3ea00, 0x3eabc, 0x3eb00, 0x3eb10, 0x3eb20, 0x3eb30, 0x3eb40, 0x3eb50, 0x3eb60, 0x3eb70, 0x3f000, 0x3f028, 0x3f030, 0x3f048, 0x3f060, 0x3f068, 0x3f070, 0x3f09c, 0x3f0f0, 0x3f128, 0x3f130, 0x3f148, 0x3f160, 0x3f168, 0x3f170, 0x3f19c, 0x3f1f0, 0x3f238, 0x3f240, 0x3f240, 0x3f248, 0x3f250, 0x3f25c, 0x3f264, 0x3f270, 0x3f2b8, 0x3f2c0, 0x3f2e4, 0x3f2f8, 0x3f338, 0x3f340, 0x3f340, 0x3f348, 0x3f350, 0x3f35c, 0x3f364, 0x3f370, 0x3f3b8, 0x3f3c0, 0x3f3e4, 0x3f3f8, 0x3f428, 0x3f430, 0x3f448, 0x3f460, 0x3f468, 0x3f470, 0x3f49c, 0x3f4f0, 0x3f528, 0x3f530, 0x3f548, 0x3f560, 0x3f568, 0x3f570, 0x3f59c, 0x3f5f0, 0x3f638, 0x3f640, 0x3f640, 0x3f648, 0x3f650, 0x3f65c, 0x3f664, 0x3f670, 0x3f6b8, 0x3f6c0, 0x3f6e4, 0x3f6f8, 0x3f738, 0x3f740, 0x3f740, 0x3f748, 0x3f750, 0x3f75c, 0x3f764, 0x3f770, 0x3f7b8, 0x3f7c0, 0x3f7e4, 0x3f7f8, 0x3f7fc, 0x3f814, 0x3f814, 0x3f82c, 0x3f82c, 0x3f880, 0x3f88c, 0x3f8e8, 0x3f8ec, 0x3f900, 0x3f928, 0x3f930, 0x3f948, 0x3f960, 0x3f968, 0x3f970, 0x3f99c, 0x3f9f0, 0x3fa38, 0x3fa40, 0x3fa40, 0x3fa48, 0x3fa50, 0x3fa5c, 0x3fa64, 0x3fa70, 0x3fab8, 0x3fac0, 0x3fae4, 0x3faf8, 0x3fb10, 0x3fb28, 0x3fb28, 0x3fb3c, 0x3fb50, 0x3fbf0, 0x3fc10, 0x3fc28, 0x3fc28, 0x3fc3c, 0x3fc50, 0x3fcf0, 0x3fcfc, 0x40000, 0x4000c, 0x40040, 0x40050, 0x40060, 0x40068, 0x4007c, 0x4008c, 0x40094, 0x400b0, 0x400c0, 0x40144, 0x40180, 0x4018c, 0x40200, 0x40254, 0x40260, 0x40264, 0x40270, 0x40288, 0x40290, 0x40298, 0x402ac, 0x402c8, 0x402d0, 0x402e0, 0x402f0, 0x402f0, 0x40300, 0x4033c, 0x403f8, 0x403fc, 0x41304, 0x413c4, 0x41400, 0x4140c, 0x41414, 0x4141c, 0x41480, 0x414d0, 0x44000, 0x44054, 0x4405c, 0x44078, 0x440c0, 0x44174, 0x44180, 0x441ac, 0x441b4, 0x441b8, 0x441c0, 0x44254, 0x4425c, 0x44278, 0x442c0, 0x44374, 0x44380, 0x443ac, 0x443b4, 0x443b8, 0x443c0, 0x44454, 0x4445c, 0x44478, 0x444c0, 0x44574, 0x44580, 0x445ac, 0x445b4, 0x445b8, 0x445c0, 0x44654, 0x4465c, 0x44678, 0x446c0, 0x44774, 0x44780, 0x447ac, 0x447b4, 0x447b8, 0x447c0, 0x44854, 0x4485c, 0x44878, 0x448c0, 0x44974, 0x44980, 0x449ac, 0x449b4, 0x449b8, 0x449c0, 0x449fc, 0x45000, 0x45004, 0x45010, 0x45030, 0x45040, 0x45060, 0x45068, 0x45068, 0x45080, 0x45084, 0x450a0, 0x450b0, 0x45200, 0x45204, 0x45210, 0x45230, 0x45240, 0x45260, 0x45268, 0x45268, 0x45280, 0x45284, 0x452a0, 0x452b0, 0x460c0, 0x460e4, 0x47000, 0x4703c, 0x47044, 0x4708c, 0x47200, 0x47250, 0x47400, 0x47408, 0x47414, 0x47420, 0x47600, 0x47618, 0x47800, 0x47814, 0x48000, 0x4800c, 0x48040, 0x48050, 0x48060, 0x48068, 0x4807c, 0x4808c, 0x48094, 0x480b0, 0x480c0, 0x48144, 0x48180, 0x4818c, 0x48200, 0x48254, 0x48260, 0x48264, 0x48270, 0x48288, 0x48290, 0x48298, 0x482ac, 0x482c8, 0x482d0, 0x482e0, 0x482f0, 0x482f0, 0x48300, 0x4833c, 0x483f8, 0x483fc, 0x49304, 0x493c4, 0x49400, 0x4940c, 0x49414, 0x4941c, 0x49480, 0x494d0, 0x4c000, 0x4c054, 0x4c05c, 0x4c078, 0x4c0c0, 0x4c174, 0x4c180, 0x4c1ac, 0x4c1b4, 0x4c1b8, 0x4c1c0, 0x4c254, 0x4c25c, 0x4c278, 0x4c2c0, 0x4c374, 0x4c380, 0x4c3ac, 0x4c3b4, 0x4c3b8, 0x4c3c0, 0x4c454, 0x4c45c, 0x4c478, 0x4c4c0, 0x4c574, 0x4c580, 0x4c5ac, 0x4c5b4, 0x4c5b8, 0x4c5c0, 0x4c654, 0x4c65c, 0x4c678, 0x4c6c0, 0x4c774, 0x4c780, 0x4c7ac, 0x4c7b4, 0x4c7b8, 0x4c7c0, 0x4c854, 0x4c85c, 0x4c878, 0x4c8c0, 0x4c974, 0x4c980, 0x4c9ac, 0x4c9b4, 0x4c9b8, 0x4c9c0, 0x4c9fc, 0x4d000, 0x4d004, 0x4d010, 0x4d030, 0x4d040, 0x4d060, 0x4d068, 0x4d068, 0x4d080, 0x4d084, 0x4d0a0, 0x4d0b0, 0x4d200, 0x4d204, 0x4d210, 0x4d230, 0x4d240, 0x4d260, 0x4d268, 0x4d268, 0x4d280, 0x4d284, 0x4d2a0, 0x4d2b0, 0x4e0c0, 0x4e0e4, 0x4f000, 0x4f03c, 0x4f044, 0x4f08c, 0x4f200, 0x4f250, 0x4f400, 0x4f408, 0x4f414, 0x4f420, 0x4f600, 0x4f618, 0x4f800, 0x4f814, 0x50000, 0x50084, 0x50090, 0x500cc, 0x50400, 0x50400, 0x50800, 0x50884, 0x50890, 0x508cc, 0x50c00, 0x50c00, 0x51000, 0x5101c, 0x51300, 0x51308, }; static const unsigned int t5vf_reg_ranges[] = { VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), VF_MPS_REG(A_MPS_VF_CTL), VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), FW_T4VF_MBDATA_BASE_ADDR, FW_T4VF_MBDATA_BASE_ADDR + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), }; static const unsigned int t6_reg_ranges[] = { 0x1008, 0x101c, 0x1024, 0x10a8, 0x10b4, 0x10f8, 0x1100, 0x1114, 0x111c, 0x112c, 0x1138, 0x113c, 0x1144, 0x114c, 0x1180, 0x1184, 0x1190, 0x1194, 0x11a0, 0x11a4, 0x11b0, 0x11b4, 0x11fc, 0x1274, 0x1280, 0x133c, 0x1800, 0x18fc, 0x3000, 0x302c, 0x3060, 0x30b0, 0x30b8, 0x30d8, 0x30e0, 0x30fc, 0x3140, 0x357c, 0x35a8, 0x35cc, 0x35ec, 0x35ec, 0x3600, 0x5624, 0x56cc, 0x56ec, 0x56f4, 0x5720, 0x5728, 0x575c, 0x580c, 0x5814, 0x5890, 0x589c, 0x58a4, 0x58ac, 0x58b8, 0x58bc, 0x5940, 0x595c, 0x5980, 0x598c, 0x59b0, 0x59c8, 0x59d0, 0x59dc, 0x59fc, 0x5a18, 0x5a60, 0x5a6c, 0x5a80, 0x5a8c, 0x5a94, 0x5a9c, 0x5b94, 0x5bfc, 0x5c10, 0x5e48, 0x5e50, 0x5e94, 0x5ea0, 0x5eb0, 0x5ec0, 0x5ec0, 0x5ec8, 0x5ed0, 0x5ee0, 0x5ee0, 0x5ef0, 0x5ef0, 0x5f00, 0x5f00, 0x6000, 0x6020, 0x6028, 0x6040, 0x6058, 0x609c, 0x60a8, 0x619c, 0x7700, 0x7798, 0x77c0, 0x7880, 0x78cc, 0x78fc, 0x7b00, 0x7b58, 0x7b60, 0x7b84, 0x7b8c, 0x7c54, 0x7d00, 0x7d38, 0x7d40, 0x7d84, 0x7d8c, 0x7ddc, 0x7de4, 0x7e04, 0x7e10, 0x7e1c, 0x7e24, 0x7e38, 0x7e40, 0x7e44, 0x7e4c, 0x7e78, 0x7e80, 0x7edc, 0x7ee8, 0x7efc, 0x8dc0, 0x8de4, 0x8df8, 0x8e04, 0x8e10, 0x8e84, 0x8ea0, 0x8f88, 0x8fb8, 0x9058, 0x9060, 0x9060, 0x9068, 0x90f8, 0x9100, 0x9124, 0x9400, 0x9470, 0x9600, 0x9600, 0x9608, 0x9638, 0x9640, 0x9704, 0x9710, 0x971c, 0x9800, 0x9808, 0x9820, 0x983c, 0x9850, 0x9864, 0x9c00, 0x9c6c, 0x9c80, 0x9cec, 0x9d00, 0x9d6c, 0x9d80, 0x9dec, 0x9e00, 0x9e6c, 0x9e80, 0x9eec, 0x9f00, 0x9f6c, 0x9f80, 0xa020, 0xd004, 0xd03c, 0xd100, 0xd118, 0xd200, 0xd214, 0xd220, 0xd234, 0xd240, 0xd254, 0xd260, 0xd274, 0xd280, 0xd294, 0xd2a0, 0xd2b4, 0xd2c0, 0xd2d4, 0xd2e0, 0xd2f4, 0xd300, 0xd31c, 0xdfc0, 0xdfe0, 0xe000, 0xf008, 0xf010, 0xf018, 0xf020, 0xf028, 0x11000, 0x11014, 0x11048, 0x1106c, 0x11074, 0x11088, 0x11098, 0x11120, 0x1112c, 0x1117c, 0x11190, 0x112e0, 0x11300, 0x1130c, 0x12000, 0x1206c, 0x19040, 0x1906c, 0x19078, 0x19080, 0x1908c, 0x190e8, 0x190f0, 0x190f8, 0x19100, 0x19110, 0x19120, 0x19124, 0x19150, 0x19194, 0x1919c, 0x191b0, 0x191d0, 0x191e8, 0x19238, 0x19290, 0x192a4, 0x192b0, 0x192bc, 0x192bc, 0x19348, 0x1934c, 0x193f8, 0x19418, 0x19420, 0x19428, 0x19430, 0x19444, 0x1944c, 0x1946c, 0x19474, 0x19474, 0x19490, 0x194cc, 0x194f0, 0x194f8, 0x19c00, 0x19c48, 0x19c50, 0x19c80, 0x19c94, 0x19c98, 0x19ca0, 0x19cbc, 0x19ce4, 0x19ce4, 0x19cf0, 0x19cf8, 0x19d00, 0x19d28, 0x19d50, 0x19d78, 0x19d94, 0x19d98, 0x19da0, 0x19dc8, 0x19df0, 0x19e10, 0x19e50, 0x19e6c, 0x19ea0, 0x19ebc, 0x19ec4, 0x19ef4, 0x19f04, 0x19f2c, 0x19f34, 0x19f34, 0x19f40, 0x19f50, 0x19f90, 0x19fac, 0x19fc4, 0x19fc8, 0x19fd0, 0x19fe4, 0x1a000, 0x1a004, 0x1a010, 0x1a06c, 0x1a0b0, 0x1a0e4, 0x1a0ec, 0x1a0f8, 0x1a100, 0x1a108, 0x1a114, 0x1a120, 0x1a128, 0x1a130, 0x1a138, 0x1a138, 0x1a190, 0x1a1c4, 0x1a1fc, 0x1a1fc, 0x1e008, 0x1e00c, 0x1e040, 0x1e044, 0x1e04c, 0x1e04c, 0x1e284, 0x1e290, 0x1e2c0, 0x1e2c0, 0x1e2e0, 0x1e2e0, 0x1e300, 0x1e384, 0x1e3c0, 0x1e3c8, 0x1e408, 0x1e40c, 0x1e440, 0x1e444, 0x1e44c, 0x1e44c, 0x1e684, 0x1e690, 0x1e6c0, 0x1e6c0, 0x1e6e0, 0x1e6e0, 0x1e700, 0x1e784, 0x1e7c0, 0x1e7c8, 0x1e808, 0x1e80c, 0x1e840, 0x1e844, 0x1e84c, 0x1e84c, 0x1ea84, 0x1ea90, 0x1eac0, 0x1eac0, 0x1eae0, 0x1eae0, 0x1eb00, 0x1eb84, 0x1ebc0, 0x1ebc8, 0x1ec08, 0x1ec0c, 0x1ec40, 0x1ec44, 0x1ec4c, 0x1ec4c, 0x1ee84, 0x1ee90, 0x1eec0, 0x1eec0, 0x1eee0, 0x1eee0, 0x1ef00, 0x1ef84, 0x1efc0, 0x1efc8, 0x1f008, 0x1f00c, 0x1f040, 0x1f044, 0x1f04c, 0x1f04c, 0x1f284, 0x1f290, 0x1f2c0, 0x1f2c0, 0x1f2e0, 0x1f2e0, 0x1f300, 0x1f384, 0x1f3c0, 0x1f3c8, 0x1f408, 0x1f40c, 0x1f440, 0x1f444, 0x1f44c, 0x1f44c, 0x1f684, 0x1f690, 0x1f6c0, 0x1f6c0, 0x1f6e0, 0x1f6e0, 0x1f700, 0x1f784, 0x1f7c0, 0x1f7c8, 0x1f808, 0x1f80c, 0x1f840, 0x1f844, 0x1f84c, 0x1f84c, 0x1fa84, 0x1fa90, 0x1fac0, 0x1fac0, 0x1fae0, 0x1fae0, 0x1fb00, 0x1fb84, 0x1fbc0, 0x1fbc8, 0x1fc08, 0x1fc0c, 0x1fc40, 0x1fc44, 0x1fc4c, 0x1fc4c, 0x1fe84, 0x1fe90, 0x1fec0, 0x1fec0, 0x1fee0, 0x1fee0, 0x1ff00, 0x1ff84, 0x1ffc0, 0x1ffc8, 0x30000, 0x30030, - 0x30038, 0x30038, - 0x30040, 0x30040, - 0x30048, 0x30048, - 0x30050, 0x30050, - 0x3005c, 0x30060, - 0x30068, 0x30068, - 0x30070, 0x30070, 0x30100, 0x30168, 0x30190, 0x301a0, 0x301a8, 0x301b8, 0x301c4, 0x301c8, 0x301d0, 0x301d0, 0x30200, 0x30320, 0x30400, 0x304b4, 0x304c0, 0x3052c, 0x30540, 0x3061c, 0x30800, 0x308a0, 0x308c0, 0x30908, 0x30910, 0x309b8, 0x30a00, 0x30a04, 0x30a0c, 0x30a14, 0x30a1c, 0x30a2c, 0x30a44, 0x30a50, 0x30a74, 0x30a74, 0x30a7c, 0x30afc, 0x30b08, 0x30c24, 0x30d00, 0x30d14, 0x30d1c, 0x30d3c, 0x30d44, 0x30d4c, 0x30d54, 0x30d74, 0x30d7c, 0x30d7c, 0x30de0, 0x30de0, 0x30e00, 0x30ed4, 0x30f00, 0x30fa4, 0x30fc0, 0x30fc4, 0x31000, 0x31004, 0x31080, 0x310fc, 0x31208, 0x31220, 0x3123c, 0x31254, 0x31300, 0x31300, 0x31308, 0x3131c, 0x31338, 0x3133c, 0x31380, 0x31380, 0x31388, 0x313a8, 0x313b4, 0x313b4, 0x31400, 0x31420, 0x31438, 0x3143c, 0x31480, 0x31480, 0x314a8, 0x314a8, 0x314b0, 0x314b4, 0x314c8, 0x314d4, 0x31a40, 0x31a4c, 0x31af0, 0x31b20, 0x31b38, 0x31b3c, 0x31b80, 0x31b80, 0x31ba8, 0x31ba8, 0x31bb0, 0x31bb4, 0x31bc8, 0x31bd4, 0x32140, 0x3218c, 0x321f0, 0x321f4, 0x32200, 0x32200, 0x32218, 0x32218, 0x32400, 0x32400, 0x32408, 0x3241c, 0x32618, 0x32620, 0x32664, 0x32664, 0x326a8, 0x326a8, 0x326ec, 0x326ec, 0x32a00, 0x32abc, - 0x32b00, 0x32b38, + 0x32b00, 0x32b18, + 0x32b20, 0x32b38, 0x32b40, 0x32b58, 0x32b60, 0x32b78, 0x32c00, 0x32c00, 0x32c08, 0x32c3c, - 0x32e00, 0x32e2c, - 0x32f00, 0x32f2c, 0x33000, 0x3302c, 0x33034, 0x33050, 0x33058, 0x33058, 0x33060, 0x3308c, 0x3309c, 0x330ac, 0x330c0, 0x330c0, 0x330c8, 0x330d0, 0x330d8, 0x330e0, 0x330ec, 0x3312c, 0x33134, 0x33150, 0x33158, 0x33158, 0x33160, 0x3318c, 0x3319c, 0x331ac, 0x331c0, 0x331c0, 0x331c8, 0x331d0, 0x331d8, 0x331e0, 0x331ec, 0x33290, 0x33298, 0x332c4, 0x332e4, 0x33390, 0x33398, 0x333c4, 0x333e4, 0x3342c, 0x33434, 0x33450, 0x33458, 0x33458, 0x33460, 0x3348c, 0x3349c, 0x334ac, 0x334c0, 0x334c0, 0x334c8, 0x334d0, 0x334d8, 0x334e0, 0x334ec, 0x3352c, 0x33534, 0x33550, 0x33558, 0x33558, 0x33560, 0x3358c, 0x3359c, 0x335ac, 0x335c0, 0x335c0, 0x335c8, 0x335d0, 0x335d8, 0x335e0, 0x335ec, 0x33690, 0x33698, 0x336c4, 0x336e4, 0x33790, 0x33798, 0x337c4, 0x337e4, 0x337fc, 0x33814, 0x33814, 0x33854, 0x33868, 0x33880, 0x3388c, 0x338c0, 0x338d0, 0x338e8, 0x338ec, 0x33900, 0x3392c, 0x33934, 0x33950, 0x33958, 0x33958, 0x33960, 0x3398c, 0x3399c, 0x339ac, 0x339c0, 0x339c0, 0x339c8, 0x339d0, 0x339d8, 0x339e0, 0x339ec, 0x33a90, 0x33a98, 0x33ac4, 0x33ae4, 0x33b10, 0x33b24, 0x33b28, 0x33b38, 0x33b50, 0x33bf0, 0x33c10, 0x33c24, 0x33c28, 0x33c38, 0x33c50, 0x33cf0, 0x33cfc, 0x34000, 0x34030, - 0x34038, 0x34038, - 0x34040, 0x34040, - 0x34048, 0x34048, - 0x34050, 0x34050, - 0x3405c, 0x34060, - 0x34068, 0x34068, - 0x34070, 0x34070, 0x34100, 0x34168, 0x34190, 0x341a0, 0x341a8, 0x341b8, 0x341c4, 0x341c8, 0x341d0, 0x341d0, 0x34200, 0x34320, 0x34400, 0x344b4, 0x344c0, 0x3452c, 0x34540, 0x3461c, 0x34800, 0x348a0, 0x348c0, 0x34908, 0x34910, 0x349b8, 0x34a00, 0x34a04, 0x34a0c, 0x34a14, 0x34a1c, 0x34a2c, 0x34a44, 0x34a50, 0x34a74, 0x34a74, 0x34a7c, 0x34afc, 0x34b08, 0x34c24, 0x34d00, 0x34d14, 0x34d1c, 0x34d3c, 0x34d44, 0x34d4c, 0x34d54, 0x34d74, 0x34d7c, 0x34d7c, 0x34de0, 0x34de0, 0x34e00, 0x34ed4, 0x34f00, 0x34fa4, 0x34fc0, 0x34fc4, 0x35000, 0x35004, 0x35080, 0x350fc, 0x35208, 0x35220, 0x3523c, 0x35254, 0x35300, 0x35300, 0x35308, 0x3531c, 0x35338, 0x3533c, 0x35380, 0x35380, 0x35388, 0x353a8, 0x353b4, 0x353b4, 0x35400, 0x35420, 0x35438, 0x3543c, 0x35480, 0x35480, 0x354a8, 0x354a8, 0x354b0, 0x354b4, 0x354c8, 0x354d4, 0x35a40, 0x35a4c, 0x35af0, 0x35b20, 0x35b38, 0x35b3c, 0x35b80, 0x35b80, 0x35ba8, 0x35ba8, 0x35bb0, 0x35bb4, 0x35bc8, 0x35bd4, 0x36140, 0x3618c, 0x361f0, 0x361f4, 0x36200, 0x36200, 0x36218, 0x36218, 0x36400, 0x36400, 0x36408, 0x3641c, 0x36618, 0x36620, 0x36664, 0x36664, 0x366a8, 0x366a8, 0x366ec, 0x366ec, 0x36a00, 0x36abc, - 0x36b00, 0x36b38, + 0x36b00, 0x36b18, + 0x36b20, 0x36b38, 0x36b40, 0x36b58, 0x36b60, 0x36b78, 0x36c00, 0x36c00, 0x36c08, 0x36c3c, - 0x36e00, 0x36e2c, - 0x36f00, 0x36f2c, 0x37000, 0x3702c, 0x37034, 0x37050, 0x37058, 0x37058, 0x37060, 0x3708c, 0x3709c, 0x370ac, 0x370c0, 0x370c0, 0x370c8, 0x370d0, 0x370d8, 0x370e0, 0x370ec, 0x3712c, 0x37134, 0x37150, 0x37158, 0x37158, 0x37160, 0x3718c, 0x3719c, 0x371ac, 0x371c0, 0x371c0, 0x371c8, 0x371d0, 0x371d8, 0x371e0, 0x371ec, 0x37290, 0x37298, 0x372c4, 0x372e4, 0x37390, 0x37398, 0x373c4, 0x373e4, 0x3742c, 0x37434, 0x37450, 0x37458, 0x37458, 0x37460, 0x3748c, 0x3749c, 0x374ac, 0x374c0, 0x374c0, 0x374c8, 0x374d0, 0x374d8, 0x374e0, 0x374ec, 0x3752c, 0x37534, 0x37550, 0x37558, 0x37558, 0x37560, 0x3758c, 0x3759c, 0x375ac, 0x375c0, 0x375c0, 0x375c8, 0x375d0, 0x375d8, 0x375e0, 0x375ec, 0x37690, 0x37698, 0x376c4, 0x376e4, 0x37790, 0x37798, 0x377c4, 0x377e4, 0x377fc, 0x37814, 0x37814, 0x37854, 0x37868, 0x37880, 0x3788c, 0x378c0, 0x378d0, 0x378e8, 0x378ec, 0x37900, 0x3792c, 0x37934, 0x37950, 0x37958, 0x37958, 0x37960, 0x3798c, 0x3799c, 0x379ac, 0x379c0, 0x379c0, 0x379c8, 0x379d0, 0x379d8, 0x379e0, 0x379ec, 0x37a90, 0x37a98, 0x37ac4, 0x37ae4, 0x37b10, 0x37b24, 0x37b28, 0x37b38, 0x37b50, 0x37bf0, 0x37c10, 0x37c24, 0x37c28, 0x37c38, 0x37c50, 0x37cf0, 0x37cfc, 0x40040, 0x40040, 0x40080, 0x40084, 0x40100, 0x40100, 0x40140, 0x401bc, 0x40200, 0x40214, 0x40228, 0x40228, 0x40240, 0x40258, 0x40280, 0x40280, 0x40304, 0x40304, 0x40330, 0x4033c, 0x41304, 0x413c8, 0x413d0, 0x413dc, 0x413f0, 0x413f0, 0x41400, 0x4140c, 0x41414, 0x4141c, 0x41480, 0x414d0, 0x44000, 0x4407c, 0x440c0, 0x441ac, 0x441b4, 0x4427c, 0x442c0, 0x443ac, 0x443b4, 0x4447c, 0x444c0, 0x445ac, 0x445b4, 0x4467c, 0x446c0, 0x447ac, 0x447b4, 0x4487c, 0x448c0, 0x449ac, 0x449b4, 0x44a7c, 0x44ac0, 0x44bac, 0x44bb4, 0x44c7c, 0x44cc0, 0x44dac, 0x44db4, 0x44e7c, 0x44ec0, 0x44fac, 0x44fb4, 0x4507c, 0x450c0, 0x451ac, 0x451b4, 0x451fc, 0x45800, 0x45804, 0x45810, 0x45830, 0x45840, 0x45860, 0x45868, 0x45868, 0x45880, 0x45884, 0x458a0, 0x458b0, 0x45a00, 0x45a04, 0x45a10, 0x45a30, 0x45a40, 0x45a60, 0x45a68, 0x45a68, 0x45a80, 0x45a84, 0x45aa0, 0x45ab0, 0x460c0, 0x460e4, 0x47000, 0x4703c, 0x47044, 0x4708c, 0x47200, 0x47250, 0x47400, 0x47408, 0x47414, 0x47420, 0x47600, 0x47618, 0x47800, 0x47814, 0x47820, 0x4782c, 0x50000, 0x50084, 0x50090, 0x500cc, 0x50300, 0x50384, 0x50400, 0x50400, 0x50800, 0x50884, 0x50890, 0x508cc, 0x50b00, 0x50b84, 0x50c00, 0x50c00, 0x51000, 0x51020, 0x51028, 0x510b0, 0x51300, 0x51324, }; static const unsigned int t6vf_reg_ranges[] = { VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), VF_MPS_REG(A_MPS_VF_CTL), VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), FW_T6VF_MBDATA_BASE_ADDR, FW_T6VF_MBDATA_BASE_ADDR + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), }; u32 *buf_end = (u32 *)(buf + buf_size); const unsigned int *reg_ranges; int reg_ranges_size, range; unsigned int chip_version = chip_id(adap); /* * Select the right set of register ranges to dump depending on the * adapter chip type. */ switch (chip_version) { case CHELSIO_T4: if (adap->flags & IS_VF) { reg_ranges = t4vf_reg_ranges; reg_ranges_size = ARRAY_SIZE(t4vf_reg_ranges); } else { reg_ranges = t4_reg_ranges; reg_ranges_size = ARRAY_SIZE(t4_reg_ranges); } break; case CHELSIO_T5: if (adap->flags & IS_VF) { reg_ranges = t5vf_reg_ranges; reg_ranges_size = ARRAY_SIZE(t5vf_reg_ranges); } else { reg_ranges = t5_reg_ranges; reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); } break; case CHELSIO_T6: if (adap->flags & IS_VF) { reg_ranges = t6vf_reg_ranges; reg_ranges_size = ARRAY_SIZE(t6vf_reg_ranges); } else { reg_ranges = t6_reg_ranges; reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); } break; default: CH_ERR(adap, "Unsupported chip version %d\n", chip_version); return; } /* * Clear the register buffer and insert the appropriate register * values selected by the above register ranges. */ memset(buf, 0, buf_size); for (range = 0; range < reg_ranges_size; range += 2) { unsigned int reg = reg_ranges[range]; unsigned int last_reg = reg_ranges[range + 1]; u32 *bufp = (u32 *)(buf + reg); /* * Iterate across the register range filling in the register * buffer but don't write past the end of the register buffer. */ while (reg <= last_reg && bufp < buf_end) { *bufp++ = t4_read_reg(adap, reg); reg += sizeof(u32); } } } /* * Partial EEPROM Vital Product Data structure. Includes only the ID and * VPD-R sections. */ struct t4_vpd_hdr { u8 id_tag; u8 id_len[2]; u8 id_data[ID_LEN]; u8 vpdr_tag; u8 vpdr_len[2]; }; /* * EEPROM reads take a few tens of us while writes can take a bit over 5 ms. */ #define EEPROM_DELAY 10 /* 10us per poll spin */ #define EEPROM_MAX_POLL 5000 /* x 5000 == 50ms */ #define EEPROM_STAT_ADDR 0x7bfc +#define VPD_SIZE 0x800 #define VPD_BASE 0x400 #define VPD_BASE_OLD 0 #define VPD_LEN 1024 #define VPD_INFO_FLD_HDR_SIZE 3 #define CHELSIO_VPD_UNIQUE_ID 0x82 /* * Small utility function to wait till any outstanding VPD Access is complete. * We have a per-adapter state variable "VPD Busy" to indicate when we have a * VPD Access in flight. This allows us to handle the problem of having a * previous VPD Access time out and prevent an attempt to inject a new VPD * Request before any in-flight VPD reguest has completed. */ static int t4_seeprom_wait(struct adapter *adapter) { unsigned int base = adapter->params.pci.vpd_cap_addr; int max_poll; /* * If no VPD Access is in flight, we can just return success right * away. */ if (!adapter->vpd_busy) return 0; /* * Poll the VPD Capability Address/Flag register waiting for it * to indicate that the operation is complete. */ max_poll = EEPROM_MAX_POLL; do { u16 val; udelay(EEPROM_DELAY); t4_os_pci_read_cfg2(adapter, base + PCI_VPD_ADDR, &val); /* * If the operation is complete, mark the VPD as no longer * busy and return success. */ if ((val & PCI_VPD_ADDR_F) == adapter->vpd_flag) { adapter->vpd_busy = 0; return 0; } } while (--max_poll); /* * Failure! Note that we leave the VPD Busy status set in order to * avoid pushing a new VPD Access request into the VPD Capability till * the current operation eventually succeeds. It's a bug to issue a * new request when an existing request is in flight and will result * in corrupt hardware state. */ return -ETIMEDOUT; } /** * t4_seeprom_read - read a serial EEPROM location * @adapter: adapter to read * @addr: EEPROM virtual address * @data: where to store the read data * * Read a 32-bit word from a location in serial EEPROM using the card's PCI * VPD capability. Note that this function must be called with a virtual * address. */ int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data) { unsigned int base = adapter->params.pci.vpd_cap_addr; int ret; /* * VPD Accesses must alway be 4-byte aligned! */ if (addr >= EEPROMVSIZE || (addr & 3)) return -EINVAL; /* * Wait for any previous operation which may still be in flight to * complete. */ ret = t4_seeprom_wait(adapter); if (ret) { CH_ERR(adapter, "VPD still busy from previous operation\n"); return ret; } /* * Issue our new VPD Read request, mark the VPD as being busy and wait * for our request to complete. If it doesn't complete, note the * error and return it to our caller. Note that we do not reset the * VPD Busy status! */ t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR, (u16)addr); adapter->vpd_busy = 1; adapter->vpd_flag = PCI_VPD_ADDR_F; ret = t4_seeprom_wait(adapter); if (ret) { CH_ERR(adapter, "VPD read of address %#x failed\n", addr); return ret; } /* * Grab the returned data, swizzle it into our endianness and * return success. */ t4_os_pci_read_cfg4(adapter, base + PCI_VPD_DATA, data); *data = le32_to_cpu(*data); return 0; } /** * t4_seeprom_write - write a serial EEPROM location * @adapter: adapter to write * @addr: virtual EEPROM address * @data: value to write * * Write a 32-bit word to a location in serial EEPROM using the card's PCI * VPD capability. Note that this function must be called with a virtual * address. */ int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data) { unsigned int base = adapter->params.pci.vpd_cap_addr; int ret; u32 stats_reg; int max_poll; /* * VPD Accesses must alway be 4-byte aligned! */ if (addr >= EEPROMVSIZE || (addr & 3)) return -EINVAL; /* * Wait for any previous operation which may still be in flight to * complete. */ ret = t4_seeprom_wait(adapter); if (ret) { CH_ERR(adapter, "VPD still busy from previous operation\n"); return ret; } /* * Issue our new VPD Read request, mark the VPD as being busy and wait * for our request to complete. If it doesn't complete, note the * error and return it to our caller. Note that we do not reset the * VPD Busy status! */ t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA, cpu_to_le32(data)); t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR, (u16)addr | PCI_VPD_ADDR_F); adapter->vpd_busy = 1; adapter->vpd_flag = 0; ret = t4_seeprom_wait(adapter); if (ret) { CH_ERR(adapter, "VPD write of address %#x failed\n", addr); return ret; } /* * Reset PCI_VPD_DATA register after a transaction and wait for our * request to complete. If it doesn't complete, return error. */ t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA, 0); max_poll = EEPROM_MAX_POLL; do { udelay(EEPROM_DELAY); t4_seeprom_read(adapter, EEPROM_STAT_ADDR, &stats_reg); } while ((stats_reg & 0x1) && --max_poll); if (!max_poll) return -ETIMEDOUT; /* Return success! */ return 0; } /** * t4_eeprom_ptov - translate a physical EEPROM address to virtual * @phys_addr: the physical EEPROM address * @fn: the PCI function number * @sz: size of function-specific area * * Translate a physical EEPROM address to virtual. The first 1K is * accessed through virtual addresses starting at 31K, the rest is * accessed through virtual addresses starting at 0. * * The mapping is as follows: * [0..1K) -> [31K..32K) * [1K..1K+A) -> [ES-A..ES) * [1K+A..ES) -> [0..ES-A-1K) * * where A = @fn * @sz, and ES = EEPROM size. */ int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz) { fn *= sz; if (phys_addr < 1024) return phys_addr + (31 << 10); if (phys_addr < 1024 + fn) return EEPROMSIZE - fn + phys_addr - 1024; if (phys_addr < EEPROMSIZE) return phys_addr - 1024 - fn; return -EINVAL; } /** * t4_seeprom_wp - enable/disable EEPROM write protection * @adapter: the adapter * @enable: whether to enable or disable write protection * * Enables or disables write protection on the serial EEPROM. */ int t4_seeprom_wp(struct adapter *adapter, int enable) { return t4_seeprom_write(adapter, EEPROM_STAT_ADDR, enable ? 0xc : 0); } /** * get_vpd_keyword_val - Locates an information field keyword in the VPD * @v: Pointer to buffered vpd data structure * @kw: The keyword to search for * * Returns the value of the information field keyword or * -ENOENT otherwise. */ static int get_vpd_keyword_val(const struct t4_vpd_hdr *v, const char *kw) { int i; unsigned int offset , len; const u8 *buf = (const u8 *)v; const u8 *vpdr_len = &v->vpdr_len[0]; offset = sizeof(struct t4_vpd_hdr); len = (u16)vpdr_len[0] + ((u16)vpdr_len[1] << 8); if (len + sizeof(struct t4_vpd_hdr) > VPD_LEN) { return -ENOENT; } for (i = offset; i + VPD_INFO_FLD_HDR_SIZE <= offset + len;) { if(memcmp(buf + i , kw , 2) == 0){ i += VPD_INFO_FLD_HDR_SIZE; return i; } i += VPD_INFO_FLD_HDR_SIZE + buf[i+2]; } return -ENOENT; } /** * get_vpd_params - read VPD parameters from VPD EEPROM * @adapter: adapter to read * @p: where to store the parameters * @vpd: caller provided temporary space to read the VPD into * * Reads card parameters stored in VPD EEPROM. */ static int get_vpd_params(struct adapter *adapter, struct vpd_params *p, u8 *vpd) { int i, ret, addr; int ec, sn, pn, na; u8 csum; const struct t4_vpd_hdr *v; /* * Card information normally starts at VPD_BASE but early cards had * it at 0. */ ret = t4_seeprom_read(adapter, VPD_BASE, (u32 *)(vpd)); if (ret) return (ret); /* * The VPD shall have a unique identifier specified by the PCI SIG. * For chelsio adapters, the identifier is 0x82. The first byte of a VPD * shall be CHELSIO_VPD_UNIQUE_ID (0x82). The VPD programming software * is expected to automatically put this entry at the * beginning of the VPD. */ addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD; for (i = 0; i < VPD_LEN; i += 4) { ret = t4_seeprom_read(adapter, addr + i, (u32 *)(vpd + i)); if (ret) return ret; } v = (const struct t4_vpd_hdr *)vpd; #define FIND_VPD_KW(var,name) do { \ var = get_vpd_keyword_val(v , name); \ if (var < 0) { \ CH_ERR(adapter, "missing VPD keyword " name "\n"); \ return -EINVAL; \ } \ } while (0) FIND_VPD_KW(i, "RV"); for (csum = 0; i >= 0; i--) csum += vpd[i]; if (csum) { CH_ERR(adapter, "corrupted VPD EEPROM, actual csum %u\n", csum); return -EINVAL; } FIND_VPD_KW(ec, "EC"); FIND_VPD_KW(sn, "SN"); FIND_VPD_KW(pn, "PN"); FIND_VPD_KW(na, "NA"); #undef FIND_VPD_KW memcpy(p->id, v->id_data, ID_LEN); strstrip(p->id); memcpy(p->ec, vpd + ec, EC_LEN); strstrip(p->ec); i = vpd[sn - VPD_INFO_FLD_HDR_SIZE + 2]; memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN)); strstrip(p->sn); i = vpd[pn - VPD_INFO_FLD_HDR_SIZE + 2]; memcpy(p->pn, vpd + pn, min(i, PN_LEN)); strstrip((char *)p->pn); i = vpd[na - VPD_INFO_FLD_HDR_SIZE + 2]; memcpy(p->na, vpd + na, min(i, MACADDR_LEN)); strstrip((char *)p->na); return 0; } /* serial flash and firmware constants and flash config file constants */ enum { SF_ATTEMPTS = 10, /* max retries for SF operations */ /* flash command opcodes */ - SF_PROG_PAGE = 2, /* program page */ + SF_PROG_PAGE = 2, /* program 256B page */ SF_WR_DISABLE = 4, /* disable writes */ SF_RD_STATUS = 5, /* read status register */ SF_WR_ENABLE = 6, /* enable writes */ SF_RD_DATA_FAST = 0xb, /* read flash */ SF_RD_ID = 0x9f, /* read ID */ - SF_ERASE_SECTOR = 0xd8, /* erase sector */ + SF_ERASE_SECTOR = 0xd8, /* erase 64KB sector */ }; /** * sf1_read - read data from the serial flash * @adapter: the adapter * @byte_cnt: number of bytes to read * @cont: whether another operation will be chained * @lock: whether to lock SF for PL access only * @valp: where to store the read data * * Reads up to 4 bytes of data from the serial flash. The location of * the read needs to be specified prior to calling this by issuing the * appropriate commands to the serial flash. */ static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont, int lock, u32 *valp) { int ret; if (!byte_cnt || byte_cnt > 4) return -EINVAL; if (t4_read_reg(adapter, A_SF_OP) & F_BUSY) return -EBUSY; t4_write_reg(adapter, A_SF_OP, V_SF_LOCK(lock) | V_CONT(cont) | V_BYTECNT(byte_cnt - 1)); ret = t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5); if (!ret) *valp = t4_read_reg(adapter, A_SF_DATA); return ret; } /** * sf1_write - write data to the serial flash * @adapter: the adapter * @byte_cnt: number of bytes to write * @cont: whether another operation will be chained * @lock: whether to lock SF for PL access only * @val: value to write * * Writes up to 4 bytes of data to the serial flash. The location of * the write needs to be specified prior to calling this by issuing the * appropriate commands to the serial flash. */ static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont, int lock, u32 val) { if (!byte_cnt || byte_cnt > 4) return -EINVAL; if (t4_read_reg(adapter, A_SF_OP) & F_BUSY) return -EBUSY; t4_write_reg(adapter, A_SF_DATA, val); t4_write_reg(adapter, A_SF_OP, V_SF_LOCK(lock) | V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1)); return t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5); } /** * flash_wait_op - wait for a flash operation to complete * @adapter: the adapter * @attempts: max number of polls of the status register * @delay: delay between polls in ms * * Wait for a flash operation to complete by polling the status register. */ static int flash_wait_op(struct adapter *adapter, int attempts, int delay) { int ret; u32 status; while (1) { if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 || (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0) return ret; if (!(status & 1)) return 0; if (--attempts == 0) return -EAGAIN; if (delay) msleep(delay); } } /** * t4_read_flash - read words from serial flash * @adapter: the adapter * @addr: the start address for the read * @nwords: how many 32-bit words to read * @data: where to store the read data * @byte_oriented: whether to store data as bytes or as words * * Read the specified number of 32-bit words from the serial flash. * If @byte_oriented is set the read data is stored as a byte array * (i.e., big-endian), otherwise as 32-bit words in the platform's * natural endianness. */ int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented) { int ret; if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3)) return -EINVAL; addr = swab32(addr) | SF_RD_DATA_FAST; if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 || (ret = sf1_read(adapter, 1, 1, 0, data)) != 0) return ret; for ( ; nwords; nwords--, data++) { ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data); if (nwords == 1) t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ if (ret) return ret; if (byte_oriented) *data = (__force __u32)(cpu_to_be32(*data)); } return 0; } /** * t4_write_flash - write up to a page of data to the serial flash * @adapter: the adapter * @addr: the start address to write * @n: length of data to write in bytes * @data: the data to write * @byte_oriented: whether to store data as bytes or as words * * Writes up to a page of data (256 bytes) to the serial flash starting * at the given address. All the data must be written to the same page. * If @byte_oriented is set the write data is stored as byte stream * (i.e. matches what on disk), otherwise in big-endian. */ int t4_write_flash(struct adapter *adapter, unsigned int addr, unsigned int n, const u8 *data, int byte_oriented) { int ret; u32 buf[SF_PAGE_SIZE / 4]; unsigned int i, c, left, val, offset = addr & 0xff; if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE) return -EINVAL; val = swab32(addr) | SF_PROG_PAGE; if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 || (ret = sf1_write(adapter, 4, 1, 1, val)) != 0) goto unlock; for (left = n; left; left -= c) { c = min(left, 4U); for (val = 0, i = 0; i < c; ++i) val = (val << 8) + *data++; if (!byte_oriented) val = cpu_to_be32(val); ret = sf1_write(adapter, c, c != left, 1, val); if (ret) goto unlock; } ret = flash_wait_op(adapter, 8, 1); if (ret) goto unlock; t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ /* Read the page to verify the write succeeded */ ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, byte_oriented); if (ret) return ret; if (memcmp(data - n, (u8 *)buf + offset, n)) { CH_ERR(adapter, "failed to correctly write the flash page at %#x\n", addr); return -EIO; } return 0; unlock: t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ return ret; } /** * t4_get_fw_version - read the firmware version * @adapter: the adapter * @vers: where to place the version * * Reads the FW version from flash. */ int t4_get_fw_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, fw_ver), 1, vers, 0); } /** * t4_get_bs_version - read the firmware bootstrap version * @adapter: the adapter * @vers: where to place the version * * Reads the FW Bootstrap version from flash. */ int t4_get_bs_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START + offsetof(struct fw_hdr, fw_ver), 1, vers, 0); } /** * t4_get_tp_version - read the TP microcode version * @adapter: the adapter * @vers: where to place the version * * Reads the TP microcode version from flash. */ int t4_get_tp_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, tp_microcode_ver), 1, vers, 0); } /** * t4_get_exprom_version - return the Expansion ROM version (if any) * @adapter: the adapter * @vers: where to place the version * * Reads the Expansion ROM header from FLASH and returns the version * number (if present) through the @vers return value pointer. We return * this in the Firmware Version Format since it's convenient. Return * 0 on success, -ENOENT if no Expansion ROM is present. */ int t4_get_exprom_version(struct adapter *adap, u32 *vers) { struct exprom_header { unsigned char hdr_arr[16]; /* must start with 0x55aa */ unsigned char hdr_ver[4]; /* Expansion ROM version */ } *hdr; u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header), sizeof(u32))]; int ret; ret = t4_read_flash(adap, FLASH_EXP_ROM_START, ARRAY_SIZE(exprom_header_buf), exprom_header_buf, 0); if (ret) return ret; hdr = (struct exprom_header *)exprom_header_buf; if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa) return -ENOENT; *vers = (V_FW_HDR_FW_VER_MAJOR(hdr->hdr_ver[0]) | V_FW_HDR_FW_VER_MINOR(hdr->hdr_ver[1]) | V_FW_HDR_FW_VER_MICRO(hdr->hdr_ver[2]) | V_FW_HDR_FW_VER_BUILD(hdr->hdr_ver[3])); return 0; } /** * t4_get_scfg_version - return the Serial Configuration version * @adapter: the adapter * @vers: where to place the version * * Reads the Serial Configuration Version via the Firmware interface * (thus this can only be called once we're ready to issue Firmware * commands). The format of the Serial Configuration version is * adapter specific. Returns 0 on success, an error on failure. * * Note that early versions of the Firmware didn't include the ability * to retrieve the Serial Configuration version, so we zero-out the * return-value parameter in that case to avoid leaving it with * garbage in it. * * Also note that the Firmware will return its cached copy of the Serial * Initialization Revision ID, not the actual Revision ID as written in * the Serial EEPROM. This is only an issue if a new VPD has been written * and the Firmware/Chip haven't yet gone through a RESET sequence. So * it's best to defer calling this routine till after a FW_RESET_CMD has * been issued if the Host Driver will be performing a full adapter * initialization. */ int t4_get_scfg_version(struct adapter *adapter, u32 *vers) { u32 scfgrev_param; int ret; scfgrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_SCFGREV)); ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 1, &scfgrev_param, vers); if (ret) *vers = 0; return ret; } /** * t4_get_vpd_version - return the VPD version * @adapter: the adapter * @vers: where to place the version * * Reads the VPD via the Firmware interface (thus this can only be called * once we're ready to issue Firmware commands). The format of the * VPD version is adapter specific. Returns 0 on success, an error on * failure. * * Note that early versions of the Firmware didn't include the ability * to retrieve the VPD version, so we zero-out the return-value parameter * in that case to avoid leaving it with garbage in it. * * Also note that the Firmware will return its cached copy of the VPD * Revision ID, not the actual Revision ID as written in the Serial * EEPROM. This is only an issue if a new VPD has been written and the * Firmware/Chip haven't yet gone through a RESET sequence. So it's best * to defer calling this routine till after a FW_RESET_CMD has been issued * if the Host Driver will be performing a full adapter initialization. */ int t4_get_vpd_version(struct adapter *adapter, u32 *vers) { u32 vpdrev_param; int ret; vpdrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_VPDREV)); ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 1, &vpdrev_param, vers); if (ret) *vers = 0; return ret; } /** * t4_get_version_info - extract various chip/firmware version information * @adapter: the adapter * * Reads various chip/firmware version numbers and stores them into the * adapter Adapter Parameters structure. If any of the efforts fails * the first failure will be returned, but all of the version numbers * will be read. */ int t4_get_version_info(struct adapter *adapter) { int ret = 0; #define FIRST_RET(__getvinfo) \ do { \ int __ret = __getvinfo; \ if (__ret && !ret) \ ret = __ret; \ } while (0) FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers)); FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers)); FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers)); FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers)); FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers)); FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers)); #undef FIRST_RET return ret; } /** * t4_flash_erase_sectors - erase a range of flash sectors * @adapter: the adapter * @start: the first sector to erase * @end: the last sector to erase * * Erases the sectors in the given inclusive range. */ int t4_flash_erase_sectors(struct adapter *adapter, int start, int end) { int ret = 0; if (end >= adapter->params.sf_nsec) return -EINVAL; while (start <= end) { if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 || (ret = sf1_write(adapter, 4, 0, 1, SF_ERASE_SECTOR | (start << 8))) != 0 || (ret = flash_wait_op(adapter, 14, 500)) != 0) { CH_ERR(adapter, "erase of flash sector %d failed, error %d\n", start, ret); break; } start++; } t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ return ret; } /** * t4_flash_cfg_addr - return the address of the flash configuration file * @adapter: the adapter * * Return the address within the flash where the Firmware Configuration * File is stored, or an error if the device FLASH is too small to contain * a Firmware Configuration File. */ int t4_flash_cfg_addr(struct adapter *adapter) { /* * If the device FLASH isn't large enough to hold a Firmware * Configuration File, return an error. */ if (adapter->params.sf_size < FLASH_CFG_START + FLASH_CFG_MAX_SIZE) return -ENOSPC; return FLASH_CFG_START; } /* * Return TRUE if the specified firmware matches the adapter. I.e. T4 * firmware for T4 adapters, T5 firmware for T5 adapters, etc. We go ahead * and emit an error message for mismatched firmware to save our caller the * effort ... */ static int t4_fw_matches_chip(struct adapter *adap, const struct fw_hdr *hdr) { /* * The expression below will return FALSE for any unsupported adapter * which will keep us "honest" in the future ... */ if ((is_t4(adap) && hdr->chip == FW_HDR_CHIP_T4) || (is_t5(adap) && hdr->chip == FW_HDR_CHIP_T5) || (is_t6(adap) && hdr->chip == FW_HDR_CHIP_T6)) return 1; CH_ERR(adap, "FW image (%d) is not suitable for this adapter (%d)\n", hdr->chip, chip_id(adap)); return 0; } /** * t4_load_fw - download firmware * @adap: the adapter * @fw_data: the firmware image to write * @size: image size * * Write the supplied firmware image to the card's serial flash. */ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size) { u32 csum; int ret, addr; unsigned int i; u8 first_page[SF_PAGE_SIZE]; const u32 *p = (const u32 *)fw_data; const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data; unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; unsigned int fw_start_sec; unsigned int fw_start; unsigned int fw_size; if (ntohl(hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP) { fw_start_sec = FLASH_FWBOOTSTRAP_START_SEC; fw_start = FLASH_FWBOOTSTRAP_START; fw_size = FLASH_FWBOOTSTRAP_MAX_SIZE; } else { fw_start_sec = FLASH_FW_START_SEC; fw_start = FLASH_FW_START; fw_size = FLASH_FW_MAX_SIZE; } if (!size) { CH_ERR(adap, "FW image has no data\n"); return -EINVAL; } if (size & 511) { CH_ERR(adap, "FW image size not multiple of 512 bytes\n"); return -EINVAL; } if ((unsigned int) be16_to_cpu(hdr->len512) * 512 != size) { CH_ERR(adap, "FW image size differs from size in FW header\n"); return -EINVAL; } if (size > fw_size) { CH_ERR(adap, "FW image too large, max is %u bytes\n", fw_size); return -EFBIG; } if (!t4_fw_matches_chip(adap, hdr)) return -EINVAL; for (csum = 0, i = 0; i < size / sizeof(csum); i++) csum += be32_to_cpu(p[i]); if (csum != 0xffffffff) { CH_ERR(adap, "corrupted firmware image, checksum %#x\n", csum); return -EINVAL; } i = DIV_ROUND_UP(size, sf_sec_size); /* # of sectors spanned */ ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1); if (ret) goto out; /* * We write the correct version at the end so the driver can see a bad * version if the FW write fails. Start by writing a copy of the * first page with a bad version. */ memcpy(first_page, fw_data, SF_PAGE_SIZE); ((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff); ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, 1); if (ret) goto out; addr = fw_start; for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { addr += SF_PAGE_SIZE; fw_data += SF_PAGE_SIZE; ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, 1); if (ret) goto out; } ret = t4_write_flash(adap, fw_start + offsetof(struct fw_hdr, fw_ver), sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, 1); out: if (ret) CH_ERR(adap, "firmware download failed, error %d\n", ret); return ret; } /** * t4_fwcache - firmware cache operation * @adap: the adapter * @op : the operation (flush or flush and invalidate) */ int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op) { struct fw_params_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PARAMS_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_PARAMS_CMD_PFN(adap->pf) | V_FW_PARAMS_CMD_VFN(0)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); c.param[0].mnem = cpu_to_be32(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_FWCACHE)); c.param[0].val = (__force __be32)op; return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL); } void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp, unsigned int *pif_req_wrptr, unsigned int *pif_rsp_wrptr) { int i, j; u32 cfg, val, req, rsp; cfg = t4_read_reg(adap, A_CIM_DEBUGCFG); if (cfg & F_LADBGEN) t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN); val = t4_read_reg(adap, A_CIM_DEBUGSTS); req = G_POLADBGWRPTR(val); rsp = G_PILADBGWRPTR(val); if (pif_req_wrptr) *pif_req_wrptr = req; if (pif_rsp_wrptr) *pif_rsp_wrptr = rsp; for (i = 0; i < CIM_PIFLA_SIZE; i++) { for (j = 0; j < 6; j++) { t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(req) | V_PILADBGRDPTR(rsp)); *pif_req++ = t4_read_reg(adap, A_CIM_PO_LA_DEBUGDATA); *pif_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_DEBUGDATA); req++; rsp++; } req = (req + 2) & M_POLADBGRDPTR; rsp = (rsp + 2) & M_PILADBGRDPTR; } t4_write_reg(adap, A_CIM_DEBUGCFG, cfg); } void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp) { u32 cfg; int i, j, idx; cfg = t4_read_reg(adap, A_CIM_DEBUGCFG); if (cfg & F_LADBGEN) t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN); for (i = 0; i < CIM_MALA_SIZE; i++) { for (j = 0; j < 5; j++) { idx = 8 * i + j; t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(idx) | V_PILADBGRDPTR(idx)); *ma_req++ = t4_read_reg(adap, A_CIM_PO_LA_MADEBUGDATA); *ma_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_MADEBUGDATA); } } t4_write_reg(adap, A_CIM_DEBUGCFG, cfg); } void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf) { unsigned int i, j; for (i = 0; i < 8; i++) { u32 *p = la_buf + i; t4_write_reg(adap, A_ULP_RX_LA_CTL, i); j = t4_read_reg(adap, A_ULP_RX_LA_WRPTR); t4_write_reg(adap, A_ULP_RX_LA_RDPTR, j); for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8) *p = t4_read_reg(adap, A_ULP_RX_LA_RDDATA); } } /** * t4_link_l1cfg - apply link configuration to MAC/PHY * @phy: the PHY to setup * @mac: the MAC to setup * @lc: the requested link configuration * * Set up a port's MAC and PHY according to a desired link configuration. * - If the PHY can auto-negotiate first decide what to advertise, then * enable/disable auto-negotiation as desired, and reset. * - If the PHY does not auto-negotiate just reset it. * - If auto-negotiation is off set the MAC to the proper speed/duplex/FC, * otherwise do it later based on the outcome of auto-negotiation. */ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, struct link_config *lc) { struct fw_port_cmd c; unsigned int mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO); unsigned int aneg, fc, fec, speed; fc = 0; if (lc->requested_fc & PAUSE_RX) fc |= FW_PORT_CAP_FC_RX; if (lc->requested_fc & PAUSE_TX) fc |= FW_PORT_CAP_FC_TX; fec = 0; if (lc->requested_fec & FEC_RS) fec = FW_PORT_CAP_FEC_RS; else if (lc->requested_fec & FEC_BASER_RS) fec = FW_PORT_CAP_FEC_BASER_RS; else if (lc->requested_fec & FEC_RESERVED) fec = FW_PORT_CAP_FEC_RESERVED; if (!(lc->supported & FW_PORT_CAP_ANEG) || lc->requested_aneg == AUTONEG_DISABLE) { aneg = 0; switch (lc->requested_speed) { case 100: speed = FW_PORT_CAP_SPEED_100G; break; case 40: speed = FW_PORT_CAP_SPEED_40G; break; case 25: speed = FW_PORT_CAP_SPEED_25G; break; case 10: speed = FW_PORT_CAP_SPEED_10G; break; case 1: speed = FW_PORT_CAP_SPEED_1G; break; default: return -EINVAL; break; } } else { aneg = FW_PORT_CAP_ANEG; speed = lc->supported & V_FW_PORT_CAP_SPEED(M_FW_PORT_CAP_SPEED); } memset(&c, 0, sizeof(c)); c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_PORT_CMD_PORTID(port)); c.action_to_len16 = cpu_to_be32(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) | FW_LEN16(c)); c.u.l1cfg.rcap = cpu_to_be32(aneg | speed | fc | fec | mdi); return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL); } /** * t4_restart_aneg - restart autonegotiation * @adap: the adapter * @mbox: mbox to use for the FW command * @port: the port id * * Restarts autonegotiation for the selected port. */ int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port) { struct fw_port_cmd c; memset(&c, 0, sizeof(c)); c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_PORT_CMD_PORTID(port)); c.action_to_len16 = cpu_to_be32(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) | FW_LEN16(c)); c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } typedef void (*int_handler_t)(struct adapter *adap); struct intr_info { unsigned int mask; /* bits to check in interrupt status */ const char *msg; /* message to print or NULL */ short stat_idx; /* stat counter to increment or -1 */ unsigned short fatal; /* whether the condition reported is fatal */ int_handler_t int_handler; /* platform-specific int handler */ }; /** * t4_handle_intr_status - table driven interrupt handler * @adapter: the adapter that generated the interrupt * @reg: the interrupt status register to process * @acts: table of interrupt actions * * A table driven interrupt handler that applies a set of masks to an * interrupt status word and performs the corresponding actions if the * interrupts described by the mask have occurred. The actions include * optionally emitting a warning or alert message. The table is terminated * by an entry specifying mask 0. Returns the number of fatal interrupt * conditions. */ static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg, const struct intr_info *acts) { int fatal = 0; unsigned int mask = 0; unsigned int status = t4_read_reg(adapter, reg); for ( ; acts->mask; ++acts) { if (!(status & acts->mask)) continue; if (acts->fatal) { fatal++; CH_ALERT(adapter, "%s (0x%x)\n", acts->msg, status & acts->mask); } else if (acts->msg) CH_WARN_RATELIMIT(adapter, "%s (0x%x)\n", acts->msg, status & acts->mask); if (acts->int_handler) acts->int_handler(adapter); mask |= acts->mask; } status &= mask; if (status) /* clear processed interrupts */ t4_write_reg(adapter, reg, status); return fatal; } /* * Interrupt handler for the PCIE module. */ static void pcie_intr_handler(struct adapter *adapter) { static const struct intr_info sysbus_intr_info[] = { { F_RNPP, "RXNP array parity error", -1, 1 }, { F_RPCP, "RXPC array parity error", -1, 1 }, { F_RCIP, "RXCIF array parity error", -1, 1 }, { F_RCCP, "Rx completions control array parity error", -1, 1 }, { F_RFTP, "RXFT array parity error", -1, 1 }, { 0 } }; static const struct intr_info pcie_port_intr_info[] = { { F_TPCP, "TXPC array parity error", -1, 1 }, { F_TNPP, "TXNP array parity error", -1, 1 }, { F_TFTP, "TXFT array parity error", -1, 1 }, { F_TCAP, "TXCA array parity error", -1, 1 }, { F_TCIP, "TXCIF array parity error", -1, 1 }, { F_RCAP, "RXCA array parity error", -1, 1 }, { F_OTDD, "outbound request TLP discarded", -1, 1 }, { F_RDPE, "Rx data parity error", -1, 1 }, { F_TDUE, "Tx uncorrectable data error", -1, 1 }, { 0 } }; static const struct intr_info pcie_intr_info[] = { { F_MSIADDRLPERR, "MSI AddrL parity error", -1, 1 }, { F_MSIADDRHPERR, "MSI AddrH parity error", -1, 1 }, { F_MSIDATAPERR, "MSI data parity error", -1, 1 }, { F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 }, { F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 }, { F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 }, { F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 }, { F_PIOCPLPERR, "PCI PIO completion FIFO parity error", -1, 1 }, { F_PIOREQPERR, "PCI PIO request FIFO parity error", -1, 1 }, { F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 }, { F_CCNTPERR, "PCI CMD channel count parity error", -1, 1 }, { F_CREQPERR, "PCI CMD channel request parity error", -1, 1 }, { F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 }, { F_DCNTPERR, "PCI DMA channel count parity error", -1, 1 }, { F_DREQPERR, "PCI DMA channel request parity error", -1, 1 }, { F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 }, { F_HCNTPERR, "PCI HMA channel count parity error", -1, 1 }, { F_HREQPERR, "PCI HMA channel request parity error", -1, 1 }, { F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 }, { F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 }, { F_FIDPERR, "PCI FID parity error", -1, 1 }, { F_INTXCLRPERR, "PCI INTx clear parity error", -1, 1 }, { F_MATAGPERR, "PCI MA tag parity error", -1, 1 }, { F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 }, { F_RXCPLPERR, "PCI Rx completion parity error", -1, 1 }, { F_RXWRPERR, "PCI Rx write parity error", -1, 1 }, { F_RPLPERR, "PCI replay buffer parity error", -1, 1 }, { F_PCIESINT, "PCI core secondary fault", -1, 1 }, { F_PCIEPINT, "PCI core primary fault", -1, 1 }, { F_UNXSPLCPLERR, "PCI unexpected split completion error", -1, 0 }, { 0 } }; static const struct intr_info t5_pcie_intr_info[] = { { F_MSTGRPPERR, "Master Response Read Queue parity error", -1, 1 }, { F_MSTTIMEOUTPERR, "Master Timeout FIFO parity error", -1, 1 }, { F_MSIXSTIPERR, "MSI-X STI SRAM parity error", -1, 1 }, { F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 }, { F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 }, { F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 }, { F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 }, { F_PIOCPLGRPPERR, "PCI PIO completion Group FIFO parity error", -1, 1 }, { F_PIOREQGRPPERR, "PCI PIO request Group FIFO parity error", -1, 1 }, { F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 }, { F_MSTTAGQPERR, "PCI master tag queue parity error", -1, 1 }, { F_CREQPERR, "PCI CMD channel request parity error", -1, 1 }, { F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 }, { F_DREQWRPERR, "PCI DMA channel write request parity error", -1, 1 }, { F_DREQPERR, "PCI DMA channel request parity error", -1, 1 }, { F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 }, { F_HREQWRPERR, "PCI HMA channel count parity error", -1, 1 }, { F_HREQPERR, "PCI HMA channel request parity error", -1, 1 }, { F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 }, { F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 }, { F_FIDPERR, "PCI FID parity error", -1, 1 }, { F_VFIDPERR, "PCI INTx clear parity error", -1, 1 }, { F_MAGRPPERR, "PCI MA group FIFO parity error", -1, 1 }, { F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 }, { F_IPRXHDRGRPPERR, "PCI IP Rx header group parity error", -1, 1 }, { F_IPRXDATAGRPPERR, "PCI IP Rx data group parity error", -1, 1 }, { F_RPLPERR, "PCI IP replay buffer parity error", -1, 1 }, { F_IPSOTPERR, "PCI IP SOT buffer parity error", -1, 1 }, { F_TRGT1GRPPERR, "PCI TRGT1 group FIFOs parity error", -1, 1 }, { F_READRSPERR, "Outbound read error", -1, 0 }, { 0 } }; int fat; if (is_t4(adapter)) fat = t4_handle_intr_status(adapter, A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS, sysbus_intr_info) + t4_handle_intr_status(adapter, A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS, pcie_port_intr_info) + t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE, pcie_intr_info); else fat = t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE, t5_pcie_intr_info); if (fat) t4_fatal_err(adapter); } /* * TP interrupt handler. */ static void tp_intr_handler(struct adapter *adapter) { static const struct intr_info tp_intr_info[] = { { 0x3fffffff, "TP parity error", -1, 1 }, { F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adapter, A_TP_INT_CAUSE, tp_intr_info)) t4_fatal_err(adapter); } /* * SGE interrupt handler. */ static void sge_intr_handler(struct adapter *adapter) { u64 v; u32 err; static const struct intr_info sge_intr_info[] = { { F_ERR_CPL_EXCEED_IQE_SIZE, "SGE received CPL exceeding IQE size", -1, 1 }, { F_ERR_INVALID_CIDX_INC, "SGE GTS CIDX increment too large", -1, 0 }, { F_ERR_CPL_OPCODE_0, "SGE received 0-length CPL", -1, 0 }, { F_DBFIFO_LP_INT, NULL, -1, 0, t4_db_full }, { F_ERR_DATA_CPL_ON_HIGH_QID1 | F_ERR_DATA_CPL_ON_HIGH_QID0, "SGE IQID > 1023 received CPL for FL", -1, 0 }, { F_ERR_BAD_DB_PIDX3, "SGE DBP 3 pidx increment too large", -1, 0 }, { F_ERR_BAD_DB_PIDX2, "SGE DBP 2 pidx increment too large", -1, 0 }, { F_ERR_BAD_DB_PIDX1, "SGE DBP 1 pidx increment too large", -1, 0 }, { F_ERR_BAD_DB_PIDX0, "SGE DBP 0 pidx increment too large", -1, 0 }, { F_ERR_ING_CTXT_PRIO, "SGE too many priority ingress contexts", -1, 0 }, { F_INGRESS_SIZE_ERR, "SGE illegal ingress QID", -1, 0 }, { F_EGRESS_SIZE_ERR, "SGE illegal egress QID", -1, 0 }, + { F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1 | + F_ERR_PCIE_ERROR2 | F_ERR_PCIE_ERROR3, + "SGE PCIe error for a DBP thread", -1, 0 }, { 0 } }; static const struct intr_info t4t5_sge_intr_info[] = { { F_ERR_DROPPED_DB, NULL, -1, 0, t4_db_dropped }, { F_DBFIFO_HP_INT, NULL, -1, 0, t4_db_full }, { F_ERR_EGR_CTXT_PRIO, "SGE too many priority egress contexts", -1, 0 }, { 0 } }; /* * For now, treat below interrupts as fatal so that we disable SGE and * get better debug */ static const struct intr_info t6_sge_intr_info[] = { - { F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1, - "SGE PCIe error for a DBP thread", -1, 1 }, { F_FATAL_WRE_LEN, "SGE Actual WRE packet is less than advertized length", -1, 1 }, { 0 } }; v = (u64)t4_read_reg(adapter, A_SGE_INT_CAUSE1) | ((u64)t4_read_reg(adapter, A_SGE_INT_CAUSE2) << 32); if (v) { CH_ALERT(adapter, "SGE parity error (%#llx)\n", (unsigned long long)v); t4_write_reg(adapter, A_SGE_INT_CAUSE1, v); t4_write_reg(adapter, A_SGE_INT_CAUSE2, v >> 32); } v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3, sge_intr_info); if (chip_id(adapter) <= CHELSIO_T5) v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3, t4t5_sge_intr_info); else v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3, t6_sge_intr_info); err = t4_read_reg(adapter, A_SGE_ERROR_STATS); if (err & F_ERROR_QID_VALID) { CH_ERR(adapter, "SGE error for queue %u\n", G_ERROR_QID(err)); if (err & F_UNCAPTURED_ERROR) CH_ERR(adapter, "SGE UNCAPTURED_ERROR set (clearing)\n"); t4_write_reg(adapter, A_SGE_ERROR_STATS, F_ERROR_QID_VALID | F_UNCAPTURED_ERROR); } if (v != 0) t4_fatal_err(adapter); } #define CIM_OBQ_INTR (F_OBQULP0PARERR | F_OBQULP1PARERR | F_OBQULP2PARERR |\ F_OBQULP3PARERR | F_OBQSGEPARERR | F_OBQNCSIPARERR) #define CIM_IBQ_INTR (F_IBQTP0PARERR | F_IBQTP1PARERR | F_IBQULPPARERR |\ F_IBQSGEHIPARERR | F_IBQSGELOPARERR | F_IBQNCSIPARERR) /* * CIM interrupt handler. */ static void cim_intr_handler(struct adapter *adapter) { static const struct intr_info cim_intr_info[] = { { F_PREFDROPINT, "CIM control register prefetch drop", -1, 1 }, { CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 }, { CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 }, { F_MBUPPARERR, "CIM mailbox uP parity error", -1, 1 }, { F_MBHOSTPARERR, "CIM mailbox host parity error", -1, 1 }, { F_TIEQINPARERRINT, "CIM TIEQ outgoing parity error", -1, 1 }, { F_TIEQOUTPARERRINT, "CIM TIEQ incoming parity error", -1, 1 }, + { F_TIMER0INT, "CIM TIMER0 interrupt", -1, 1 }, { 0 } }; static const struct intr_info cim_upintr_info[] = { { F_RSVDSPACEINT, "CIM reserved space access", -1, 1 }, { F_ILLTRANSINT, "CIM illegal transaction", -1, 1 }, { F_ILLWRINT, "CIM illegal write", -1, 1 }, { F_ILLRDINT, "CIM illegal read", -1, 1 }, { F_ILLRDBEINT, "CIM illegal read BE", -1, 1 }, { F_ILLWRBEINT, "CIM illegal write BE", -1, 1 }, { F_SGLRDBOOTINT, "CIM single read from boot space", -1, 1 }, { F_SGLWRBOOTINT, "CIM single write to boot space", -1, 1 }, { F_BLKWRBOOTINT, "CIM block write to boot space", -1, 1 }, { F_SGLRDFLASHINT, "CIM single read from flash space", -1, 1 }, { F_SGLWRFLASHINT, "CIM single write to flash space", -1, 1 }, { F_BLKWRFLASHINT, "CIM block write to flash space", -1, 1 }, { F_SGLRDEEPROMINT, "CIM single EEPROM read", -1, 1 }, { F_SGLWREEPROMINT, "CIM single EEPROM write", -1, 1 }, { F_BLKRDEEPROMINT, "CIM block EEPROM read", -1, 1 }, { F_BLKWREEPROMINT, "CIM block EEPROM write", -1, 1 }, { F_SGLRDCTLINT , "CIM single read from CTL space", -1, 1 }, { F_SGLWRCTLINT , "CIM single write to CTL space", -1, 1 }, { F_BLKRDCTLINT , "CIM block read from CTL space", -1, 1 }, { F_BLKWRCTLINT , "CIM block write to CTL space", -1, 1 }, { F_SGLRDPLINT , "CIM single read from PL space", -1, 1 }, { F_SGLWRPLINT , "CIM single write to PL space", -1, 1 }, { F_BLKRDPLINT , "CIM block read from PL space", -1, 1 }, { F_BLKWRPLINT , "CIM block write to PL space", -1, 1 }, { F_REQOVRLOOKUPINT , "CIM request FIFO overwrite", -1, 1 }, { F_RSPOVRLOOKUPINT , "CIM response FIFO overwrite", -1, 1 }, { F_TIMEOUTINT , "CIM PIF timeout", -1, 1 }, { F_TIMEOUTMAINT , "CIM PIF MA timeout", -1, 1 }, { 0 } }; + u32 val, fw_err; int fat; - if (t4_read_reg(adapter, A_PCIE_FW) & F_PCIE_FW_ERR) + fw_err = t4_read_reg(adapter, A_PCIE_FW); + if (fw_err & F_PCIE_FW_ERR) t4_report_fw_error(adapter); + /* When the Firmware detects an internal error which normally wouldn't + * raise a Host Interrupt, it forces a CIM Timer0 interrupt in order + * to make sure the Host sees the Firmware Crash. So if we have a + * Timer0 interrupt and don't see a Firmware Crash, ignore the Timer0 + * interrupt. + */ + val = t4_read_reg(adapter, A_CIM_HOST_INT_CAUSE); + if (val & F_TIMER0INT) + if (!(fw_err & F_PCIE_FW_ERR) || + (G_PCIE_FW_EVAL(fw_err) != PCIE_FW_EVAL_CRASH)) + t4_write_reg(adapter, A_CIM_HOST_INT_CAUSE, + F_TIMER0INT); + fat = t4_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, cim_intr_info) + t4_handle_intr_status(adapter, A_CIM_HOST_UPACC_INT_CAUSE, cim_upintr_info); if (fat) t4_fatal_err(adapter); } /* * ULP RX interrupt handler. */ static void ulprx_intr_handler(struct adapter *adapter) { static const struct intr_info ulprx_intr_info[] = { { F_CAUSE_CTX_1, "ULPRX channel 1 context error", -1, 1 }, { F_CAUSE_CTX_0, "ULPRX channel 0 context error", -1, 1 }, { 0x7fffff, "ULPRX parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adapter, A_ULP_RX_INT_CAUSE, ulprx_intr_info)) t4_fatal_err(adapter); } /* * ULP TX interrupt handler. */ static void ulptx_intr_handler(struct adapter *adapter) { static const struct intr_info ulptx_intr_info[] = { { F_PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds", -1, 0 }, { F_PBL_BOUND_ERR_CH2, "ULPTX channel 2 PBL out of bounds", -1, 0 }, { F_PBL_BOUND_ERR_CH1, "ULPTX channel 1 PBL out of bounds", -1, 0 }, { F_PBL_BOUND_ERR_CH0, "ULPTX channel 0 PBL out of bounds", -1, 0 }, { 0xfffffff, "ULPTX parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adapter, A_ULP_TX_INT_CAUSE, ulptx_intr_info)) t4_fatal_err(adapter); } /* * PM TX interrupt handler. */ static void pmtx_intr_handler(struct adapter *adapter) { static const struct intr_info pmtx_intr_info[] = { { F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large", -1, 1 }, { F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large", -1, 1 }, { F_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large", -1, 1 }, { F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1 }, { 0xffffff0, "PMTX framing error", -1, 1 }, { F_OESPI_PAR_ERROR, "PMTX oespi parity error", -1, 1 }, { F_DB_OPTIONS_PAR_ERROR, "PMTX db_options parity error", -1, 1 }, { F_ICSPI_PAR_ERROR, "PMTX icspi parity error", -1, 1 }, { F_C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error", -1, 1}, { 0 } }; if (t4_handle_intr_status(adapter, A_PM_TX_INT_CAUSE, pmtx_intr_info)) t4_fatal_err(adapter); } /* * PM RX interrupt handler. */ static void pmrx_intr_handler(struct adapter *adapter) { static const struct intr_info pmrx_intr_info[] = { { F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1 }, { 0x3ffff0, "PMRX framing error", -1, 1 }, { F_OCSPI_PAR_ERROR, "PMRX ocspi parity error", -1, 1 }, { F_DB_OPTIONS_PAR_ERROR, "PMRX db_options parity error", -1, 1 }, { F_IESPI_PAR_ERROR, "PMRX iespi parity error", -1, 1 }, { F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error", -1, 1}, { 0 } }; if (t4_handle_intr_status(adapter, A_PM_RX_INT_CAUSE, pmrx_intr_info)) t4_fatal_err(adapter); } /* * CPL switch interrupt handler. */ static void cplsw_intr_handler(struct adapter *adapter) { static const struct intr_info cplsw_intr_info[] = { { F_CIM_OP_MAP_PERR, "CPLSW CIM op_map parity error", -1, 1 }, { F_CIM_OVFL_ERROR, "CPLSW CIM overflow", -1, 1 }, { F_TP_FRAMING_ERROR, "CPLSW TP framing error", -1, 1 }, { F_SGE_FRAMING_ERROR, "CPLSW SGE framing error", -1, 1 }, { F_CIM_FRAMING_ERROR, "CPLSW CIM framing error", -1, 1 }, { F_ZERO_SWITCH_ERROR, "CPLSW no-switch error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adapter, A_CPL_INTR_CAUSE, cplsw_intr_info)) t4_fatal_err(adapter); } /* * LE interrupt handler. */ static void le_intr_handler(struct adapter *adap) { unsigned int chip_ver = chip_id(adap); static const struct intr_info le_intr_info[] = { { F_LIPMISS, "LE LIP miss", -1, 0 }, { F_LIP0, "LE 0 LIP error", -1, 0 }, { F_PARITYERR, "LE parity error", -1, 1 }, { F_UNKNOWNCMD, "LE unknown command", -1, 1 }, { F_REQQPARERR, "LE request queue parity error", -1, 1 }, { 0 } }; static const struct intr_info t6_le_intr_info[] = { { F_T6_LIPMISS, "LE LIP miss", -1, 0 }, { F_T6_LIP0, "LE 0 LIP error", -1, 0 }, { F_TCAMINTPERR, "LE parity error", -1, 1 }, { F_T6_UNKNOWNCMD, "LE unknown command", -1, 1 }, { F_SSRAMINTPERR, "LE request queue parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adap, A_LE_DB_INT_CAUSE, (chip_ver <= CHELSIO_T5) ? le_intr_info : t6_le_intr_info)) t4_fatal_err(adap); } /* * MPS interrupt handler. */ static void mps_intr_handler(struct adapter *adapter) { static const struct intr_info mps_rx_intr_info[] = { { 0xffffff, "MPS Rx parity error", -1, 1 }, { 0 } }; static const struct intr_info mps_tx_intr_info[] = { { V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error", -1, 1 }, { F_NCSIFIFO, "MPS Tx NC-SI FIFO parity error", -1, 1 }, { V_TXDATAFIFO(M_TXDATAFIFO), "MPS Tx data FIFO parity error", -1, 1 }, { V_TXDESCFIFO(M_TXDESCFIFO), "MPS Tx desc FIFO parity error", -1, 1 }, { F_BUBBLE, "MPS Tx underflow", -1, 1 }, { F_SECNTERR, "MPS Tx SOP/EOP error", -1, 1 }, { F_FRMERR, "MPS Tx framing error", -1, 1 }, { 0 } }; static const struct intr_info mps_trc_intr_info[] = { { V_FILTMEM(M_FILTMEM), "MPS TRC filter parity error", -1, 1 }, { V_PKTFIFO(M_PKTFIFO), "MPS TRC packet FIFO parity error", -1, 1 }, { F_MISCPERR, "MPS TRC misc parity error", -1, 1 }, { 0 } }; static const struct intr_info mps_stat_sram_intr_info[] = { { 0x1fffff, "MPS statistics SRAM parity error", -1, 1 }, { 0 } }; static const struct intr_info mps_stat_tx_intr_info[] = { { 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 }, { 0 } }; static const struct intr_info mps_stat_rx_intr_info[] = { { 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 }, { 0 } }; static const struct intr_info mps_cls_intr_info[] = { { F_MATCHSRAM, "MPS match SRAM parity error", -1, 1 }, { F_MATCHTCAM, "MPS match TCAM parity error", -1, 1 }, { F_HASHSRAM, "MPS hash SRAM parity error", -1, 1 }, { 0 } }; int fat; fat = t4_handle_intr_status(adapter, A_MPS_RX_PERR_INT_CAUSE, mps_rx_intr_info) + t4_handle_intr_status(adapter, A_MPS_TX_INT_CAUSE, mps_tx_intr_info) + t4_handle_intr_status(adapter, A_MPS_TRC_INT_CAUSE, mps_trc_intr_info) + t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_SRAM, mps_stat_sram_intr_info) + t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO, mps_stat_tx_intr_info) + t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_RX_FIFO, mps_stat_rx_intr_info) + t4_handle_intr_status(adapter, A_MPS_CLS_INT_CAUSE, mps_cls_intr_info); t4_write_reg(adapter, A_MPS_INT_CAUSE, 0); t4_read_reg(adapter, A_MPS_INT_CAUSE); /* flush */ if (fat) t4_fatal_err(adapter); } #define MEM_INT_MASK (F_PERR_INT_CAUSE | F_ECC_CE_INT_CAUSE | \ F_ECC_UE_INT_CAUSE) /* * EDC/MC interrupt handler. */ static void mem_intr_handler(struct adapter *adapter, int idx) { static const char name[4][7] = { "EDC0", "EDC1", "MC/MC0", "MC1" }; unsigned int addr, cnt_addr, v; if (idx <= MEM_EDC1) { addr = EDC_REG(A_EDC_INT_CAUSE, idx); cnt_addr = EDC_REG(A_EDC_ECC_STATUS, idx); } else if (idx == MEM_MC) { if (is_t4(adapter)) { addr = A_MC_INT_CAUSE; cnt_addr = A_MC_ECC_STATUS; } else { addr = A_MC_P_INT_CAUSE; cnt_addr = A_MC_P_ECC_STATUS; } } else { addr = MC_REG(A_MC_P_INT_CAUSE, 1); cnt_addr = MC_REG(A_MC_P_ECC_STATUS, 1); } v = t4_read_reg(adapter, addr) & MEM_INT_MASK; if (v & F_PERR_INT_CAUSE) CH_ALERT(adapter, "%s FIFO parity error\n", name[idx]); if (v & F_ECC_CE_INT_CAUSE) { u32 cnt = G_ECC_CECNT(t4_read_reg(adapter, cnt_addr)); if (idx <= MEM_EDC1) t4_edc_err_read(adapter, idx); t4_write_reg(adapter, cnt_addr, V_ECC_CECNT(M_ECC_CECNT)); CH_WARN_RATELIMIT(adapter, "%u %s correctable ECC data error%s\n", cnt, name[idx], cnt > 1 ? "s" : ""); } if (v & F_ECC_UE_INT_CAUSE) CH_ALERT(adapter, "%s uncorrectable ECC data error\n", name[idx]); t4_write_reg(adapter, addr, v); if (v & (F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE)) t4_fatal_err(adapter); } /* * MA interrupt handler. */ static void ma_intr_handler(struct adapter *adapter) { u32 v, status = t4_read_reg(adapter, A_MA_INT_CAUSE); if (status & F_MEM_PERR_INT_CAUSE) { CH_ALERT(adapter, "MA parity error, parity status %#x\n", t4_read_reg(adapter, A_MA_PARITY_ERROR_STATUS1)); if (is_t5(adapter)) CH_ALERT(adapter, "MA parity error, parity status %#x\n", t4_read_reg(adapter, A_MA_PARITY_ERROR_STATUS2)); } if (status & F_MEM_WRAP_INT_CAUSE) { v = t4_read_reg(adapter, A_MA_INT_WRAP_STATUS); CH_ALERT(adapter, "MA address wrap-around error by " "client %u to address %#x\n", G_MEM_WRAP_CLIENT_NUM(v), G_MEM_WRAP_ADDRESS(v) << 4); } t4_write_reg(adapter, A_MA_INT_CAUSE, status); t4_fatal_err(adapter); } /* * SMB interrupt handler. */ static void smb_intr_handler(struct adapter *adap) { static const struct intr_info smb_intr_info[] = { { F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error", -1, 1 }, { F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error", -1, 1 }, { F_SLVFIFOPARINT, "SMB slave FIFO parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adap, A_SMB_INT_CAUSE, smb_intr_info)) t4_fatal_err(adap); } /* * NC-SI interrupt handler. */ static void ncsi_intr_handler(struct adapter *adap) { static const struct intr_info ncsi_intr_info[] = { { F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error", -1, 1 }, { F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error", -1, 1 }, { F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error", -1, 1 }, { F_RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adap, A_NCSI_INT_CAUSE, ncsi_intr_info)) t4_fatal_err(adap); } /* * XGMAC interrupt handler. */ static void xgmac_intr_handler(struct adapter *adap, int port) { u32 v, int_cause_reg; if (is_t4(adap)) int_cause_reg = PORT_REG(port, A_XGMAC_PORT_INT_CAUSE); else int_cause_reg = T5_PORT_REG(port, A_MAC_PORT_INT_CAUSE); v = t4_read_reg(adap, int_cause_reg); v &= (F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR); if (!v) return; if (v & F_TXFIFO_PRTY_ERR) CH_ALERT(adap, "XGMAC %d Tx FIFO parity error\n", port); if (v & F_RXFIFO_PRTY_ERR) CH_ALERT(adap, "XGMAC %d Rx FIFO parity error\n", port); t4_write_reg(adap, int_cause_reg, v); t4_fatal_err(adap); } /* * PL interrupt handler. */ static void pl_intr_handler(struct adapter *adap) { static const struct intr_info pl_intr_info[] = { { F_FATALPERR, "Fatal parity error", -1, 1 }, { F_PERRVFID, "PL VFID_MAP parity error", -1, 1 }, { 0 } }; static const struct intr_info t5_pl_intr_info[] = { { F_FATALPERR, "Fatal parity error", -1, 1 }, { 0 } }; if (t4_handle_intr_status(adap, A_PL_PL_INT_CAUSE, is_t4(adap) ? pl_intr_info : t5_pl_intr_info)) t4_fatal_err(adap); } #define PF_INTR_MASK (F_PFSW | F_PFCIM) /** * t4_slow_intr_handler - control path interrupt handler * @adapter: the adapter * * T4 interrupt handler for non-data global interrupt events, e.g., errors. * The designation 'slow' is because it involves register reads, while * data interrupts typically don't involve any MMIOs. */ int t4_slow_intr_handler(struct adapter *adapter) { u32 cause = t4_read_reg(adapter, A_PL_INT_CAUSE); if (!(cause & GLBL_INTR_MASK)) return 0; if (cause & F_CIM) cim_intr_handler(adapter); if (cause & F_MPS) mps_intr_handler(adapter); if (cause & F_NCSI) ncsi_intr_handler(adapter); if (cause & F_PL) pl_intr_handler(adapter); if (cause & F_SMB) smb_intr_handler(adapter); if (cause & F_MAC0) xgmac_intr_handler(adapter, 0); if (cause & F_MAC1) xgmac_intr_handler(adapter, 1); if (cause & F_MAC2) xgmac_intr_handler(adapter, 2); if (cause & F_MAC3) xgmac_intr_handler(adapter, 3); if (cause & F_PCIE) pcie_intr_handler(adapter); if (cause & F_MC0) mem_intr_handler(adapter, MEM_MC); if (is_t5(adapter) && (cause & F_MC1)) mem_intr_handler(adapter, MEM_MC1); if (cause & F_EDC0) mem_intr_handler(adapter, MEM_EDC0); if (cause & F_EDC1) mem_intr_handler(adapter, MEM_EDC1); if (cause & F_LE) le_intr_handler(adapter); if (cause & F_TP) tp_intr_handler(adapter); if (cause & F_MA) ma_intr_handler(adapter); if (cause & F_PM_TX) pmtx_intr_handler(adapter); if (cause & F_PM_RX) pmrx_intr_handler(adapter); if (cause & F_ULP_RX) ulprx_intr_handler(adapter); if (cause & F_CPL_SWITCH) cplsw_intr_handler(adapter); if (cause & F_SGE) sge_intr_handler(adapter); if (cause & F_ULP_TX) ulptx_intr_handler(adapter); /* Clear the interrupts just processed for which we are the master. */ t4_write_reg(adapter, A_PL_INT_CAUSE, cause & GLBL_INTR_MASK); (void)t4_read_reg(adapter, A_PL_INT_CAUSE); /* flush */ return 1; } /** * t4_intr_enable - enable interrupts * @adapter: the adapter whose interrupts should be enabled * * Enable PF-specific interrupts for the calling function and the top-level * interrupt concentrator for global interrupts. Interrupts are already * enabled at each module, here we just enable the roots of the interrupt * hierarchies. * * Note: this function should be called only when the driver manages * non PF-specific interrupts from the various HW modules. Only one PCI * function at a time should be doing this. */ void t4_intr_enable(struct adapter *adapter) { u32 val = 0; u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); u32 pf = (chip_id(adapter) <= CHELSIO_T5 ? G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami)); if (chip_id(adapter) <= CHELSIO_T5) val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT; else val = F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1 | F_FATAL_WRE_LEN; t4_write_reg(adapter, A_SGE_INT_ENABLE3, F_ERR_CPL_EXCEED_IQE_SIZE | F_ERR_INVALID_CIDX_INC | F_ERR_CPL_OPCODE_0 | F_ERR_DATA_CPL_ON_HIGH_QID1 | F_INGRESS_SIZE_ERR | F_ERR_DATA_CPL_ON_HIGH_QID0 | F_ERR_BAD_DB_PIDX3 | F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 | F_ERR_BAD_DB_PIDX0 | F_ERR_ING_CTXT_PRIO | F_DBFIFO_LP_INT | F_EGRESS_SIZE_ERR | val); t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), PF_INTR_MASK); t4_set_reg_field(adapter, A_PL_INT_MAP0, 0, 1 << pf); } /** * t4_intr_disable - disable interrupts * @adapter: the adapter whose interrupts should be disabled * * Disable interrupts. We only disable the top-level interrupt * concentrators. The caller must be a PCI function managing global * interrupts. */ void t4_intr_disable(struct adapter *adapter) { u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); u32 pf = (chip_id(adapter) <= CHELSIO_T5 ? G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami)); t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), 0); t4_set_reg_field(adapter, A_PL_INT_MAP0, 1 << pf, 0); } /** * t4_intr_clear - clear all interrupts * @adapter: the adapter whose interrupts should be cleared * * Clears all interrupts. The caller must be a PCI function managing * global interrupts. */ void t4_intr_clear(struct adapter *adapter) { static const unsigned int cause_reg[] = { A_SGE_INT_CAUSE1, A_SGE_INT_CAUSE2, A_SGE_INT_CAUSE3, A_PCIE_NONFAT_ERR, A_PCIE_INT_CAUSE, A_MA_INT_WRAP_STATUS, A_MA_PARITY_ERROR_STATUS1, A_MA_INT_CAUSE, A_EDC_INT_CAUSE, EDC_REG(A_EDC_INT_CAUSE, 1), A_CIM_HOST_INT_CAUSE, A_CIM_HOST_UPACC_INT_CAUSE, MYPF_REG(A_CIM_PF_HOST_INT_CAUSE), A_TP_INT_CAUSE, A_ULP_RX_INT_CAUSE, A_ULP_TX_INT_CAUSE, A_PM_RX_INT_CAUSE, A_PM_TX_INT_CAUSE, A_MPS_RX_PERR_INT_CAUSE, A_CPL_INTR_CAUSE, MYPF_REG(A_PL_PF_INT_CAUSE), A_PL_PL_INT_CAUSE, A_LE_DB_INT_CAUSE, }; unsigned int i; for (i = 0; i < ARRAY_SIZE(cause_reg); ++i) t4_write_reg(adapter, cause_reg[i], 0xffffffff); t4_write_reg(adapter, is_t4(adapter) ? A_MC_INT_CAUSE : A_MC_P_INT_CAUSE, 0xffffffff); if (is_t4(adapter)) { t4_write_reg(adapter, A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS, 0xffffffff); t4_write_reg(adapter, A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS, 0xffffffff); } else t4_write_reg(adapter, A_MA_PARITY_ERROR_STATUS2, 0xffffffff); t4_write_reg(adapter, A_PL_INT_CAUSE, GLBL_INTR_MASK); (void) t4_read_reg(adapter, A_PL_INT_CAUSE); /* flush */ } /** * hash_mac_addr - return the hash value of a MAC address * @addr: the 48-bit Ethernet MAC address * * Hashes a MAC address according to the hash function used by HW inexact * (hash) address matching. */ static int hash_mac_addr(const u8 *addr) { u32 a = ((u32)addr[0] << 16) | ((u32)addr[1] << 8) | addr[2]; u32 b = ((u32)addr[3] << 16) | ((u32)addr[4] << 8) | addr[5]; a ^= b; a ^= (a >> 12); a ^= (a >> 6); return a & 0x3f; } /** * t4_config_rss_range - configure a portion of the RSS mapping table * @adapter: the adapter * @mbox: mbox to use for the FW command * @viid: virtual interface whose RSS subtable is to be written * @start: start entry in the table to write * @n: how many table entries to write * @rspq: values for the "response queue" (Ingress Queue) lookup table * @nrspq: number of values in @rspq * * Programs the selected part of the VI's RSS mapping table with the * provided values. If @nrspq < @n the supplied values are used repeatedly * until the full table range is populated. * * The caller must ensure the values in @rspq are in the range allowed for * @viid. */ int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid, int start, int n, const u16 *rspq, unsigned int nrspq) { int ret; const u16 *rsp = rspq; const u16 *rsp_end = rspq + nrspq; struct fw_rss_ind_tbl_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_IND_TBL_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_RSS_IND_TBL_CMD_VIID(viid)); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); /* * Each firmware RSS command can accommodate up to 32 RSS Ingress * Queue Identifiers. These Ingress Queue IDs are packed three to * a 32-bit word as 10-bit values with the upper remaining 2 bits * reserved. */ while (n > 0) { int nq = min(n, 32); int nq_packed = 0; __be32 *qp = &cmd.iq0_to_iq2; /* * Set up the firmware RSS command header to send the next * "nq" Ingress Queue IDs to the firmware. */ cmd.niqid = cpu_to_be16(nq); cmd.startidx = cpu_to_be16(start); /* * "nq" more done for the start of the next loop. */ start += nq; n -= nq; /* * While there are still Ingress Queue IDs to stuff into the * current firmware RSS command, retrieve them from the * Ingress Queue ID array and insert them into the command. */ while (nq > 0) { /* * Grab up to the next 3 Ingress Queue IDs (wrapping * around the Ingress Queue ID array if necessary) and * insert them into the firmware RSS command at the * current 3-tuple position within the commad. */ u16 qbuf[3]; u16 *qbp = qbuf; int nqbuf = min(3, nq); nq -= nqbuf; qbuf[0] = qbuf[1] = qbuf[2] = 0; while (nqbuf && nq_packed < 32) { nqbuf--; nq_packed++; *qbp++ = *rsp++; if (rsp >= rsp_end) rsp = rspq; } *qp++ = cpu_to_be32(V_FW_RSS_IND_TBL_CMD_IQ0(qbuf[0]) | V_FW_RSS_IND_TBL_CMD_IQ1(qbuf[1]) | V_FW_RSS_IND_TBL_CMD_IQ2(qbuf[2])); } /* * Send this portion of the RRS table update to the firmware; * bail out on any errors. */ ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL); if (ret) return ret; } return 0; } /** * t4_config_glbl_rss - configure the global RSS mode * @adapter: the adapter * @mbox: mbox to use for the FW command * @mode: global RSS mode * @flags: mode-specific flags * * Sets the global RSS mode. */ int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode, unsigned int flags) { struct fw_rss_glb_config_cmd c; memset(&c, 0, sizeof(c)); c.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_RSS_GLB_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) { c.u.manual.mode_pkd = cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode)); } else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { c.u.basicvirtual.mode_keymode = cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode)); c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags); } else return -EINVAL; return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL); } /** * t4_config_vi_rss - configure per VI RSS settings * @adapter: the adapter * @mbox: mbox to use for the FW command * @viid: the VI id * @flags: RSS flags * @defq: id of the default RSS queue for the VI. * @skeyidx: RSS secret key table index for non-global mode * @skey: RSS vf_scramble key for VI. * * Configures VI-specific RSS properties. */ int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid, unsigned int flags, unsigned int defq, unsigned int skeyidx, unsigned int skey) { struct fw_rss_vi_config_cmd c; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_VI_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_RSS_VI_CONFIG_CMD_VIID(viid)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags | V_FW_RSS_VI_CONFIG_CMD_DEFAULTQ(defq)); c.u.basicvirtual.secretkeyidx_pkd = cpu_to_be32( V_FW_RSS_VI_CONFIG_CMD_SECRETKEYIDX(skeyidx)); c.u.basicvirtual.secretkeyxor = cpu_to_be32(skey); return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL); } /* Read an RSS table row */ static int rd_rss_row(struct adapter *adap, int row, u32 *val) { t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row); return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE, F_LKPTBLROWVLD, 1, 5, 0, val); } /** * t4_read_rss - read the contents of the RSS mapping table * @adapter: the adapter * @map: holds the contents of the RSS mapping table * * Reads the contents of the RSS hash->queue mapping table. */ int t4_read_rss(struct adapter *adapter, u16 *map) { u32 val; int i, ret; for (i = 0; i < RSS_NENTRIES / 2; ++i) { ret = rd_rss_row(adapter, i, &val); if (ret) return ret; *map++ = G_LKPTBLQUEUE0(val); *map++ = G_LKPTBLQUEUE1(val); } return 0; } /** - * t4_fw_tp_pio_rw - Access TP PIO through LDST - * @adap: the adapter - * @vals: where the indirect register values are stored/written - * @nregs: how many indirect registers to read/write - * @start_idx: index of first indirect register to read/write - * @rw: Read (1) or Write (0) + * t4_tp_fw_ldst_rw - Access TP indirect register through LDST + * @adap: the adapter + * @cmd: TP fw ldst address space type + * @vals: where the indirect register values are stored/written + * @nregs: how many indirect registers to read/write + * @start_idx: index of first indirect register to read/write + * @rw: Read (1) or Write (0) + * @sleep_ok: if true we may sleep while awaiting command completion * - * Access TP PIO registers through LDST - */ -void t4_fw_tp_pio_rw(struct adapter *adap, u32 *vals, unsigned int nregs, - unsigned int start_index, unsigned int rw) + * Access TP indirect registers through LDST + **/ +static int t4_tp_fw_ldst_rw(struct adapter *adap, int cmd, u32 *vals, + unsigned int nregs, unsigned int start_index, + unsigned int rw, bool sleep_ok) { - int ret, i; - int cmd = FW_LDST_ADDRSPC_TP_PIO; + int ret = 0; + unsigned int i; struct fw_ldst_cmd c; - for (i = 0 ; i < nregs; i++) { + for (i = 0; i < nregs; i++) { memset(&c, 0, sizeof(c)); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | (rw ? F_FW_CMD_READ : - F_FW_CMD_WRITE) | + F_FW_CMD_WRITE) | V_FW_LDST_CMD_ADDRSPACE(cmd)); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.addrval.addr = cpu_to_be32(start_index + i); c.u.addrval.val = rw ? 0 : cpu_to_be32(vals[i]); - ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); - if (ret == 0) { - if (rw) - vals[i] = be32_to_cpu(c.u.addrval.val); - } + ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, + sleep_ok); + if (ret) + return ret; + + if (rw) + vals[i] = be32_to_cpu(c.u.addrval.val); } + return 0; } /** + * t4_tp_indirect_rw - Read/Write TP indirect register through LDST or backdoor + * @adap: the adapter + * @reg_addr: Address Register + * @reg_data: Data register + * @buff: where the indirect register values are stored/written + * @nregs: how many indirect registers to read/write + * @start_index: index of first indirect register to read/write + * @rw: READ(1) or WRITE(0) + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read/Write TP indirect registers through LDST if possible. + * Else, use backdoor access + **/ +static void t4_tp_indirect_rw(struct adapter *adap, u32 reg_addr, u32 reg_data, + u32 *buff, u32 nregs, u32 start_index, int rw, + bool sleep_ok) +{ + int rc = -EINVAL; + int cmd; + + switch (reg_addr) { + case A_TP_PIO_ADDR: + cmd = FW_LDST_ADDRSPC_TP_PIO; + break; + case A_TP_TM_PIO_ADDR: + cmd = FW_LDST_ADDRSPC_TP_TM_PIO; + break; + case A_TP_MIB_INDEX: + cmd = FW_LDST_ADDRSPC_TP_MIB; + break; + default: + goto indirect_access; + } + + if (t4_use_ldst(adap)) + rc = t4_tp_fw_ldst_rw(adap, cmd, buff, nregs, start_index, rw, + sleep_ok); + +indirect_access: + + if (rc) { + if (rw) + t4_read_indirect(adap, reg_addr, reg_data, buff, nregs, + start_index); + else + t4_write_indirect(adap, reg_addr, reg_data, buff, nregs, + start_index); + } +} + +/** + * t4_tp_pio_read - Read TP PIO registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP PIO Registers + **/ +void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, buff, nregs, + start_index, 1, sleep_ok); +} + +/** + * t4_tp_pio_write - Write TP PIO registers + * @adap: the adapter + * @buff: where the indirect register values are stored + * @nregs: how many indirect registers to write + * @start_index: index of first indirect register to write + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Write TP PIO Registers + **/ +void t4_tp_pio_write(struct adapter *adap, const u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, + __DECONST(u32 *, buff), nregs, start_index, 0, sleep_ok); +} + +/** + * t4_tp_tm_pio_read - Read TP TM PIO registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP TM PIO Registers + **/ +void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, A_TP_TM_PIO_ADDR, A_TP_TM_PIO_DATA, buff, + nregs, start_index, 1, sleep_ok); +} + +/** + * t4_tp_mib_read - Read TP MIB registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP MIB Registers + **/ +void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index, + bool sleep_ok) +{ + t4_tp_indirect_rw(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, buff, nregs, + start_index, 1, sleep_ok); +} + +/** * t4_read_rss_key - read the global RSS key * @adap: the adapter * @key: 10-entry array holding the 320-bit RSS key + * @sleep_ok: if true we may sleep while awaiting command completion * * Reads the global 320-bit RSS key. */ -void t4_read_rss_key(struct adapter *adap, u32 *key) +void t4_read_rss_key(struct adapter *adap, u32 *key, bool sleep_ok) { - if (t4_use_ldst(adap)) - t4_fw_tp_pio_rw(adap, key, 10, A_TP_RSS_SECRET_KEY0, 1); - else - t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10, - A_TP_RSS_SECRET_KEY0); + t4_tp_pio_read(adap, key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok); } /** * t4_write_rss_key - program one of the RSS keys * @adap: the adapter * @key: 10-entry array holding the 320-bit RSS key * @idx: which RSS key to write + * @sleep_ok: if true we may sleep while awaiting command completion * * Writes one of the RSS keys with the given 320-bit value. If @idx is * 0..15 the corresponding entry in the RSS key table is written, * otherwise the global RSS key is written. */ -void t4_write_rss_key(struct adapter *adap, u32 *key, int idx) +void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx, + bool sleep_ok) { u8 rss_key_addr_cnt = 16; u32 vrt = t4_read_reg(adap, A_TP_RSS_CONFIG_VRT); /* * T6 and later: for KeyMode 3 (per-vf and per-vf scramble), * allows access to key addresses 16-63 by using KeyWrAddrX * as index[5:4](upper 2) into key table */ if ((chip_id(adap) > CHELSIO_T5) && (vrt & F_KEYEXTEND) && (G_KEYMODE(vrt) == 3)) rss_key_addr_cnt = 32; - if (t4_use_ldst(adap)) - t4_fw_tp_pio_rw(adap, key, 10, A_TP_RSS_SECRET_KEY0, 0); - else - t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10, - A_TP_RSS_SECRET_KEY0); + t4_tp_pio_write(adap, key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok); if (idx >= 0 && idx < rss_key_addr_cnt) { if (rss_key_addr_cnt > 16) t4_write_reg(adap, A_TP_RSS_CONFIG_VRT, vrt | V_KEYWRADDRX(idx >> 4) | V_T6_VFWRADDR(idx) | F_KEYWREN); else t4_write_reg(adap, A_TP_RSS_CONFIG_VRT, vrt| V_KEYWRADDR(idx) | F_KEYWREN); } } /** * t4_read_rss_pf_config - read PF RSS Configuration Table * @adapter: the adapter * @index: the entry in the PF RSS table to read * @valp: where to store the returned value + * @sleep_ok: if true we may sleep while awaiting command completion * * Reads the PF RSS Configuration Table at the specified index and returns * the value found there. */ void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, - u32 *valp) + u32 *valp, bool sleep_ok) { - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, valp, 1, - A_TP_RSS_PF0_CONFIG + index, 1); - else - t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - valp, 1, A_TP_RSS_PF0_CONFIG + index); + t4_tp_pio_read(adapter, valp, 1, A_TP_RSS_PF0_CONFIG + index, sleep_ok); } /** * t4_write_rss_pf_config - write PF RSS Configuration Table * @adapter: the adapter * @index: the entry in the VF RSS table to read * @val: the value to store + * @sleep_ok: if true we may sleep while awaiting command completion * * Writes the PF RSS Configuration Table at the specified index with the * specified value. */ void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, - u32 val) + u32 val, bool sleep_ok) { - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, &val, 1, - A_TP_RSS_PF0_CONFIG + index, 0); - else - t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &val, 1, A_TP_RSS_PF0_CONFIG + index); + t4_tp_pio_write(adapter, &val, 1, A_TP_RSS_PF0_CONFIG + index, + sleep_ok); } /** * t4_read_rss_vf_config - read VF RSS Configuration Table * @adapter: the adapter * @index: the entry in the VF RSS table to read * @vfl: where to store the returned VFL * @vfh: where to store the returned VFH + * @sleep_ok: if true we may sleep while awaiting command completion * * Reads the VF RSS Configuration Table at the specified index and returns * the (VFL, VFH) values found there. */ void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index, - u32 *vfl, u32 *vfh) + u32 *vfl, u32 *vfh, bool sleep_ok) { u32 vrt, mask, data; if (chip_id(adapter) <= CHELSIO_T5) { mask = V_VFWRADDR(M_VFWRADDR); data = V_VFWRADDR(index); } else { mask = V_T6_VFWRADDR(M_T6_VFWRADDR); data = V_T6_VFWRADDR(index); } /* * Request that the index'th VF Table values be read into VFL/VFH. */ vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT); vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask); vrt |= data | F_VFRDEN; t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt); /* * Grab the VFL/VFH values ... */ - if (t4_use_ldst(adapter)) { - t4_fw_tp_pio_rw(adapter, vfl, 1, A_TP_RSS_VFL_CONFIG, 1); - t4_fw_tp_pio_rw(adapter, vfh, 1, A_TP_RSS_VFH_CONFIG, 1); - } else { - t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - vfl, 1, A_TP_RSS_VFL_CONFIG); - t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - vfh, 1, A_TP_RSS_VFH_CONFIG); - } + t4_tp_pio_read(adapter, vfl, 1, A_TP_RSS_VFL_CONFIG, sleep_ok); + t4_tp_pio_read(adapter, vfh, 1, A_TP_RSS_VFH_CONFIG, sleep_ok); } /** * t4_write_rss_vf_config - write VF RSS Configuration Table * * @adapter: the adapter * @index: the entry in the VF RSS table to write * @vfl: the VFL to store * @vfh: the VFH to store * * Writes the VF RSS Configuration Table at the specified index with the * specified (VFL, VFH) values. */ void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index, - u32 vfl, u32 vfh) + u32 vfl, u32 vfh, bool sleep_ok) { u32 vrt, mask, data; if (chip_id(adapter) <= CHELSIO_T5) { mask = V_VFWRADDR(M_VFWRADDR); data = V_VFWRADDR(index); } else { mask = V_T6_VFWRADDR(M_T6_VFWRADDR); data = V_T6_VFWRADDR(index); } /* * Load up VFL/VFH with the values to be written ... */ - if (t4_use_ldst(adapter)) { - t4_fw_tp_pio_rw(adapter, &vfl, 1, A_TP_RSS_VFL_CONFIG, 0); - t4_fw_tp_pio_rw(adapter, &vfh, 1, A_TP_RSS_VFH_CONFIG, 0); - } else { - t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &vfl, 1, A_TP_RSS_VFL_CONFIG); - t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &vfh, 1, A_TP_RSS_VFH_CONFIG); - } + t4_tp_pio_write(adapter, &vfl, 1, A_TP_RSS_VFL_CONFIG, sleep_ok); + t4_tp_pio_write(adapter, &vfh, 1, A_TP_RSS_VFH_CONFIG, sleep_ok); /* * Write the VFL/VFH into the VF Table at index'th location. */ vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT); vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask); vrt |= data | F_VFRDEN; t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt); } /** * t4_read_rss_pf_map - read PF RSS Map * @adapter: the adapter + * @sleep_ok: if true we may sleep while awaiting command completion * * Reads the PF RSS Map register and returns its value. */ -u32 t4_read_rss_pf_map(struct adapter *adapter) +u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok) { u32 pfmap; - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, 1); - else - t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &pfmap, 1, A_TP_RSS_PF_MAP); + t4_tp_pio_read(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, sleep_ok); + return pfmap; } /** * t4_write_rss_pf_map - write PF RSS Map * @adapter: the adapter * @pfmap: PF RSS Map value * * Writes the specified value to the PF RSS Map register. */ -void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap) +void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap, bool sleep_ok) { - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, 0); - else - t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &pfmap, 1, A_TP_RSS_PF_MAP); + t4_tp_pio_write(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, sleep_ok); } /** * t4_read_rss_pf_mask - read PF RSS Mask * @adapter: the adapter + * @sleep_ok: if true we may sleep while awaiting command completion * * Reads the PF RSS Mask register and returns its value. */ -u32 t4_read_rss_pf_mask(struct adapter *adapter) +u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok) { u32 pfmask; - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, 1); - else - t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &pfmask, 1, A_TP_RSS_PF_MSK); + t4_tp_pio_read(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, sleep_ok); + return pfmask; } /** * t4_write_rss_pf_mask - write PF RSS Mask * @adapter: the adapter * @pfmask: PF RSS Mask value * * Writes the specified value to the PF RSS Mask register. */ -void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask) +void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask, bool sleep_ok) { - if (t4_use_ldst(adapter)) - t4_fw_tp_pio_rw(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, 0); - else - t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &pfmask, 1, A_TP_RSS_PF_MSK); + t4_tp_pio_write(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, sleep_ok); } /** * t4_tp_get_tcp_stats - read TP's TCP MIB counters * @adap: the adapter * @v4: holds the TCP/IP counter values * @v6: holds the TCP/IPv6 counter values + * @sleep_ok: if true we may sleep while awaiting command completion * * Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters. * Either @v4 or @v6 may be %NULL to skip the corresponding stats. */ void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4, - struct tp_tcp_stats *v6) + struct tp_tcp_stats *v6, bool sleep_ok) { u32 val[A_TP_MIB_TCP_RXT_SEG_LO - A_TP_MIB_TCP_OUT_RST + 1]; #define STAT_IDX(x) ((A_TP_MIB_TCP_##x) - A_TP_MIB_TCP_OUT_RST) #define STAT(x) val[STAT_IDX(x)] #define STAT64(x) (((u64)STAT(x##_HI) << 32) | STAT(x##_LO)) if (v4) { - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val, - ARRAY_SIZE(val), A_TP_MIB_TCP_OUT_RST); + t4_tp_mib_read(adap, val, ARRAY_SIZE(val), + A_TP_MIB_TCP_OUT_RST, sleep_ok); v4->tcp_out_rsts = STAT(OUT_RST); v4->tcp_in_segs = STAT64(IN_SEG); v4->tcp_out_segs = STAT64(OUT_SEG); v4->tcp_retrans_segs = STAT64(RXT_SEG); } if (v6) { - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val, - ARRAY_SIZE(val), A_TP_MIB_TCP_V6OUT_RST); + t4_tp_mib_read(adap, val, ARRAY_SIZE(val), + A_TP_MIB_TCP_V6OUT_RST, sleep_ok); v6->tcp_out_rsts = STAT(OUT_RST); v6->tcp_in_segs = STAT64(IN_SEG); v6->tcp_out_segs = STAT64(OUT_SEG); v6->tcp_retrans_segs = STAT64(RXT_SEG); } #undef STAT64 #undef STAT #undef STAT_IDX } /** * t4_tp_get_err_stats - read TP's error MIB counters * @adap: the adapter * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion * * Returns the values of TP's error counters. */ -void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st) +void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st, + bool sleep_ok) { int nchan = adap->chip_params->nchan; - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->mac_in_errs, nchan, A_TP_MIB_MAC_IN_ERR_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->hdr_in_errs, nchan, A_TP_MIB_HDR_IN_ERR_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->tcp_in_errs, nchan, A_TP_MIB_TCP_IN_ERR_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->tnl_cong_drops, nchan, A_TP_MIB_TNL_CNG_DROP_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->ofld_chan_drops, nchan, A_TP_MIB_OFD_CHN_DROP_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->tnl_tx_drops, nchan, A_TP_MIB_TNL_DROP_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->ofld_vlan_drops, nchan, A_TP_MIB_OFD_VLN_DROP_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - st->tcp6_in_errs, nchan, A_TP_MIB_TCP_V6IN_ERR_0); + t4_tp_mib_read(adap, st->mac_in_errs, nchan, A_TP_MIB_MAC_IN_ERR_0, + sleep_ok); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, - &st->ofld_no_neigh, 2, A_TP_MIB_OFD_ARP_DROP); + t4_tp_mib_read(adap, st->hdr_in_errs, nchan, A_TP_MIB_HDR_IN_ERR_0, + sleep_ok); + + t4_tp_mib_read(adap, st->tcp_in_errs, nchan, A_TP_MIB_TCP_IN_ERR_0, + sleep_ok); + + t4_tp_mib_read(adap, st->tnl_cong_drops, nchan, + A_TP_MIB_TNL_CNG_DROP_0, sleep_ok); + + t4_tp_mib_read(adap, st->ofld_chan_drops, nchan, + A_TP_MIB_OFD_CHN_DROP_0, sleep_ok); + + t4_tp_mib_read(adap, st->tnl_tx_drops, nchan, A_TP_MIB_TNL_DROP_0, + sleep_ok); + + t4_tp_mib_read(adap, st->ofld_vlan_drops, nchan, + A_TP_MIB_OFD_VLN_DROP_0, sleep_ok); + + t4_tp_mib_read(adap, st->tcp6_in_errs, nchan, + A_TP_MIB_TCP_V6IN_ERR_0, sleep_ok); + + t4_tp_mib_read(adap, &st->ofld_no_neigh, 2, A_TP_MIB_OFD_ARP_DROP, + sleep_ok); } /** * t4_tp_get_proxy_stats - read TP's proxy MIB counters * @adap: the adapter * @st: holds the counter values * * Returns the values of TP's proxy counters. */ -void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st) +void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st, + bool sleep_ok) { int nchan = adap->chip_params->nchan; - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->proxy, - nchan, A_TP_MIB_TNL_LPBK_0); + t4_tp_mib_read(adap, st->proxy, nchan, A_TP_MIB_TNL_LPBK_0, sleep_ok); } /** * t4_tp_get_cpl_stats - read TP's CPL MIB counters * @adap: the adapter * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion * * Returns the values of TP's CPL counters. */ -void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st) +void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st, + bool sleep_ok) { int nchan = adap->chip_params->nchan; - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->req, - nchan, A_TP_MIB_CPL_IN_REQ_0); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->rsp, - nchan, A_TP_MIB_CPL_OUT_RSP_0); + t4_tp_mib_read(adap, st->req, nchan, A_TP_MIB_CPL_IN_REQ_0, sleep_ok); + + t4_tp_mib_read(adap, st->rsp, nchan, A_TP_MIB_CPL_OUT_RSP_0, sleep_ok); } /** * t4_tp_get_rdma_stats - read TP's RDMA MIB counters * @adap: the adapter * @st: holds the counter values * * Returns the values of TP's RDMA counters. */ -void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st) +void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st, + bool sleep_ok) { - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->rqe_dfr_pkt, - 2, A_TP_MIB_RQE_DFR_PKT); + t4_tp_mib_read(adap, &st->rqe_dfr_pkt, 2, A_TP_MIB_RQE_DFR_PKT, + sleep_ok); } /** * t4_get_fcoe_stats - read TP's FCoE MIB counters for a port * @adap: the adapter * @idx: the port index * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion * * Returns the values of TP's FCoE counters for the selected port. */ void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx, - struct tp_fcoe_stats *st) + struct tp_fcoe_stats *st, bool sleep_ok) { u32 val[2]; - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->frames_ddp, - 1, A_TP_MIB_FCOE_DDP_0 + idx); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->frames_drop, - 1, A_TP_MIB_FCOE_DROP_0 + idx); - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val, - 2, A_TP_MIB_FCOE_BYTE_0_HI + 2 * idx); + t4_tp_mib_read(adap, &st->frames_ddp, 1, A_TP_MIB_FCOE_DDP_0 + idx, + sleep_ok); + + t4_tp_mib_read(adap, &st->frames_drop, 1, + A_TP_MIB_FCOE_DROP_0 + idx, sleep_ok); + + t4_tp_mib_read(adap, val, 2, A_TP_MIB_FCOE_BYTE_0_HI + 2 * idx, + sleep_ok); + st->octets_ddp = ((u64)val[0] << 32) | val[1]; } /** * t4_get_usm_stats - read TP's non-TCP DDP MIB counters * @adap: the adapter * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion * * Returns the values of TP's counters for non-TCP directly-placed packets. */ -void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st) +void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st, + bool sleep_ok) { u32 val[4]; - t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val, 4, - A_TP_MIB_USM_PKTS); + t4_tp_mib_read(adap, val, 4, A_TP_MIB_USM_PKTS, sleep_ok); + st->frames = val[0]; st->drops = val[1]; st->octets = ((u64)val[2] << 32) | val[3]; } /** * t4_read_mtu_tbl - returns the values in the HW path MTU table * @adap: the adapter * @mtus: where to store the MTU values * @mtu_log: where to store the MTU base-2 log (may be %NULL) * * Reads the HW path MTU table. */ void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log) { u32 v; int i; for (i = 0; i < NMTUS; ++i) { t4_write_reg(adap, A_TP_MTU_TABLE, V_MTUINDEX(0xff) | V_MTUVALUE(i)); v = t4_read_reg(adap, A_TP_MTU_TABLE); mtus[i] = G_MTUVALUE(v); if (mtu_log) mtu_log[i] = G_MTUWIDTH(v); } } /** * t4_read_cong_tbl - reads the congestion control table * @adap: the adapter * @incr: where to store the alpha values * * Reads the additive increments programmed into the HW congestion * control table. */ void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN]) { unsigned int mtu, w; for (mtu = 0; mtu < NMTUS; ++mtu) for (w = 0; w < NCCTRL_WIN; ++w) { t4_write_reg(adap, A_TP_CCTRL_TABLE, V_ROWINDEX(0xffff) | (mtu << 5) | w); incr[mtu][w] = (u16)t4_read_reg(adap, A_TP_CCTRL_TABLE) & 0x1fff; } } /** * t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register * @adap: the adapter * @addr: the indirect TP register address * @mask: specifies the field within the register to modify * @val: new value for the field * * Sets a field of an indirect TP register to the given value. */ void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr, unsigned int mask, unsigned int val) { t4_write_reg(adap, A_TP_PIO_ADDR, addr); val |= t4_read_reg(adap, A_TP_PIO_DATA) & ~mask; t4_write_reg(adap, A_TP_PIO_DATA, val); } /** * init_cong_ctrl - initialize congestion control parameters * @a: the alpha values for congestion control * @b: the beta values for congestion control * * Initialize the congestion control parameters. */ static void init_cong_ctrl(unsigned short *a, unsigned short *b) { a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1; a[9] = 2; a[10] = 3; a[11] = 4; a[12] = 5; a[13] = 6; a[14] = 7; a[15] = 8; a[16] = 9; a[17] = 10; a[18] = 14; a[19] = 17; a[20] = 21; a[21] = 25; a[22] = 30; a[23] = 35; a[24] = 45; a[25] = 60; a[26] = 80; a[27] = 100; a[28] = 200; a[29] = 300; a[30] = 400; a[31] = 500; b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0; b[9] = b[10] = 1; b[11] = b[12] = 2; b[13] = b[14] = b[15] = b[16] = 3; b[17] = b[18] = b[19] = b[20] = b[21] = 4; b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5; b[28] = b[29] = 6; b[30] = b[31] = 7; } /* The minimum additive increment value for the congestion control table */ #define CC_MIN_INCR 2U /** * t4_load_mtus - write the MTU and congestion control HW tables * @adap: the adapter * @mtus: the values for the MTU table * @alpha: the values for the congestion control alpha parameter * @beta: the values for the congestion control beta parameter * * Write the HW MTU table with the supplied MTUs and the high-speed * congestion control table with the supplied alpha, beta, and MTUs. * We write the two tables together because the additive increments * depend on the MTUs. */ void t4_load_mtus(struct adapter *adap, const unsigned short *mtus, const unsigned short *alpha, const unsigned short *beta) { static const unsigned int avg_pkts[NCCTRL_WIN] = { 2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640, 896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480, 28672, 40960, 57344, 81920, 114688, 163840, 229376 }; unsigned int i, w; for (i = 0; i < NMTUS; ++i) { unsigned int mtu = mtus[i]; unsigned int log2 = fls(mtu); if (!(mtu & ((1 << log2) >> 2))) /* round */ log2--; t4_write_reg(adap, A_TP_MTU_TABLE, V_MTUINDEX(i) | V_MTUWIDTH(log2) | V_MTUVALUE(mtu)); for (w = 0; w < NCCTRL_WIN; ++w) { unsigned int inc; inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w], CC_MIN_INCR); t4_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) | (w << 16) | (beta[w] << 13) | inc); } } } /** * t4_set_pace_tbl - set the pace table * @adap: the adapter * @pace_vals: the pace values in microseconds * @start: index of the first entry in the HW pace table to set * @n: how many entries to set * * Sets (a subset of the) HW pace table. */ int t4_set_pace_tbl(struct adapter *adap, const unsigned int *pace_vals, unsigned int start, unsigned int n) { unsigned int vals[NTX_SCHED], i; unsigned int tick_ns = dack_ticks_to_usec(adap, 1000); if (n > NTX_SCHED) return -ERANGE; /* convert values from us to dack ticks, rounding to closest value */ for (i = 0; i < n; i++, pace_vals++) { vals[i] = (1000 * *pace_vals + tick_ns / 2) / tick_ns; if (vals[i] > 0x7ff) return -ERANGE; if (*pace_vals && vals[i] == 0) return -ERANGE; } for (i = 0; i < n; i++, start++) t4_write_reg(adap, A_TP_PACE_TABLE, (start << 16) | vals[i]); return 0; } /** * t4_set_sched_bps - set the bit rate for a HW traffic scheduler * @adap: the adapter * @kbps: target rate in Kbps * @sched: the scheduler index * * Configure a Tx HW scheduler for the target rate. */ int t4_set_sched_bps(struct adapter *adap, int sched, unsigned int kbps) { unsigned int v, tps, cpt, bpt, delta, mindelta = ~0; unsigned int clk = adap->params.vpd.cclk * 1000; unsigned int selected_cpt = 0, selected_bpt = 0; if (kbps > 0) { kbps *= 125; /* -> bytes */ for (cpt = 1; cpt <= 255; cpt++) { tps = clk / cpt; bpt = (kbps + tps / 2) / tps; if (bpt > 0 && bpt <= 255) { v = bpt * tps; delta = v >= kbps ? v - kbps : kbps - v; if (delta < mindelta) { mindelta = delta; selected_cpt = cpt; selected_bpt = bpt; } } else if (selected_cpt) break; } if (!selected_cpt) return -EINVAL; } t4_write_reg(adap, A_TP_TM_PIO_ADDR, A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2); v = t4_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v = (v & 0xffff) | (selected_cpt << 16) | (selected_bpt << 24); else v = (v & 0xffff0000) | selected_cpt | (selected_bpt << 8); t4_write_reg(adap, A_TP_TM_PIO_DATA, v); return 0; } /** * t4_set_sched_ipg - set the IPG for a Tx HW packet rate scheduler * @adap: the adapter * @sched: the scheduler index * @ipg: the interpacket delay in tenths of nanoseconds * * Set the interpacket delay for a HW packet rate scheduler. */ int t4_set_sched_ipg(struct adapter *adap, int sched, unsigned int ipg) { unsigned int v, addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2; /* convert ipg to nearest number of core clocks */ ipg *= core_ticks_per_usec(adap); ipg = (ipg + 5000) / 10000; if (ipg > M_TXTIMERSEPQ0) return -EINVAL; t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr); v = t4_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v = (v & V_TXTIMERSEPQ0(M_TXTIMERSEPQ0)) | V_TXTIMERSEPQ1(ipg); else v = (v & V_TXTIMERSEPQ1(M_TXTIMERSEPQ1)) | V_TXTIMERSEPQ0(ipg); t4_write_reg(adap, A_TP_TM_PIO_DATA, v); t4_read_reg(adap, A_TP_TM_PIO_DATA); return 0; } /* * Calculates a rate in bytes/s given the number of 256-byte units per 4K core * clocks. The formula is * * bytes/s = bytes256 * 256 * ClkFreq / 4096 * * which is equivalent to * * bytes/s = 62.5 * bytes256 * ClkFreq_ms */ static u64 chan_rate(struct adapter *adap, unsigned int bytes256) { u64 v = bytes256 * adap->params.vpd.cclk; return v * 62 + v / 2; } /** * t4_get_chan_txrate - get the current per channel Tx rates * @adap: the adapter * @nic_rate: rates for NIC traffic * @ofld_rate: rates for offloaded traffic * * Return the current Tx rates in bytes/s for NIC and offloaded traffic * for each channel. */ void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate) { u32 v; v = t4_read_reg(adap, A_TP_TX_TRATE); nic_rate[0] = chan_rate(adap, G_TNLRATE0(v)); nic_rate[1] = chan_rate(adap, G_TNLRATE1(v)); if (adap->chip_params->nchan > 2) { nic_rate[2] = chan_rate(adap, G_TNLRATE2(v)); nic_rate[3] = chan_rate(adap, G_TNLRATE3(v)); } v = t4_read_reg(adap, A_TP_TX_ORATE); ofld_rate[0] = chan_rate(adap, G_OFDRATE0(v)); ofld_rate[1] = chan_rate(adap, G_OFDRATE1(v)); if (adap->chip_params->nchan > 2) { ofld_rate[2] = chan_rate(adap, G_OFDRATE2(v)); ofld_rate[3] = chan_rate(adap, G_OFDRATE3(v)); } } /** * t4_set_trace_filter - configure one of the tracing filters * @adap: the adapter * @tp: the desired trace filter parameters * @idx: which filter to configure * @enable: whether to enable or disable the filter * * Configures one of the tracing filters available in HW. If @tp is %NULL * it indicates that the filter is already written in the register and it * just needs to be enabled or disabled. */ int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp, int idx, int enable) { int i, ofst = idx * 4; u32 data_reg, mask_reg, cfg; u32 multitrc = F_TRCMULTIFILTER; u32 en = is_t4(adap) ? F_TFEN : F_T5_TFEN; if (idx < 0 || idx >= NTRACE) return -EINVAL; if (tp == NULL || !enable) { t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en, enable ? en : 0); return 0; } /* * TODO - After T4 data book is updated, specify the exact * section below. * * See T4 data book - MPS section for a complete description * of the below if..else handling of A_MPS_TRC_CFG register * value. */ cfg = t4_read_reg(adap, A_MPS_TRC_CFG); if (cfg & F_TRCMULTIFILTER) { /* * If multiple tracers are enabled, then maximum * capture size is 2.5KB (FIFO size of a single channel) * minus 2 flits for CPL_TRACE_PKT header. */ if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8))) return -EINVAL; } else { /* * If multiple tracers are disabled, to avoid deadlocks * maximum packet capture size of 9600 bytes is recommended. * Also in this mode, only trace0 can be enabled and running. */ multitrc = 0; if (tp->snap_len > 9600 || idx) return -EINVAL; } if (tp->port > (is_t4(adap) ? 11 : 19) || tp->invert > 1 || tp->skip_len > M_TFLENGTH || tp->skip_ofst > M_TFOFFSET || tp->min_len > M_TFMINPKTSIZE) return -EINVAL; /* stop the tracer we'll be changing */ t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en, 0); idx *= (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH); data_reg = A_MPS_TRC_FILTER0_MATCH + idx; mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + idx; for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) { t4_write_reg(adap, data_reg, tp->data[i]); t4_write_reg(adap, mask_reg, ~tp->mask[i]); } t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst, V_TFCAPTUREMAX(tp->snap_len) | V_TFMINPKTSIZE(tp->min_len)); t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, V_TFOFFSET(tp->skip_ofst) | V_TFLENGTH(tp->skip_len) | en | (is_t4(adap) ? V_TFPORT(tp->port) | V_TFINVERTMATCH(tp->invert) : V_T5_TFPORT(tp->port) | V_T5_TFINVERTMATCH(tp->invert))); return 0; } /** * t4_get_trace_filter - query one of the tracing filters * @adap: the adapter * @tp: the current trace filter parameters * @idx: which trace filter to query * @enabled: non-zero if the filter is enabled * * Returns the current settings of one of the HW tracing filters. */ void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx, int *enabled) { u32 ctla, ctlb; int i, ofst = idx * 4; u32 data_reg, mask_reg; ctla = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst); ctlb = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst); if (is_t4(adap)) { *enabled = !!(ctla & F_TFEN); tp->port = G_TFPORT(ctla); tp->invert = !!(ctla & F_TFINVERTMATCH); } else { *enabled = !!(ctla & F_T5_TFEN); tp->port = G_T5_TFPORT(ctla); tp->invert = !!(ctla & F_T5_TFINVERTMATCH); } tp->snap_len = G_TFCAPTUREMAX(ctlb); tp->min_len = G_TFMINPKTSIZE(ctlb); tp->skip_ofst = G_TFOFFSET(ctla); tp->skip_len = G_TFLENGTH(ctla); ofst = (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH) * idx; data_reg = A_MPS_TRC_FILTER0_MATCH + ofst; mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + ofst; for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) { tp->mask[i] = ~t4_read_reg(adap, mask_reg); tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i]; } } /** * t4_pmtx_get_stats - returns the HW stats from PMTX * @adap: the adapter * @cnt: where to store the count statistics * @cycles: where to store the cycle statistics * * Returns performance statistics from PMTX. */ void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]) { int i; u32 data[2]; for (i = 0; i < adap->chip_params->pm_stats_cnt; i++) { t4_write_reg(adap, A_PM_TX_STAT_CONFIG, i + 1); cnt[i] = t4_read_reg(adap, A_PM_TX_STAT_COUNT); if (is_t4(adap)) cycles[i] = t4_read_reg64(adap, A_PM_TX_STAT_LSB); else { t4_read_indirect(adap, A_PM_TX_DBG_CTRL, A_PM_TX_DBG_DATA, data, 2, A_PM_TX_DBG_STAT_MSB); cycles[i] = (((u64)data[0] << 32) | data[1]); } } } /** * t4_pmrx_get_stats - returns the HW stats from PMRX * @adap: the adapter * @cnt: where to store the count statistics * @cycles: where to store the cycle statistics * * Returns performance statistics from PMRX. */ void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]) { int i; u32 data[2]; for (i = 0; i < adap->chip_params->pm_stats_cnt; i++) { t4_write_reg(adap, A_PM_RX_STAT_CONFIG, i + 1); cnt[i] = t4_read_reg(adap, A_PM_RX_STAT_COUNT); if (is_t4(adap)) { cycles[i] = t4_read_reg64(adap, A_PM_RX_STAT_LSB); } else { t4_read_indirect(adap, A_PM_RX_DBG_CTRL, A_PM_RX_DBG_DATA, data, 2, A_PM_RX_DBG_STAT_MSB); cycles[i] = (((u64)data[0] << 32) | data[1]); } } } /** * t4_get_mps_bg_map - return the buffer groups associated with a port * @adap: the adapter * @idx: the port index * * Returns a bitmap indicating which MPS buffer groups are associated * with the given port. Bit i is set if buffer group i is used by the * port. */ static unsigned int t4_get_mps_bg_map(struct adapter *adap, int idx) { u32 n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL)); if (n == 0) return idx == 0 ? 0xf : 0; if (n == 1 && chip_id(adap) <= CHELSIO_T5) return idx < 2 ? (3 << (2 * idx)) : 0; return 1 << idx; } /** * t4_get_port_type_description - return Port Type string description * @port_type: firmware Port Type enumeration */ const char *t4_get_port_type_description(enum fw_port_type port_type) { static const char *const port_type_description[] = { "Fiber_XFI", "Fiber_XAUI", "BT_SGMII", "BT_XFI", "BT_XAUI", "KX4", "CX4", "KX", "KR", "SFP", "BP_AP", "BP4_AP", "QSFP_10G", "QSA", "QSFP", "BP40_BA", "KR4_100G", "CR4_QSFP", "CR_QSFP", "CR2_QSFP", "SFP28", "KR_SFP28", }; if (port_type < ARRAY_SIZE(port_type_description)) return port_type_description[port_type]; return "UNKNOWN"; } /** * t4_get_port_stats_offset - collect port stats relative to a previous * snapshot * @adap: The adapter * @idx: The port * @stats: Current stats to fill * @offset: Previous stats snapshot */ void t4_get_port_stats_offset(struct adapter *adap, int idx, struct port_stats *stats, struct port_stats *offset) { u64 *s, *o; int i; t4_get_port_stats(adap, idx, stats); for (i = 0, s = (u64 *)stats, o = (u64 *)offset ; i < (sizeof(struct port_stats)/sizeof(u64)) ; i++, s++, o++) *s -= *o; } /** * t4_get_port_stats - collect port statistics * @adap: the adapter * @idx: the port index * @p: the stats structure to fill * * Collect statistics related to the given port from HW. */ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p) { u32 bgmap = t4_get_mps_bg_map(adap, idx); - u32 stat_ctl; + u32 stat_ctl = t4_read_reg(adap, A_MPS_STAT_CTL); #define GET_STAT(name) \ t4_read_reg64(adap, \ (is_t4(adap) ? PORT_REG(idx, A_MPS_PORT_STAT_##name##_L) : \ T5_PORT_REG(idx, A_MPS_PORT_STAT_##name##_L))) #define GET_STAT_COM(name) t4_read_reg64(adap, A_MPS_STAT_##name##_L) - stat_ctl = t4_read_reg(adap, A_MPS_STAT_CTL); - p->tx_pause = GET_STAT(TX_PORT_PAUSE); p->tx_octets = GET_STAT(TX_PORT_BYTES); p->tx_frames = GET_STAT(TX_PORT_FRAMES); p->tx_bcast_frames = GET_STAT(TX_PORT_BCAST); p->tx_mcast_frames = GET_STAT(TX_PORT_MCAST); p->tx_ucast_frames = GET_STAT(TX_PORT_UCAST); p->tx_error_frames = GET_STAT(TX_PORT_ERROR); p->tx_frames_64 = GET_STAT(TX_PORT_64B); p->tx_frames_65_127 = GET_STAT(TX_PORT_65B_127B); p->tx_frames_128_255 = GET_STAT(TX_PORT_128B_255B); p->tx_frames_256_511 = GET_STAT(TX_PORT_256B_511B); p->tx_frames_512_1023 = GET_STAT(TX_PORT_512B_1023B); p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B); p->tx_frames_1519_max = GET_STAT(TX_PORT_1519B_MAX); p->tx_drop = GET_STAT(TX_PORT_DROP); p->tx_ppp0 = GET_STAT(TX_PORT_PPP0); p->tx_ppp1 = GET_STAT(TX_PORT_PPP1); p->tx_ppp2 = GET_STAT(TX_PORT_PPP2); p->tx_ppp3 = GET_STAT(TX_PORT_PPP3); p->tx_ppp4 = GET_STAT(TX_PORT_PPP4); p->tx_ppp5 = GET_STAT(TX_PORT_PPP5); p->tx_ppp6 = GET_STAT(TX_PORT_PPP6); p->tx_ppp7 = GET_STAT(TX_PORT_PPP7); if (chip_id(adap) >= CHELSIO_T5) { if (stat_ctl & F_COUNTPAUSESTATTX) { p->tx_frames -= p->tx_pause; p->tx_octets -= p->tx_pause * 64; } if (stat_ctl & F_COUNTPAUSEMCTX) p->tx_mcast_frames -= p->tx_pause; } p->rx_pause = GET_STAT(RX_PORT_PAUSE); p->rx_octets = GET_STAT(RX_PORT_BYTES); p->rx_frames = GET_STAT(RX_PORT_FRAMES); p->rx_bcast_frames = GET_STAT(RX_PORT_BCAST); p->rx_mcast_frames = GET_STAT(RX_PORT_MCAST); p->rx_ucast_frames = GET_STAT(RX_PORT_UCAST); p->rx_too_long = GET_STAT(RX_PORT_MTU_ERROR); p->rx_jabber = GET_STAT(RX_PORT_MTU_CRC_ERROR); p->rx_fcs_err = GET_STAT(RX_PORT_CRC_ERROR); p->rx_len_err = GET_STAT(RX_PORT_LEN_ERROR); p->rx_symbol_err = GET_STAT(RX_PORT_SYM_ERROR); p->rx_runt = GET_STAT(RX_PORT_LESS_64B); p->rx_frames_64 = GET_STAT(RX_PORT_64B); p->rx_frames_65_127 = GET_STAT(RX_PORT_65B_127B); p->rx_frames_128_255 = GET_STAT(RX_PORT_128B_255B); p->rx_frames_256_511 = GET_STAT(RX_PORT_256B_511B); p->rx_frames_512_1023 = GET_STAT(RX_PORT_512B_1023B); p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B); p->rx_frames_1519_max = GET_STAT(RX_PORT_1519B_MAX); p->rx_ppp0 = GET_STAT(RX_PORT_PPP0); p->rx_ppp1 = GET_STAT(RX_PORT_PPP1); p->rx_ppp2 = GET_STAT(RX_PORT_PPP2); p->rx_ppp3 = GET_STAT(RX_PORT_PPP3); p->rx_ppp4 = GET_STAT(RX_PORT_PPP4); p->rx_ppp5 = GET_STAT(RX_PORT_PPP5); p->rx_ppp6 = GET_STAT(RX_PORT_PPP6); p->rx_ppp7 = GET_STAT(RX_PORT_PPP7); if (chip_id(adap) >= CHELSIO_T5) { if (stat_ctl & F_COUNTPAUSESTATRX) { p->rx_frames -= p->rx_pause; p->rx_octets -= p->rx_pause * 64; } if (stat_ctl & F_COUNTPAUSEMCRX) p->rx_mcast_frames -= p->rx_pause; } p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0; p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0; p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0; p->rx_ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0; p->rx_trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0; p->rx_trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0; p->rx_trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0; p->rx_trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0; #undef GET_STAT #undef GET_STAT_COM } /** * t4_get_lb_stats - collect loopback port statistics * @adap: the adapter * @idx: the loopback port index * @p: the stats structure to fill * * Return HW statistics for the given loopback port. */ void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p) { u32 bgmap = t4_get_mps_bg_map(adap, idx); #define GET_STAT(name) \ t4_read_reg64(adap, \ (is_t4(adap) ? \ PORT_REG(idx, A_MPS_PORT_STAT_LB_PORT_##name##_L) : \ T5_PORT_REG(idx, A_MPS_PORT_STAT_LB_PORT_##name##_L))) #define GET_STAT_COM(name) t4_read_reg64(adap, A_MPS_STAT_##name##_L) p->octets = GET_STAT(BYTES); p->frames = GET_STAT(FRAMES); p->bcast_frames = GET_STAT(BCAST); p->mcast_frames = GET_STAT(MCAST); p->ucast_frames = GET_STAT(UCAST); p->error_frames = GET_STAT(ERROR); p->frames_64 = GET_STAT(64B); p->frames_65_127 = GET_STAT(65B_127B); p->frames_128_255 = GET_STAT(128B_255B); p->frames_256_511 = GET_STAT(256B_511B); p->frames_512_1023 = GET_STAT(512B_1023B); p->frames_1024_1518 = GET_STAT(1024B_1518B); p->frames_1519_max = GET_STAT(1519B_MAX); p->drop = GET_STAT(DROP_FRAMES); p->ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_DROP_FRAME) : 0; p->ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_DROP_FRAME) : 0; p->ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_DROP_FRAME) : 0; p->ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_DROP_FRAME) : 0; p->trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_TRUNC_FRAME) : 0; p->trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_TRUNC_FRAME) : 0; p->trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_TRUNC_FRAME) : 0; p->trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_TRUNC_FRAME) : 0; #undef GET_STAT #undef GET_STAT_COM } /** * t4_wol_magic_enable - enable/disable magic packet WoL * @adap: the adapter * @port: the physical port index * @addr: MAC address expected in magic packets, %NULL to disable * * Enables/disables magic packet wake-on-LAN for the selected port. */ void t4_wol_magic_enable(struct adapter *adap, unsigned int port, const u8 *addr) { u32 mag_id_reg_l, mag_id_reg_h, port_cfg_reg; if (is_t4(adap)) { mag_id_reg_l = PORT_REG(port, A_XGMAC_PORT_MAGIC_MACID_LO); mag_id_reg_h = PORT_REG(port, A_XGMAC_PORT_MAGIC_MACID_HI); port_cfg_reg = PORT_REG(port, A_XGMAC_PORT_CFG2); } else { mag_id_reg_l = T5_PORT_REG(port, A_MAC_PORT_MAGIC_MACID_LO); mag_id_reg_h = T5_PORT_REG(port, A_MAC_PORT_MAGIC_MACID_HI); port_cfg_reg = T5_PORT_REG(port, A_MAC_PORT_CFG2); } if (addr) { t4_write_reg(adap, mag_id_reg_l, (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) | addr[5]); t4_write_reg(adap, mag_id_reg_h, (addr[0] << 8) | addr[1]); } t4_set_reg_field(adap, port_cfg_reg, F_MAGICEN, V_MAGICEN(addr != NULL)); } /** * t4_wol_pat_enable - enable/disable pattern-based WoL * @adap: the adapter * @port: the physical port index * @map: bitmap of which HW pattern filters to set * @mask0: byte mask for bytes 0-63 of a packet * @mask1: byte mask for bytes 64-127 of a packet * @crc: Ethernet CRC for selected bytes * @enable: enable/disable switch * * Sets the pattern filters indicated in @map to mask out the bytes * specified in @mask0/@mask1 in received packets and compare the CRC of * the resulting packet against @crc. If @enable is %true pattern-based * WoL is enabled, otherwise disabled. */ int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map, u64 mask0, u64 mask1, unsigned int crc, bool enable) { int i; u32 port_cfg_reg; if (is_t4(adap)) port_cfg_reg = PORT_REG(port, A_XGMAC_PORT_CFG2); else port_cfg_reg = T5_PORT_REG(port, A_MAC_PORT_CFG2); if (!enable) { t4_set_reg_field(adap, port_cfg_reg, F_PATEN, 0); return 0; } if (map > 0xff) return -EINVAL; #define EPIO_REG(name) \ (is_t4(adap) ? PORT_REG(port, A_XGMAC_PORT_EPIO_##name) : \ T5_PORT_REG(port, A_MAC_PORT_EPIO_##name)) t4_write_reg(adap, EPIO_REG(DATA1), mask0 >> 32); t4_write_reg(adap, EPIO_REG(DATA2), mask1); t4_write_reg(adap, EPIO_REG(DATA3), mask1 >> 32); for (i = 0; i < NWOL_PAT; i++, map >>= 1) { if (!(map & 1)) continue; /* write byte masks */ t4_write_reg(adap, EPIO_REG(DATA0), mask0); t4_write_reg(adap, EPIO_REG(OP), V_ADDRESS(i) | F_EPIOWR); t4_read_reg(adap, EPIO_REG(OP)); /* flush */ if (t4_read_reg(adap, EPIO_REG(OP)) & F_BUSY) return -ETIMEDOUT; /* write CRC */ t4_write_reg(adap, EPIO_REG(DATA0), crc); t4_write_reg(adap, EPIO_REG(OP), V_ADDRESS(i + 32) | F_EPIOWR); t4_read_reg(adap, EPIO_REG(OP)); /* flush */ if (t4_read_reg(adap, EPIO_REG(OP)) & F_BUSY) return -ETIMEDOUT; } #undef EPIO_REG t4_set_reg_field(adap, port_cfg_reg, 0, F_PATEN); return 0; } /* t4_mk_filtdelwr - create a delete filter WR * @ftid: the filter ID * @wr: the filter work request to populate * @qid: ingress queue to receive the delete notification * * Creates a filter work request to delete the supplied filter. If @qid is * negative the delete notification is suppressed. */ void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid) { memset(wr, 0, sizeof(*wr)); wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR)); wr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*wr) / 16)); wr->tid_to_iq = cpu_to_be32(V_FW_FILTER_WR_TID(ftid) | V_FW_FILTER_WR_NOREPLY(qid < 0)); wr->del_filter_to_l2tix = cpu_to_be32(F_FW_FILTER_WR_DEL_FILTER); if (qid >= 0) wr->rx_chan_rx_rpl_iq = cpu_to_be16(V_FW_FILTER_WR_RX_RPL_IQ(qid)); } #define INIT_CMD(var, cmd, rd_wr) do { \ (var).op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_##cmd##_CMD) | \ F_FW_CMD_REQUEST | \ F_FW_CMD_##rd_wr); \ (var).retval_len16 = cpu_to_be32(FW_LEN16(var)); \ } while (0) int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, u32 addr, u32 val) { u32 ldst_addrspace; struct fw_ldst_cmd c; memset(&c, 0, sizeof(c)); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FIRMWARE); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | ldst_addrspace); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.addrval.addr = cpu_to_be32(addr); c.u.addrval.val = cpu_to_be32(val); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_mdio_rd - read a PHY register through MDIO * @adap: the adapter * @mbox: mailbox to use for the FW command * @phy_addr: the PHY address * @mmd: the PHY MMD to access (0 for clause 22 PHYs) * @reg: the register to read * @valp: where to store the value * * Issues a FW command through the given mailbox to read a PHY register. */ int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, unsigned int mmd, unsigned int reg, unsigned int *valp) { int ret; u32 ldst_addrspace; struct fw_ldst_cmd c; memset(&c, 0, sizeof(c)); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MDIO); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | ldst_addrspace); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.mdio.paddr_mmd = cpu_to_be16(V_FW_LDST_CMD_PADDR(phy_addr) | V_FW_LDST_CMD_MMD(mmd)); c.u.mdio.raddr = cpu_to_be16(reg); ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret == 0) *valp = be16_to_cpu(c.u.mdio.rval); return ret; } /** * t4_mdio_wr - write a PHY register through MDIO * @adap: the adapter * @mbox: mailbox to use for the FW command * @phy_addr: the PHY address * @mmd: the PHY MMD to access (0 for clause 22 PHYs) * @reg: the register to write * @valp: value to write * * Issues a FW command through the given mailbox to write a PHY register. */ int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, unsigned int mmd, unsigned int reg, unsigned int val) { u32 ldst_addrspace; struct fw_ldst_cmd c; memset(&c, 0, sizeof(c)); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MDIO); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | ldst_addrspace); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.mdio.paddr_mmd = cpu_to_be16(V_FW_LDST_CMD_PADDR(phy_addr) | V_FW_LDST_CMD_MMD(mmd)); c.u.mdio.raddr = cpu_to_be16(reg); c.u.mdio.rval = cpu_to_be16(val); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * * t4_sge_decode_idma_state - decode the idma state * @adap: the adapter * @state: the state idma is stuck in */ void t4_sge_decode_idma_state(struct adapter *adapter, int state) { static const char * const t4_decode[] = { "IDMA_IDLE", "IDMA_PUSH_MORE_CPL_FIFO", "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", "Not used", "IDMA_PHYSADDR_SEND_PCIEHDR", "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", "IDMA_PHYSADDR_SEND_PAYLOAD", "IDMA_SEND_FIFO_TO_IMSG", "IDMA_FL_REQ_DATA_FL_PREP", "IDMA_FL_REQ_DATA_FL", "IDMA_FL_DROP", "IDMA_FL_H_REQ_HEADER_FL", "IDMA_FL_H_SEND_PCIEHDR", "IDMA_FL_H_PUSH_CPL_FIFO", "IDMA_FL_H_SEND_CPL", "IDMA_FL_H_SEND_IP_HDR_FIRST", "IDMA_FL_H_SEND_IP_HDR", "IDMA_FL_H_REQ_NEXT_HEADER_FL", "IDMA_FL_H_SEND_NEXT_PCIEHDR", "IDMA_FL_H_SEND_IP_HDR_PADDING", "IDMA_FL_D_SEND_PCIEHDR", "IDMA_FL_D_SEND_CPL_AND_IP_HDR", "IDMA_FL_D_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_PCIEHDR", "IDMA_FL_PUSH_CPL_FIFO", "IDMA_FL_SEND_CPL", "IDMA_FL_SEND_PAYLOAD_FIRST", "IDMA_FL_SEND_PAYLOAD", "IDMA_FL_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_NEXT_PCIEHDR", "IDMA_FL_SEND_PADDING", "IDMA_FL_SEND_COMPLETION_TO_IMSG", "IDMA_FL_SEND_FIFO_TO_IMSG", "IDMA_FL_REQ_DATAFL_DONE", "IDMA_FL_REQ_HEADERFL_DONE", }; static const char * const t5_decode[] = { "IDMA_IDLE", "IDMA_ALMOST_IDLE", "IDMA_PUSH_MORE_CPL_FIFO", "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", "IDMA_SGEFLRFLUSH_SEND_PCIEHDR", "IDMA_PHYSADDR_SEND_PCIEHDR", "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", "IDMA_PHYSADDR_SEND_PAYLOAD", "IDMA_SEND_FIFO_TO_IMSG", "IDMA_FL_REQ_DATA_FL", "IDMA_FL_DROP", "IDMA_FL_DROP_SEND_INC", "IDMA_FL_H_REQ_HEADER_FL", "IDMA_FL_H_SEND_PCIEHDR", "IDMA_FL_H_PUSH_CPL_FIFO", "IDMA_FL_H_SEND_CPL", "IDMA_FL_H_SEND_IP_HDR_FIRST", "IDMA_FL_H_SEND_IP_HDR", "IDMA_FL_H_REQ_NEXT_HEADER_FL", "IDMA_FL_H_SEND_NEXT_PCIEHDR", "IDMA_FL_H_SEND_IP_HDR_PADDING", "IDMA_FL_D_SEND_PCIEHDR", "IDMA_FL_D_SEND_CPL_AND_IP_HDR", "IDMA_FL_D_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_PCIEHDR", "IDMA_FL_PUSH_CPL_FIFO", "IDMA_FL_SEND_CPL", "IDMA_FL_SEND_PAYLOAD_FIRST", "IDMA_FL_SEND_PAYLOAD", "IDMA_FL_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_NEXT_PCIEHDR", "IDMA_FL_SEND_PADDING", "IDMA_FL_SEND_COMPLETION_TO_IMSG", }; static const char * const t6_decode[] = { "IDMA_IDLE", "IDMA_PUSH_MORE_CPL_FIFO", "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", "IDMA_SGEFLRFLUSH_SEND_PCIEHDR", "IDMA_PHYSADDR_SEND_PCIEHDR", "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", "IDMA_PHYSADDR_SEND_PAYLOAD", "IDMA_FL_REQ_DATA_FL", "IDMA_FL_DROP", "IDMA_FL_DROP_SEND_INC", "IDMA_FL_H_REQ_HEADER_FL", "IDMA_FL_H_SEND_PCIEHDR", "IDMA_FL_H_PUSH_CPL_FIFO", "IDMA_FL_H_SEND_CPL", "IDMA_FL_H_SEND_IP_HDR_FIRST", "IDMA_FL_H_SEND_IP_HDR", "IDMA_FL_H_REQ_NEXT_HEADER_FL", "IDMA_FL_H_SEND_NEXT_PCIEHDR", "IDMA_FL_H_SEND_IP_HDR_PADDING", "IDMA_FL_D_SEND_PCIEHDR", "IDMA_FL_D_SEND_CPL_AND_IP_HDR", "IDMA_FL_D_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_PCIEHDR", "IDMA_FL_PUSH_CPL_FIFO", "IDMA_FL_SEND_CPL", "IDMA_FL_SEND_PAYLOAD_FIRST", "IDMA_FL_SEND_PAYLOAD", "IDMA_FL_REQ_NEXT_DATA_FL", "IDMA_FL_SEND_NEXT_PCIEHDR", "IDMA_FL_SEND_PADDING", "IDMA_FL_SEND_COMPLETION_TO_IMSG", }; static const u32 sge_regs[] = { A_SGE_DEBUG_DATA_LOW_INDEX_2, A_SGE_DEBUG_DATA_LOW_INDEX_3, A_SGE_DEBUG_DATA_HIGH_INDEX_10, }; const char * const *sge_idma_decode; int sge_idma_decode_nstates; int i; unsigned int chip_version = chip_id(adapter); /* Select the right set of decode strings to dump depending on the * adapter chip type. */ switch (chip_version) { case CHELSIO_T4: sge_idma_decode = (const char * const *)t4_decode; sge_idma_decode_nstates = ARRAY_SIZE(t4_decode); break; case CHELSIO_T5: sge_idma_decode = (const char * const *)t5_decode; sge_idma_decode_nstates = ARRAY_SIZE(t5_decode); break; case CHELSIO_T6: sge_idma_decode = (const char * const *)t6_decode; sge_idma_decode_nstates = ARRAY_SIZE(t6_decode); break; default: CH_ERR(adapter, "Unsupported chip version %d\n", chip_version); return; } if (state < sge_idma_decode_nstates) CH_WARN(adapter, "idma state %s\n", sge_idma_decode[state]); else CH_WARN(adapter, "idma state %d unknown\n", state); for (i = 0; i < ARRAY_SIZE(sge_regs); i++) CH_WARN(adapter, "SGE register %#x value %#x\n", sge_regs[i], t4_read_reg(adapter, sge_regs[i])); } /** * t4_sge_ctxt_flush - flush the SGE context cache * @adap: the adapter * @mbox: mailbox to use for the FW command * * Issues a FW command through the given mailbox to flush the * SGE context cache. */ int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox) { int ret; u32 ldst_addrspace; struct fw_ldst_cmd c; memset(&c, 0, sizeof(c)); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_SGE_EGRC); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | ldst_addrspace); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.idctxt.msg_ctxtflush = cpu_to_be32(F_FW_LDST_CMD_CTXTFLUSH); ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); return ret; } /** * t4_fw_hello - establish communication with FW * @adap: the adapter * @mbox: mailbox to use for the FW command * @evt_mbox: mailbox to receive async FW events * @master: specifies the caller's willingness to be the device master * @state: returns the current device state (if non-NULL) * * Issues a command to establish communication with FW. Returns either * an error (negative integer) or the mailbox of the Master PF. */ int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox, enum dev_master master, enum dev_state *state) { int ret; struct fw_hello_cmd c; u32 v; unsigned int master_mbox; int retries = FW_CMD_HELLO_RETRIES; retry: memset(&c, 0, sizeof(c)); INIT_CMD(c, HELLO, WRITE); c.err_to_clearinit = cpu_to_be32( V_FW_HELLO_CMD_MASTERDIS(master == MASTER_CANT) | V_FW_HELLO_CMD_MASTERFORCE(master == MASTER_MUST) | V_FW_HELLO_CMD_MBMASTER(master == MASTER_MUST ? mbox : M_FW_HELLO_CMD_MBMASTER) | V_FW_HELLO_CMD_MBASYNCNOT(evt_mbox) | V_FW_HELLO_CMD_STAGE(FW_HELLO_CMD_STAGE_OS) | F_FW_HELLO_CMD_CLEARINIT); /* * Issue the HELLO command to the firmware. If it's not successful * but indicates that we got a "busy" or "timeout" condition, retry * the HELLO until we exhaust our retry limit. If we do exceed our * retry limit, check to see if the firmware left us any error * information and report that if so ... */ ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret != FW_SUCCESS) { if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0) goto retry; if (t4_read_reg(adap, A_PCIE_FW) & F_PCIE_FW_ERR) t4_report_fw_error(adap); return ret; } v = be32_to_cpu(c.err_to_clearinit); master_mbox = G_FW_HELLO_CMD_MBMASTER(v); if (state) { if (v & F_FW_HELLO_CMD_ERR) *state = DEV_STATE_ERR; else if (v & F_FW_HELLO_CMD_INIT) *state = DEV_STATE_INIT; else *state = DEV_STATE_UNINIT; } /* * If we're not the Master PF then we need to wait around for the * Master PF Driver to finish setting up the adapter. * * Note that we also do this wait if we're a non-Master-capable PF and * there is no current Master PF; a Master PF may show up momentarily * and we wouldn't want to fail pointlessly. (This can happen when an * OS loads lots of different drivers rapidly at the same time). In * this case, the Master PF returned by the firmware will be * M_PCIE_FW_MASTER so the test below will work ... */ if ((v & (F_FW_HELLO_CMD_ERR|F_FW_HELLO_CMD_INIT)) == 0 && master_mbox != mbox) { int waiting = FW_CMD_HELLO_TIMEOUT; /* * Wait for the firmware to either indicate an error or * initialized state. If we see either of these we bail out * and report the issue to the caller. If we exhaust the * "hello timeout" and we haven't exhausted our retries, try * again. Otherwise bail with a timeout error. */ for (;;) { u32 pcie_fw; msleep(50); waiting -= 50; /* * If neither Error nor Initialialized are indicated * by the firmware keep waiting till we exhaust our * timeout ... and then retry if we haven't exhausted * our retries ... */ pcie_fw = t4_read_reg(adap, A_PCIE_FW); if (!(pcie_fw & (F_PCIE_FW_ERR|F_PCIE_FW_INIT))) { if (waiting <= 0) { if (retries-- > 0) goto retry; return -ETIMEDOUT; } continue; } /* * We either have an Error or Initialized condition * report errors preferentially. */ if (state) { if (pcie_fw & F_PCIE_FW_ERR) *state = DEV_STATE_ERR; else if (pcie_fw & F_PCIE_FW_INIT) *state = DEV_STATE_INIT; } /* * If we arrived before a Master PF was selected and * there's not a valid Master PF, grab its identity * for our caller. */ if (master_mbox == M_PCIE_FW_MASTER && (pcie_fw & F_PCIE_FW_MASTER_VLD)) master_mbox = G_PCIE_FW_MASTER(pcie_fw); break; } } return master_mbox; } /** * t4_fw_bye - end communication with FW * @adap: the adapter * @mbox: mailbox to use for the FW command * * Issues a command to terminate communication with FW. */ int t4_fw_bye(struct adapter *adap, unsigned int mbox) { struct fw_bye_cmd c; memset(&c, 0, sizeof(c)); INIT_CMD(c, BYE, WRITE); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_fw_reset - issue a reset to FW * @adap: the adapter * @mbox: mailbox to use for the FW command * @reset: specifies the type of reset to perform * * Issues a reset command of the specified type to FW. */ int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset) { struct fw_reset_cmd c; memset(&c, 0, sizeof(c)); INIT_CMD(c, RESET, WRITE); c.val = cpu_to_be32(reset); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_fw_halt - issue a reset/halt to FW and put uP into RESET * @adap: the adapter * @mbox: mailbox to use for the FW RESET command (if desired) * @force: force uP into RESET even if FW RESET command fails * * Issues a RESET command to firmware (if desired) with a HALT indication * and then puts the microprocessor into RESET state. The RESET command * will only be issued if a legitimate mailbox is provided (mbox <= * M_PCIE_FW_MASTER). * * This is generally used in order for the host to safely manipulate the * adapter without fear of conflicting with whatever the firmware might * be doing. The only way out of this state is to RESTART the firmware * ... */ int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force) { int ret = 0; /* * If a legitimate mailbox is provided, issue a RESET command * with a HALT indication. */ if (mbox <= M_PCIE_FW_MASTER) { struct fw_reset_cmd c; memset(&c, 0, sizeof(c)); INIT_CMD(c, RESET, WRITE); c.val = cpu_to_be32(F_PIORST | F_PIORSTMODE); c.halt_pkd = cpu_to_be32(F_FW_RESET_CMD_HALT); ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /* * Normally we won't complete the operation if the firmware RESET * command fails but if our caller insists we'll go ahead and put the * uP into RESET. This can be useful if the firmware is hung or even * missing ... We'll have to take the risk of putting the uP into * RESET without the cooperation of firmware in that case. * * We also force the firmware's HALT flag to be on in case we bypassed * the firmware RESET command above or we're dealing with old firmware * which doesn't have the HALT capability. This will serve as a flag * for the incoming firmware to know that it's coming out of a HALT * rather than a RESET ... if it's new enough to understand that ... */ if (ret == 0 || force) { t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, F_UPCRST); t4_set_reg_field(adap, A_PCIE_FW, F_PCIE_FW_HALT, F_PCIE_FW_HALT); } /* * And we always return the result of the firmware RESET command * even when we force the uP into RESET ... */ return ret; } /** * t4_fw_restart - restart the firmware by taking the uP out of RESET * @adap: the adapter * @reset: if we want to do a RESET to restart things * * Restart firmware previously halted by t4_fw_halt(). On successful * return the previous PF Master remains as the new PF Master and there * is no need to issue a new HELLO command, etc. * * We do this in two ways: * * 1. If we're dealing with newer firmware we'll simply want to take * the chip's microprocessor out of RESET. This will cause the * firmware to start up from its start vector. And then we'll loop * until the firmware indicates it's started again (PCIE_FW.HALT * reset to 0) or we timeout. * * 2. If we're dealing with older firmware then we'll need to RESET * the chip since older firmware won't recognize the PCIE_FW.HALT * flag and automatically RESET itself on startup. */ int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset) { if (reset) { /* * Since we're directing the RESET instead of the firmware * doing it automatically, we need to clear the PCIE_FW.HALT * bit. */ t4_set_reg_field(adap, A_PCIE_FW, F_PCIE_FW_HALT, 0); /* * If we've been given a valid mailbox, first try to get the * firmware to do the RESET. If that works, great and we can * return success. Otherwise, if we haven't been given a * valid mailbox or the RESET command failed, fall back to * hitting the chip with a hammer. */ if (mbox <= M_PCIE_FW_MASTER) { t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, 0); msleep(100); if (t4_fw_reset(adap, mbox, F_PIORST | F_PIORSTMODE) == 0) return 0; } t4_write_reg(adap, A_PL_RST, F_PIORST | F_PIORSTMODE); msleep(2000); } else { int ms; t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, 0); for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) { if (!(t4_read_reg(adap, A_PCIE_FW) & F_PCIE_FW_HALT)) return FW_SUCCESS; msleep(100); ms += 100; } return -ETIMEDOUT; } return 0; } /** * t4_fw_upgrade - perform all of the steps necessary to upgrade FW * @adap: the adapter * @mbox: mailbox to use for the FW RESET command (if desired) * @fw_data: the firmware image to write * @size: image size * @force: force upgrade even if firmware doesn't cooperate * * Perform all of the steps necessary for upgrading an adapter's * firmware image. Normally this requires the cooperation of the * existing firmware in order to halt all existing activities * but if an invalid mailbox token is passed in we skip that step * (though we'll still put the adapter microprocessor into RESET in * that case). * * On successful return the new firmware will have been loaded and * the adapter will have been fully RESET losing all previous setup * state. On unsuccessful return the adapter may be completely hosed ... * positive errno indicates that the adapter is ~probably~ intact, a * negative errno indicates that things are looking bad ... */ int t4_fw_upgrade(struct adapter *adap, unsigned int mbox, const u8 *fw_data, unsigned int size, int force) { const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data; unsigned int bootstrap = be32_to_cpu(fw_hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP; int reset, ret; if (!t4_fw_matches_chip(adap, fw_hdr)) return -EINVAL; if (!bootstrap) { ret = t4_fw_halt(adap, mbox, force); if (ret < 0 && !force) return ret; } ret = t4_load_fw(adap, fw_data, size); if (ret < 0 || bootstrap) return ret; /* * Older versions of the firmware don't understand the new * PCIE_FW.HALT flag and so won't know to perform a RESET when they * restart. So for newly loaded older firmware we'll have to do the * RESET for it so it starts up on a clean slate. We can tell if * the newly loaded firmware will handle this right by checking * its header flags to see if it advertises the capability. */ reset = ((be32_to_cpu(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0); return t4_fw_restart(adap, mbox, reset); } /* * Card doesn't have a firmware, install one. */ int t4_fw_forceinstall(struct adapter *adap, const u8 *fw_data, unsigned int size) { const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data; unsigned int bootstrap = be32_to_cpu(fw_hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP; int ret; if (!t4_fw_matches_chip(adap, fw_hdr) || bootstrap) return -EINVAL; t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, F_UPCRST); t4_write_reg(adap, A_PCIE_FW, 0); /* Clobber internal state */ ret = t4_load_fw(adap, fw_data, size); if (ret < 0) return ret; t4_write_reg(adap, A_PL_RST, F_PIORST | F_PIORSTMODE); msleep(1000); return (0); } /** * t4_fw_initialize - ask FW to initialize the device * @adap: the adapter * @mbox: mailbox to use for the FW command * * Issues a command to FW to partially initialize the device. This * performs initialization that generally doesn't depend on user input. */ int t4_fw_initialize(struct adapter *adap, unsigned int mbox) { struct fw_initialize_cmd c; memset(&c, 0, sizeof(c)); INIT_CMD(c, INITIALIZE, WRITE); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_query_params_rw - query FW or device parameters * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF * @vf: the VF * @nparams: the number of parameters * @params: the parameter names * @val: the parameter values * @rw: Write and read flag * * Reads the value of FW or device parameters. Up to 7 parameters can be * queried at once. */ int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, u32 *val, int rw) { int i, ret; struct fw_params_cmd c; __be32 *p = &c.param[0].mnem; if (nparams > 7) return -EINVAL; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PARAMS_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_PARAMS_CMD_PFN(pf) | V_FW_PARAMS_CMD_VFN(vf)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); for (i = 0; i < nparams; i++) { *p++ = cpu_to_be32(*params++); if (rw) *p = cpu_to_be32(*(val + i)); p++; } ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret == 0) for (i = 0, p = &c.param[0].val; i < nparams; i++, p += 2) *val++ = be32_to_cpu(*p); return ret; } int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, u32 *val) { return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0); } /** * t4_set_params_timeout - sets FW or device parameters * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF * @vf: the VF * @nparams: the number of parameters * @params: the parameter names * @val: the parameter values * @timeout: the timeout time * * Sets the value of FW or device parameters. Up to 7 parameters can be * specified at once. */ int t4_set_params_timeout(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val, int timeout) { struct fw_params_cmd c; __be32 *p = &c.param[0].mnem; if (nparams > 7) return -EINVAL; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PARAMS_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_PARAMS_CMD_PFN(pf) | V_FW_PARAMS_CMD_VFN(vf)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); while (nparams--) { *p++ = cpu_to_be32(*params++); *p++ = cpu_to_be32(*val++); } return t4_wr_mbox_timeout(adap, mbox, &c, sizeof(c), NULL, timeout); } /** * t4_set_params - sets FW or device parameters * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF * @vf: the VF * @nparams: the number of parameters * @params: the parameter names * @val: the parameter values * * Sets the value of FW or device parameters. Up to 7 parameters can be * specified at once. */ int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val) { return t4_set_params_timeout(adap, mbox, pf, vf, nparams, params, val, FW_CMD_MAX_TIMEOUT); } /** * t4_cfg_pfvf - configure PF/VF resource limits * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF being configured * @vf: the VF being configured * @txq: the max number of egress queues * @txq_eth_ctrl: the max number of egress Ethernet or control queues * @rxqi: the max number of interrupt-capable ingress queues * @rxq: the max number of interruptless ingress queues * @tc: the PCI traffic class * @vi: the max number of virtual interfaces * @cmask: the channel access rights mask for the PF/VF * @pmask: the port access rights mask for the PF/VF * @nexact: the maximum number of exact MPS filters * @rcaps: read capabilities * @wxcaps: write/execute capabilities * * Configures resource limits and capabilities for a physical or virtual * function. */ int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl, unsigned int rxqi, unsigned int rxq, unsigned int tc, unsigned int vi, unsigned int cmask, unsigned int pmask, unsigned int nexact, unsigned int rcaps, unsigned int wxcaps) { struct fw_pfvf_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PFVF_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_PFVF_CMD_PFN(pf) | V_FW_PFVF_CMD_VFN(vf)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); c.niqflint_niq = cpu_to_be32(V_FW_PFVF_CMD_NIQFLINT(rxqi) | V_FW_PFVF_CMD_NIQ(rxq)); c.type_to_neq = cpu_to_be32(V_FW_PFVF_CMD_CMASK(cmask) | V_FW_PFVF_CMD_PMASK(pmask) | V_FW_PFVF_CMD_NEQ(txq)); c.tc_to_nexactf = cpu_to_be32(V_FW_PFVF_CMD_TC(tc) | V_FW_PFVF_CMD_NVI(vi) | V_FW_PFVF_CMD_NEXACTF(nexact)); c.r_caps_to_nethctrl = cpu_to_be32(V_FW_PFVF_CMD_R_CAPS(rcaps) | V_FW_PFVF_CMD_WX_CAPS(wxcaps) | V_FW_PFVF_CMD_NETHCTRL(txq_eth_ctrl)); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_alloc_vi_func - allocate a virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @port: physical port associated with the VI * @pf: the PF owning the VI * @vf: the VF owning the VI * @nmac: number of MAC addresses needed (1 to 5) * @mac: the MAC addresses of the VI * @rss_size: size of RSS table slice associated with this VI * @portfunc: which Port Application Function MAC Address is desired * @idstype: Intrusion Detection Type * * Allocates a virtual interface for the given physical port. If @mac is * not %NULL it contains the MAC addresses of the VI as assigned by FW. * If @rss_size is %NULL the VI is not assigned any RSS slice by FW. * @mac should be large enough to hold @nmac Ethernet addresses, they are * stored consecutively so the space needed is @nmac * 6 bytes. * Returns a negative error number or the non-negative VI id. */ int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox, unsigned int port, unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, u16 *rss_size, unsigned int portfunc, unsigned int idstype) { int ret; struct fw_vi_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_VI_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_VI_CMD_PFN(pf) | V_FW_VI_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_VI_CMD_ALLOC | FW_LEN16(c)); c.type_to_viid = cpu_to_be16(V_FW_VI_CMD_TYPE(idstype) | V_FW_VI_CMD_FUNC(portfunc)); c.portid_pkd = V_FW_VI_CMD_PORTID(port); c.nmac = nmac - 1; if(!rss_size) c.norss_rsssize = F_FW_VI_CMD_NORSS; ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret) return ret; if (mac) { memcpy(mac, c.mac, sizeof(c.mac)); switch (nmac) { case 5: memcpy(mac + 24, c.nmac3, sizeof(c.nmac3)); case 4: memcpy(mac + 18, c.nmac2, sizeof(c.nmac2)); case 3: memcpy(mac + 12, c.nmac1, sizeof(c.nmac1)); case 2: memcpy(mac + 6, c.nmac0, sizeof(c.nmac0)); } } if (rss_size) *rss_size = G_FW_VI_CMD_RSSSIZE(be16_to_cpu(c.norss_rsssize)); return G_FW_VI_CMD_VIID(be16_to_cpu(c.type_to_viid)); } /** * t4_alloc_vi - allocate an [Ethernet Function] virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @port: physical port associated with the VI * @pf: the PF owning the VI * @vf: the VF owning the VI * @nmac: number of MAC addresses needed (1 to 5) * @mac: the MAC addresses of the VI * @rss_size: size of RSS table slice associated with this VI * * backwards compatible and convieniance routine to allocate a Virtual * Interface with a Ethernet Port Application Function and Intrustion * Detection System disabled. */ int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port, unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, u16 *rss_size) { return t4_alloc_vi_func(adap, mbox, port, pf, vf, nmac, mac, rss_size, FW_VI_FUNC_ETH, 0); } /** * t4_free_vi - free a virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the VI * @vf: the VF owning the VI * @viid: virtual interface identifiler * * Free a previously allocated virtual interface. */ int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int viid) { struct fw_vi_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_VI_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_VI_CMD_PFN(pf) | V_FW_VI_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_VI_CMD_FREE | FW_LEN16(c)); c.type_to_viid = cpu_to_be16(V_FW_VI_CMD_VIID(viid)); return t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); } /** * t4_set_rxmode - set Rx properties of a virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @mtu: the new MTU or -1 * @promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change * @all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change * @bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change * @vlanex: 1 to enable HW VLAN extraction, 0 to disable it, -1 no change * @sleep_ok: if true we may sleep while awaiting command completion * * Sets Rx properties of a virtual interface. */ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid, int mtu, int promisc, int all_multi, int bcast, int vlanex, bool sleep_ok) { struct fw_vi_rxmode_cmd c; /* convert to FW values */ if (mtu < 0) mtu = M_FW_VI_RXMODE_CMD_MTU; if (promisc < 0) promisc = M_FW_VI_RXMODE_CMD_PROMISCEN; if (all_multi < 0) all_multi = M_FW_VI_RXMODE_CMD_ALLMULTIEN; if (bcast < 0) bcast = M_FW_VI_RXMODE_CMD_BROADCASTEN; if (vlanex < 0) vlanex = M_FW_VI_RXMODE_CMD_VLANEXEN; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_RXMODE_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_VI_RXMODE_CMD_VIID(viid)); c.retval_len16 = cpu_to_be32(FW_LEN16(c)); c.mtu_to_vlanexen = cpu_to_be32(V_FW_VI_RXMODE_CMD_MTU(mtu) | V_FW_VI_RXMODE_CMD_PROMISCEN(promisc) | V_FW_VI_RXMODE_CMD_ALLMULTIEN(all_multi) | V_FW_VI_RXMODE_CMD_BROADCASTEN(bcast) | V_FW_VI_RXMODE_CMD_VLANEXEN(vlanex)); return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok); } /** * t4_alloc_mac_filt - allocates exact-match filters for MAC addresses * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @free: if true any existing filters for this VI id are first removed * @naddr: the number of MAC addresses to allocate filters for (up to 7) * @addr: the MAC address(es) * @idx: where to store the index of each allocated filter * @hash: pointer to hash address filter bitmap * @sleep_ok: call is allowed to sleep * * Allocates an exact-match filter for each of the supplied addresses and * sets it to the corresponding address. If @idx is not %NULL it should * have at least @naddr entries, each of which will be set to the index of * the filter allocated for the corresponding MAC address. If a filter * could not be allocated for an address its index is set to 0xffff. * If @hash is not %NULL addresses that fail to allocate an exact filter * are hashed and update the hash filter bitmap pointed at by @hash. * * Returns a negative error number or the number of filters allocated. */ int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox, unsigned int viid, bool free, unsigned int naddr, const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok) { int offset, ret = 0; struct fw_vi_mac_cmd c; unsigned int nfilters = 0; unsigned int max_naddr = adap->chip_params->mps_tcam_size; unsigned int rem = naddr; if (naddr > max_naddr) return -EINVAL; for (offset = 0; offset < naddr ; /**/) { unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact) ? rem : ARRAY_SIZE(c.u.exact)); size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, u.exact[fw_naddr]), 16); struct fw_vi_mac_exact *p; int i; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_CMD_EXEC(free) | V_FW_VI_MAC_CMD_VIID(viid)); c.freemacs_to_len16 = cpu_to_be32(V_FW_VI_MAC_CMD_FREEMACS(free) | V_FW_CMD_LEN16(len16)); for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) { p->valid_to_idx = cpu_to_be16(F_FW_VI_MAC_CMD_VALID | V_FW_VI_MAC_CMD_IDX(FW_VI_MAC_ADD_MAC)); memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr)); } /* * It's okay if we run out of space in our MAC address arena. * Some of the addresses we submit may get stored so we need * to run through the reply to see what the results were ... */ ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok); if (ret && ret != -FW_ENOMEM) break; for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) { u16 index = G_FW_VI_MAC_CMD_IDX( be16_to_cpu(p->valid_to_idx)); if (idx) idx[offset+i] = (index >= max_naddr ? 0xffff : index); if (index < max_naddr) nfilters++; else if (hash) *hash |= (1ULL << hash_mac_addr(addr[offset+i])); } free = false; offset += fw_naddr; rem -= fw_naddr; } if (ret == 0 || ret == -FW_ENOMEM) ret = nfilters; return ret; } /** * t4_change_mac - modifies the exact-match filter for a MAC address * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @idx: index of existing filter for old value of MAC address, or -1 * @addr: the new MAC address value * @persist: whether a new MAC allocation should be persistent * @add_smt: if true also add the address to the HW SMT * * Modifies an exact-match filter and sets it to the new MAC address if * @idx >= 0, or adds the MAC address to a new filter if @idx < 0. In the * latter case the address is added persistently if @persist is %true. * * Note that in general it is not possible to modify the value of a given * filter so the generic way to modify an address filter is to free the one * being used by the old address value and allocate a new filter for the * new address value. * * Returns a negative error number or the index of the filter with the new * MAC value. Note that this index may differ from @idx. */ int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid, int idx, const u8 *addr, bool persist, bool add_smt) { int ret, mode; struct fw_vi_mac_cmd c; struct fw_vi_mac_exact *p = c.u.exact; unsigned int max_mac_addr = adap->chip_params->mps_tcam_size; if (idx < 0) /* new allocation */ idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC; mode = add_smt ? FW_VI_MAC_SMT_AND_MPSTCAM : FW_VI_MAC_MPS_TCAM_ENTRY; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_VI_MAC_CMD_VIID(viid)); c.freemacs_to_len16 = cpu_to_be32(V_FW_CMD_LEN16(1)); p->valid_to_idx = cpu_to_be16(F_FW_VI_MAC_CMD_VALID | V_FW_VI_MAC_CMD_SMAC_RESULT(mode) | V_FW_VI_MAC_CMD_IDX(idx)); memcpy(p->macaddr, addr, sizeof(p->macaddr)); ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret == 0) { ret = G_FW_VI_MAC_CMD_IDX(be16_to_cpu(p->valid_to_idx)); if (ret >= max_mac_addr) ret = -ENOMEM; } return ret; } /** * t4_set_addr_hash - program the MAC inexact-match hash filter * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @ucast: whether the hash filter should also match unicast addresses * @vec: the value to be written to the hash filter * @sleep_ok: call is allowed to sleep * * Sets the 64-bit inexact-match hash filter for a virtual interface. */ int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid, bool ucast, u64 vec, bool sleep_ok) { struct fw_vi_mac_cmd c; u32 val; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_VI_ENABLE_CMD_VIID(viid)); val = V_FW_VI_MAC_CMD_ENTRY_TYPE(FW_VI_MAC_TYPE_HASHVEC) | V_FW_VI_MAC_CMD_HASHUNIEN(ucast) | V_FW_CMD_LEN16(1); c.freemacs_to_len16 = cpu_to_be32(val); c.u.hash.hashvec = cpu_to_be64(vec); return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok); } /** * t4_enable_vi_params - enable/disable a virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @rx_en: 1=enable Rx, 0=disable Rx * @tx_en: 1=enable Tx, 0=disable Tx * @dcb_en: 1=enable delivery of Data Center Bridging messages. * * Enables/disables a virtual interface. Note that setting DCB Enable * only makes sense when enabling a Virtual Interface ... */ int t4_enable_vi_params(struct adapter *adap, unsigned int mbox, unsigned int viid, bool rx_en, bool tx_en, bool dcb_en) { struct fw_vi_enable_cmd c; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_ENABLE_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_VI_ENABLE_CMD_VIID(viid)); c.ien_to_len16 = cpu_to_be32(V_FW_VI_ENABLE_CMD_IEN(rx_en) | V_FW_VI_ENABLE_CMD_EEN(tx_en) | V_FW_VI_ENABLE_CMD_DCB_INFO(dcb_en) | FW_LEN16(c)); return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL); } /** * t4_enable_vi - enable/disable a virtual interface * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @rx_en: 1=enable Rx, 0=disable Rx * @tx_en: 1=enable Tx, 0=disable Tx * * Enables/disables a virtual interface. Note that setting DCB Enable * only makes sense when enabling a Virtual Interface ... */ int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid, bool rx_en, bool tx_en) { return t4_enable_vi_params(adap, mbox, viid, rx_en, tx_en, 0); } /** * t4_identify_port - identify a VI's port by blinking its LED * @adap: the adapter * @mbox: mailbox to use for the FW command * @viid: the VI id * @nblinks: how many times to blink LED at 2.5 Hz * * Identifies a VI's port by blinking its LED. */ int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid, unsigned int nblinks) { struct fw_vi_enable_cmd c; memset(&c, 0, sizeof(c)); c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_ENABLE_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_VI_ENABLE_CMD_VIID(viid)); c.ien_to_len16 = cpu_to_be32(F_FW_VI_ENABLE_CMD_LED | FW_LEN16(c)); c.blinkdur = cpu_to_be16(nblinks); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_iq_stop - stop an ingress queue and its FLs * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queues * @vf: the VF owning the queues * @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.) * @iqid: ingress queue id * @fl0id: FL0 queue id or 0xffff if no attached FL0 * @fl1id: FL1 queue id or 0xffff if no attached FL1 * * Stops an ingress queue and its associated FLs, if any. This causes * any current or future data/messages destined for these queues to be * tossed. */ int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id) { struct fw_iq_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(pf) | V_FW_IQ_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_IQSTOP | FW_LEN16(c)); c.type_to_iqandstindex = cpu_to_be32(V_FW_IQ_CMD_TYPE(iqtype)); c.iqid = cpu_to_be16(iqid); c.fl0id = cpu_to_be16(fl0id); c.fl1id = cpu_to_be16(fl1id); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_iq_free - free an ingress queue and its FLs * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queues * @vf: the VF owning the queues * @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.) * @iqid: ingress queue id * @fl0id: FL0 queue id or 0xffff if no attached FL0 * @fl1id: FL1 queue id or 0xffff if no attached FL1 * * Frees an ingress queue and its associated FLs, if any. */ int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id) { struct fw_iq_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(pf) | V_FW_IQ_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_FREE | FW_LEN16(c)); c.type_to_iqandstindex = cpu_to_be32(V_FW_IQ_CMD_TYPE(iqtype)); c.iqid = cpu_to_be16(iqid); c.fl0id = cpu_to_be16(fl0id); c.fl1id = cpu_to_be16(fl1id); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_eth_eq_free - free an Ethernet egress queue * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queue * @vf: the VF owning the queue * @eqid: egress queue id * * Frees an Ethernet egress queue. */ int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid) { struct fw_eq_eth_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(pf) | V_FW_EQ_ETH_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_EQ_ETH_CMD_FREE | FW_LEN16(c)); c.eqid_pkd = cpu_to_be32(V_FW_EQ_ETH_CMD_EQID(eqid)); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_ctrl_eq_free - free a control egress queue * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queue * @vf: the VF owning the queue * @eqid: egress queue id * * Frees a control egress queue. */ int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid) { struct fw_eq_ctrl_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(pf) | V_FW_EQ_CTRL_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_EQ_CTRL_CMD_FREE | FW_LEN16(c)); c.cmpliqid_eqid = cpu_to_be32(V_FW_EQ_CTRL_CMD_EQID(eqid)); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_ofld_eq_free - free an offload egress queue * @adap: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queue * @vf: the VF owning the queue * @eqid: egress queue id * * Frees a control egress queue. */ int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid) { struct fw_eq_ofld_cmd c; memset(&c, 0, sizeof(c)); c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(pf) | V_FW_EQ_OFLD_CMD_VFN(vf)); c.alloc_to_len16 = cpu_to_be32(F_FW_EQ_OFLD_CMD_FREE | FW_LEN16(c)); c.eqid_pkd = cpu_to_be32(V_FW_EQ_OFLD_CMD_EQID(eqid)); return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); } /** * t4_link_down_rc_str - return a string for a Link Down Reason Code * @link_down_rc: Link Down Reason Code * * Returns a string representation of the Link Down Reason Code. */ const char *t4_link_down_rc_str(unsigned char link_down_rc) { static const char *reason[] = { "Link Down", "Remote Fault", "Auto-negotiation Failure", "Reserved3", "Insufficient Airflow", "Unable To Determine Reason", "No RX Signal Detected", "Reserved7", }; if (link_down_rc >= ARRAY_SIZE(reason)) return "Bad Reason Code"; return reason[link_down_rc]; } /* * Updates all fields owned by the common code in port_info and link_config * based on information provided by the firmware. Does not touch any * requested_* field. */ static void handle_port_info(struct port_info *pi, const struct fw_port_info *p) { struct link_config *lc = &pi->link_cfg; int speed; unsigned char fc, fec; u32 stat = be32_to_cpu(p->lstatus_to_modtype); pi->port_type = G_FW_PORT_CMD_PTYPE(stat); pi->mod_type = G_FW_PORT_CMD_MODTYPE(stat); pi->mdio_addr = stat & F_FW_PORT_CMD_MDIOCAP ? G_FW_PORT_CMD_MDIOADDR(stat) : -1; lc->supported = be16_to_cpu(p->pcap); lc->advertising = be16_to_cpu(p->acap); lc->lp_advertising = be16_to_cpu(p->lpacap); lc->link_ok = (stat & F_FW_PORT_CMD_LSTATUS) != 0; lc->link_down_rc = G_FW_PORT_CMD_LINKDNRC(stat); speed = 0; if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_100M)) speed = 100; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_1G)) speed = 1000; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_10G)) speed = 10000; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_25G)) speed = 25000; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_40G)) speed = 40000; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_100G)) speed = 100000; lc->speed = speed; fc = 0; if (stat & F_FW_PORT_CMD_RXPAUSE) fc |= PAUSE_RX; if (stat & F_FW_PORT_CMD_TXPAUSE) fc |= PAUSE_TX; lc->fc = fc; fec = 0; if (lc->advertising & FW_PORT_CAP_FEC_RS) fec |= FEC_RS; if (lc->advertising & FW_PORT_CAP_FEC_BASER_RS) fec |= FEC_BASER_RS; if (lc->advertising & FW_PORT_CAP_FEC_RESERVED) fec |= FEC_RESERVED; lc->fec = fec; } /** * t4_update_port_info - retrieve and update port information if changed * @pi: the port_info * * We issue a Get Port Information Command to the Firmware and, if * successful, we check to see if anything is different from what we * last recorded and update things accordingly. */ int t4_update_port_info(struct port_info *pi) { struct fw_port_cmd port_cmd; int ret; memset(&port_cmd, 0, sizeof port_cmd); port_cmd.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_PORT_CMD_PORTID(pi->tx_chan)); port_cmd.action_to_len16 = cpu_to_be32( V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_GET_PORT_INFO) | FW_LEN16(port_cmd)); ret = t4_wr_mbox_ns(pi->adapter, pi->adapter->mbox, &port_cmd, sizeof(port_cmd), &port_cmd); if (ret) return ret; handle_port_info(pi, &port_cmd.u.info); return 0; } /** * t4_handle_fw_rpl - process a FW reply message * @adap: the adapter * @rpl: start of the FW message * * Processes a FW message, such as link state change messages. */ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) { u8 opcode = *(const u8 *)rpl; const struct fw_port_cmd *p = (const void *)rpl; unsigned int action = G_FW_PORT_CMD_ACTION(be32_to_cpu(p->action_to_len16)); if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) { /* link/module state change message */ int i, old_ptype, old_mtype; int chan = G_FW_PORT_CMD_PORTID(be32_to_cpu(p->op_to_portid)); struct port_info *pi = NULL; struct link_config *lc, old_lc; for_each_port(adap, i) { pi = adap2pinfo(adap, i); if (pi->tx_chan == chan) break; } lc = &pi->link_cfg; old_lc = *lc; old_ptype = pi->port_type; old_mtype = pi->mod_type; handle_port_info(pi, &p->u.info); if (old_ptype != pi->port_type || old_mtype != pi->mod_type) { t4_os_portmod_changed(pi, old_ptype, old_mtype, &old_lc); } if (old_lc.link_ok != lc->link_ok || old_lc.speed != lc->speed || old_lc.fc != lc->fc) { t4_os_link_changed(pi, &old_lc); } } else { CH_WARN_RATELIMIT(adap, "Unknown firmware reply %d\n", opcode); return -EINVAL; } return 0; } /** * get_pci_mode - determine a card's PCI mode * @adapter: the adapter * @p: where to store the PCI settings * * Determines a card's PCI mode and associated parameters, such as speed * and width. */ static void get_pci_mode(struct adapter *adapter, struct pci_params *p) { u16 val; u32 pcie_cap; pcie_cap = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP); if (pcie_cap) { t4_os_pci_read_cfg2(adapter, pcie_cap + PCI_EXP_LNKSTA, &val); p->speed = val & PCI_EXP_LNKSTA_CLS; p->width = (val & PCI_EXP_LNKSTA_NLW) >> 4; } } struct flash_desc { u32 vendor_and_model_id; u32 size_mb; }; int t4_get_flash_params(struct adapter *adapter) { /* - * Table for non-Numonix supported flash parts. Numonix parts are left - * to the preexisting well-tested code. All flash parts have 64KB - * sectors. + * Table for non-standard supported Flash parts. Note, all Flash + * parts must have 64KB sectors. */ static struct flash_desc supported_flash[] = { - { 0x150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ + { 0x00150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ }; int ret; - u32 info = 0; + u32 flashid = 0; + unsigned int part, manufacturer; + unsigned int density, size; + + /* + * Issue a Read ID Command to the Flash part. We decode supported + * Flash parts and their sizes from this. There's a newer Query + * Command which can retrieve detailed geometry information but many + * Flash parts don't support it. + */ ret = sf1_write(adapter, 1, 1, 0, SF_RD_ID); if (!ret) - ret = sf1_read(adapter, 3, 0, 1, &info); + ret = sf1_read(adapter, 3, 0, 1, &flashid); t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ if (ret < 0) return ret; - for (ret = 0; ret < ARRAY_SIZE(supported_flash); ++ret) - if (supported_flash[ret].vendor_and_model_id == info) { - adapter->params.sf_size = supported_flash[ret].size_mb; + /* + * Check to see if it's one of our non-standard supported Flash parts. + */ + for (part = 0; part < ARRAY_SIZE(supported_flash); part++) + if (supported_flash[part].vendor_and_model_id == flashid) { + adapter->params.sf_size = + supported_flash[part].size_mb; adapter->params.sf_nsec = adapter->params.sf_size / SF_SEC_SIZE; - return 0; + goto found; } - if ((info & 0xff) != 0x20) /* not a Numonix flash */ + /* + * Decode Flash part size. The code below looks repetative with + * common encodings, but that's not guaranteed in the JEDEC + * specification for the Read JADEC ID command. The only thing that + * we're guaranteed by the JADEC specification is where the + * Manufacturer ID is in the returned result. After that each + * Manufacturer ~could~ encode things completely differently. + * Note, all Flash parts must have 64KB sectors. + */ + manufacturer = flashid & 0xff; + switch (manufacturer) { + case 0x20: { /* Micron/Numonix */ + /* + * This Density -> Size decoding table is taken from Micron + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x14: size = 1 << 20; break; /* 1MB */ + case 0x15: size = 1 << 21; break; /* 2MB */ + case 0x16: size = 1 << 22; break; /* 4MB */ + case 0x17: size = 1 << 23; break; /* 8MB */ + case 0x18: size = 1 << 24; break; /* 16MB */ + case 0x19: size = 1 << 25; break; /* 32MB */ + case 0x20: size = 1 << 26; break; /* 64MB */ + case 0x21: size = 1 << 27; break; /* 128MB */ + case 0x22: size = 1 << 28; break; /* 256MB */ + + default: + CH_ERR(adapter, "Micron Flash Part has bad size, " + "ID = %#x, Density code = %#x\n", + flashid, density); + return -EINVAL; + } + break; + } + + case 0xef: { /* Winbond */ + /* + * This Density -> Size decoding table is taken from Winbond + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x17: size = 1 << 23; break; /* 8MB */ + case 0x18: size = 1 << 24; break; /* 16MB */ + + default: + CH_ERR(adapter, "Winbond Flash Part has bad size, " + "ID = %#x, Density code = %#x\n", + flashid, density); + return -EINVAL; + } + break; + } + + default: + CH_ERR(adapter, "Unsupported Flash Part, ID = %#x\n", flashid); return -EINVAL; - info >>= 16; /* log2 of size */ - if (info >= 0x14 && info < 0x18) - adapter->params.sf_nsec = 1 << (info - 16); - else if (info == 0x18) - adapter->params.sf_nsec = 64; - else - return -EINVAL; - adapter->params.sf_size = 1 << info; + } /* + * Store decoded Flash size and fall through into vetting code. + */ + adapter->params.sf_size = size; + adapter->params.sf_nsec = size / SF_SEC_SIZE; + + found: + /* * We should ~probably~ reject adapters with FLASHes which are too * small but we have some legacy FPGAs with small FLASHes that we'd * still like to use. So instead we emit a scary message ... */ if (adapter->params.sf_size < FLASH_MIN_SIZE) - CH_WARN(adapter, "WARNING!!! FLASH size %#x < %#x!!!\n", - adapter->params.sf_size, FLASH_MIN_SIZE); + CH_WARN(adapter, "WARNING: Flash Part ID %#x, size %#x < %#x\n", + flashid, adapter->params.sf_size, FLASH_MIN_SIZE); return 0; } static void set_pcie_completion_timeout(struct adapter *adapter, u8 range) { u16 val; u32 pcie_cap; pcie_cap = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP); if (pcie_cap) { t4_os_pci_read_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, &val); val &= 0xfff0; val |= range ; t4_os_pci_write_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, val); } } const struct chip_params *t4_get_chip_params(int chipid) { static const struct chip_params chip_params[] = { { /* T4 */ .nchan = NCHAN, .pm_stats_cnt = PM_NSTATS, .cng_ch_bits_log = 2, .nsched_cls = 15, .cim_num_obq = CIM_NUM_OBQ, .mps_rplc_size = 128, .vfcount = 128, .sge_fl_db = F_DBPRIO, .mps_tcam_size = NUM_MPS_CLS_SRAM_L_INSTANCES, }, { /* T5 */ .nchan = NCHAN, .pm_stats_cnt = PM_NSTATS, .cng_ch_bits_log = 2, .nsched_cls = 16, .cim_num_obq = CIM_NUM_OBQ_T5, .mps_rplc_size = 128, .vfcount = 128, .sge_fl_db = F_DBPRIO | F_DBTYPE, .mps_tcam_size = NUM_MPS_T5_CLS_SRAM_L_INSTANCES, }, { /* T6 */ .nchan = T6_NCHAN, .pm_stats_cnt = T6_PM_NSTATS, .cng_ch_bits_log = 3, .nsched_cls = 16, .cim_num_obq = CIM_NUM_OBQ_T5, .mps_rplc_size = 256, .vfcount = 256, .sge_fl_db = 0, .mps_tcam_size = NUM_MPS_T5_CLS_SRAM_L_INSTANCES, }, }; chipid -= CHELSIO_T4; if (chipid < 0 || chipid >= ARRAY_SIZE(chip_params)) return NULL; return &chip_params[chipid]; } /** * t4_prep_adapter - prepare SW and HW for operation * @adapter: the adapter * @buf: temporary space of at least VPD_LEN size provided by the caller. * * Initialize adapter SW state for the various HW modules, set initial * values for some adapter tunables, take PHYs out of reset, and * initialize the MDIO interface. */ int t4_prep_adapter(struct adapter *adapter, u8 *buf) { int ret; uint16_t device_id; uint32_t pl_rev; get_pci_mode(adapter, &adapter->params.pci); pl_rev = t4_read_reg(adapter, A_PL_REV); adapter->params.chipid = G_CHIPID(pl_rev); adapter->params.rev = G_REV(pl_rev); if (adapter->params.chipid == 0) { /* T4 did not have chipid in PL_REV (T5 onwards do) */ adapter->params.chipid = CHELSIO_T4; /* T4A1 chip is not supported */ if (adapter->params.rev == 1) { CH_ALERT(adapter, "T4 rev 1 chip is not supported.\n"); return -EINVAL; } } adapter->chip_params = t4_get_chip_params(chip_id(adapter)); if (adapter->chip_params == NULL) return -EINVAL; adapter->params.pci.vpd_cap_addr = t4_os_find_pci_capability(adapter, PCI_CAP_ID_VPD); ret = t4_get_flash_params(adapter); if (ret < 0) return ret; ret = get_vpd_params(adapter, &adapter->params.vpd, buf); if (ret < 0) return ret; /* Cards with real ASICs have the chipid in the PCIe device id */ t4_os_pci_read_cfg2(adapter, PCI_DEVICE_ID, &device_id); if (device_id >> 12 == chip_id(adapter)) adapter->params.cim_la_size = CIMLA_SIZE; else { /* FPGA */ adapter->params.fpga = 1; adapter->params.cim_la_size = 2 * CIMLA_SIZE; } init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd); /* * Default port and clock for debugging in case we can't reach FW. */ adapter->params.nports = 1; adapter->params.portvec = 1; adapter->params.vpd.cclk = 50000; /* Set pci completion timeout value to 4 seconds. */ set_pcie_completion_timeout(adapter, 0xd); return 0; } /** * t4_shutdown_adapter - shut down adapter, host & wire * @adapter: the adapter * * Perform an emergency shutdown of the adapter and stop it from * continuing any further communication on the ports or DMA to the * host. This is typically used when the adapter and/or firmware * have crashed and we want to prevent any further accidental * communication with the rest of the world. This will also force * the port Link Status to go down -- if register writes work -- * which should help our peers figure out that we're down. */ int t4_shutdown_adapter(struct adapter *adapter) { int port; t4_intr_disable(adapter); t4_write_reg(adapter, A_DBG_GPIO_EN, 0); for_each_port(adapter, port) { - u32 a_port_cfg = PORT_REG(port, - is_t4(adapter) - ? A_XGMAC_PORT_CFG - : A_MAC_PORT_CFG); + u32 a_port_cfg = is_t4(adapter) ? + PORT_REG(port, A_XGMAC_PORT_CFG) : + T5_PORT_REG(port, A_MAC_PORT_CFG); t4_write_reg(adapter, a_port_cfg, t4_read_reg(adapter, a_port_cfg) & ~V_SIGNAL_DET(1)); } t4_set_reg_field(adapter, A_SGE_CONTROL, F_GLOBALENABLE, 0); return 0; } /** * t4_init_devlog_params - initialize adapter->params.devlog * @adap: the adapter * @fw_attach: whether we can talk to the firmware * * Initialize various fields of the adapter's Firmware Device Log * Parameters structure. */ int t4_init_devlog_params(struct adapter *adap, int fw_attach) { struct devlog_params *dparams = &adap->params.devlog; u32 pf_dparams; unsigned int devlog_meminfo; struct fw_devlog_cmd devlog_cmd; int ret; /* If we're dealing with newer firmware, the Device Log Paramerters * are stored in a designated register which allows us to access the * Device Log even if we can't talk to the firmware. */ pf_dparams = t4_read_reg(adap, PCIE_FW_REG(A_PCIE_FW_PF, PCIE_FW_PF_DEVLOG)); if (pf_dparams) { unsigned int nentries, nentries128; dparams->memtype = G_PCIE_FW_PF_DEVLOG_MEMTYPE(pf_dparams); dparams->start = G_PCIE_FW_PF_DEVLOG_ADDR16(pf_dparams) << 4; nentries128 = G_PCIE_FW_PF_DEVLOG_NENTRIES128(pf_dparams); nentries = (nentries128 + 1) * 128; dparams->size = nentries * sizeof(struct fw_devlog_e); return 0; } /* * For any failing returns ... */ memset(dparams, 0, sizeof *dparams); /* * If we can't talk to the firmware, there's really nothing we can do * at this point. */ if (!fw_attach) return -ENXIO; /* Otherwise, ask the firmware for it's Device Log Parameters. */ memset(&devlog_cmd, 0, sizeof devlog_cmd); devlog_cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_DEVLOG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd)); ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd), &devlog_cmd); if (ret) return ret; devlog_meminfo = be32_to_cpu(devlog_cmd.memtype_devlog_memaddr16_devlog); dparams->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(devlog_meminfo); dparams->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(devlog_meminfo) << 4; dparams->size = be32_to_cpu(devlog_cmd.memsize_devlog); return 0; } /** * t4_init_sge_params - initialize adap->params.sge * @adapter: the adapter * * Initialize various fields of the adapter's SGE Parameters structure. */ int t4_init_sge_params(struct adapter *adapter) { u32 r; struct sge_params *sp = &adapter->params.sge; unsigned i, tscale = 1; r = t4_read_reg(adapter, A_SGE_INGRESS_RX_THRESHOLD); sp->counter_val[0] = G_THRESHOLD_0(r); sp->counter_val[1] = G_THRESHOLD_1(r); sp->counter_val[2] = G_THRESHOLD_2(r); sp->counter_val[3] = G_THRESHOLD_3(r); if (chip_id(adapter) >= CHELSIO_T6) { r = t4_read_reg(adapter, A_SGE_ITP_CONTROL); tscale = G_TSCALE(r); if (tscale == 0) tscale = 1; else tscale += 2; } r = t4_read_reg(adapter, A_SGE_TIMER_VALUE_0_AND_1); sp->timer_val[0] = core_ticks_to_us(adapter, G_TIMERVALUE0(r)) * tscale; sp->timer_val[1] = core_ticks_to_us(adapter, G_TIMERVALUE1(r)) * tscale; r = t4_read_reg(adapter, A_SGE_TIMER_VALUE_2_AND_3); sp->timer_val[2] = core_ticks_to_us(adapter, G_TIMERVALUE2(r)) * tscale; sp->timer_val[3] = core_ticks_to_us(adapter, G_TIMERVALUE3(r)) * tscale; r = t4_read_reg(adapter, A_SGE_TIMER_VALUE_4_AND_5); sp->timer_val[4] = core_ticks_to_us(adapter, G_TIMERVALUE4(r)) * tscale; sp->timer_val[5] = core_ticks_to_us(adapter, G_TIMERVALUE5(r)) * tscale; r = t4_read_reg(adapter, A_SGE_CONM_CTRL); sp->fl_starve_threshold = G_EGRTHRESHOLD(r) * 2 + 1; if (is_t4(adapter)) sp->fl_starve_threshold2 = sp->fl_starve_threshold; else if (is_t5(adapter)) sp->fl_starve_threshold2 = G_EGRTHRESHOLDPACKING(r) * 2 + 1; else sp->fl_starve_threshold2 = G_T6_EGRTHRESHOLDPACKING(r) * 2 + 1; /* egress queues: log2 of # of doorbells per BAR2 page */ r = t4_read_reg(adapter, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); r >>= S_QUEUESPERPAGEPF0 + (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * adapter->pf; sp->eq_s_qpp = r & M_QUEUESPERPAGEPF0; /* ingress queues: log2 of # of doorbells per BAR2 page */ r = t4_read_reg(adapter, A_SGE_INGRESS_QUEUES_PER_PAGE_PF); r >>= S_QUEUESPERPAGEPF0 + (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * adapter->pf; sp->iq_s_qpp = r & M_QUEUESPERPAGEPF0; r = t4_read_reg(adapter, A_SGE_HOST_PAGE_SIZE); r >>= S_HOSTPAGESIZEPF0 + (S_HOSTPAGESIZEPF1 - S_HOSTPAGESIZEPF0) * adapter->pf; sp->page_shift = (r & M_HOSTPAGESIZEPF0) + 10; r = t4_read_reg(adapter, A_SGE_CONTROL); sp->sge_control = r; sp->spg_len = r & F_EGRSTATUSPAGESIZE ? 128 : 64; sp->fl_pktshift = G_PKTSHIFT(r); if (chip_id(adapter) <= CHELSIO_T5) { sp->pad_boundary = 1 << (G_INGPADBOUNDARY(r) + X_INGPADBOUNDARY_SHIFT); } else { sp->pad_boundary = 1 << (G_INGPADBOUNDARY(r) + X_T6_INGPADBOUNDARY_SHIFT); } if (is_t4(adapter)) sp->pack_boundary = sp->pad_boundary; else { r = t4_read_reg(adapter, A_SGE_CONTROL2); if (G_INGPACKBOUNDARY(r) == 0) sp->pack_boundary = 16; else sp->pack_boundary = 1 << (G_INGPACKBOUNDARY(r) + 5); } for (i = 0; i < SGE_FLBUF_SIZES; i++) sp->sge_fl_buffer_size[i] = t4_read_reg(adapter, A_SGE_FL_BUFFER_SIZE0 + (4 * i)); return 0; } /* * Read and cache the adapter's compressed filter mode and ingress config. */ -static void read_filter_mode_and_ingress_config(struct adapter *adap) +static void read_filter_mode_and_ingress_config(struct adapter *adap, + bool sleep_ok) { struct tp_params *tpp = &adap->params.tp; - if (t4_use_ldst(adap)) { - t4_fw_tp_pio_rw(adap, &tpp->vlan_pri_map, 1, - A_TP_VLAN_PRI_MAP, 1); - t4_fw_tp_pio_rw(adap, &tpp->ingress_config, 1, - A_TP_INGRESS_CONFIG, 1); - } else { - t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &tpp->vlan_pri_map, 1, A_TP_VLAN_PRI_MAP); - t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, - &tpp->ingress_config, 1, A_TP_INGRESS_CONFIG); - } + t4_tp_pio_read(adap, &tpp->vlan_pri_map, 1, A_TP_VLAN_PRI_MAP, + sleep_ok); + t4_tp_pio_read(adap, &tpp->ingress_config, 1, A_TP_INGRESS_CONFIG, + sleep_ok); /* * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field * shift positions of several elements of the Compressed Filter Tuple * for this adapter which we need frequently ... */ tpp->fcoe_shift = t4_filter_field_shift(adap, F_FCOE); tpp->port_shift = t4_filter_field_shift(adap, F_PORT); tpp->vnic_shift = t4_filter_field_shift(adap, F_VNIC_ID); tpp->vlan_shift = t4_filter_field_shift(adap, F_VLAN); tpp->tos_shift = t4_filter_field_shift(adap, F_TOS); tpp->protocol_shift = t4_filter_field_shift(adap, F_PROTOCOL); tpp->ethertype_shift = t4_filter_field_shift(adap, F_ETHERTYPE); tpp->macmatch_shift = t4_filter_field_shift(adap, F_MACMATCH); tpp->matchtype_shift = t4_filter_field_shift(adap, F_MPSHITTYPE); tpp->frag_shift = t4_filter_field_shift(adap, F_FRAGMENTATION); /* * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID * represents the presence of an Outer VLAN instead of a VNIC ID. */ if ((tpp->ingress_config & F_VNIC) == 0) tpp->vnic_shift = -1; } /** * t4_init_tp_params - initialize adap->params.tp * @adap: the adapter * * Initialize various fields of the adapter's TP Parameters structure. */ -int t4_init_tp_params(struct adapter *adap) +int t4_init_tp_params(struct adapter *adap, bool sleep_ok) { int chan; u32 v; struct tp_params *tpp = &adap->params.tp; v = t4_read_reg(adap, A_TP_TIMER_RESOLUTION); tpp->tre = G_TIMERRESOLUTION(v); tpp->dack_re = G_DELAYEDACKRESOLUTION(v); /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */ for (chan = 0; chan < MAX_NCHAN; chan++) tpp->tx_modq[chan] = chan; - read_filter_mode_and_ingress_config(adap); + read_filter_mode_and_ingress_config(adap, sleep_ok); /* * Cache a mask of the bits that represent the error vector portion of * rx_pkt.err_vec. T6+ can use a compressed error vector to make room * for information about outer encapsulation (GENEVE/VXLAN/NVGRE). */ tpp->err_vec_mask = htobe16(0xffff); if (chip_id(adap) > CHELSIO_T5) { v = t4_read_reg(adap, A_TP_OUT_CONFIG); if (v & F_CRXPKTENC) { tpp->err_vec_mask = htobe16(V_T6_COMPR_RXERR_VEC(M_T6_COMPR_RXERR_VEC)); } } return 0; } /** * t4_filter_field_shift - calculate filter field shift * @adap: the adapter * @filter_sel: the desired field (from TP_VLAN_PRI_MAP bits) * * Return the shift position of a filter field within the Compressed * Filter Tuple. The filter field is specified via its selection bit * within TP_VLAN_PRI_MAL (filter mode). E.g. F_VLAN. */ int t4_filter_field_shift(const struct adapter *adap, int filter_sel) { unsigned int filter_mode = adap->params.tp.vlan_pri_map; unsigned int sel; int field_shift; if ((filter_mode & filter_sel) == 0) return -1; for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) { switch (filter_mode & sel) { case F_FCOE: field_shift += W_FT_FCOE; break; case F_PORT: field_shift += W_FT_PORT; break; case F_VNIC_ID: field_shift += W_FT_VNIC_ID; break; case F_VLAN: field_shift += W_FT_VLAN; break; case F_TOS: field_shift += W_FT_TOS; break; case F_PROTOCOL: field_shift += W_FT_PROTOCOL; break; case F_ETHERTYPE: field_shift += W_FT_ETHERTYPE; break; case F_MACMATCH: field_shift += W_FT_MACMATCH; break; case F_MPSHITTYPE: field_shift += W_FT_MPSHITTYPE; break; case F_FRAGMENTATION: field_shift += W_FT_FRAGMENTATION; break; } } return field_shift; } int t4_port_init(struct adapter *adap, int mbox, int pf, int vf, int port_id) { u8 addr[6]; int ret, i, j; struct fw_port_cmd c; u16 rss_size; struct port_info *p = adap2pinfo(adap, port_id); u32 param, val; memset(&c, 0, sizeof(c)); for (i = 0, j = -1; i <= p->port_id; i++) { do { j++; } while ((adap->params.portvec & (1 << j)) == 0); } if (!(adap->flags & IS_VF) || adap->params.vfres.r_caps & FW_CMD_CAP_PORT) { t4_update_port_info(p); } ret = t4_alloc_vi(adap, mbox, j, pf, vf, 1, addr, &rss_size); if (ret < 0) return ret; p->vi[0].viid = ret; if (chip_id(adap) <= CHELSIO_T5) p->vi[0].smt_idx = (ret & 0x7f) << 1; else p->vi[0].smt_idx = (ret & 0x7f); p->tx_chan = j; p->rx_chan_map = t4_get_mps_bg_map(adap, j); p->lport = j; p->vi[0].rss_size = rss_size; t4_os_set_hw_addr(p, addr); param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | V_FW_PARAMS_PARAM_YZ(p->vi[0].viid); ret = t4_query_params(adap, mbox, pf, vf, 1, ¶m, &val); if (ret) p->vi[0].rss_base = 0xffff; else { /* MPASS((val >> 16) == rss_size); */ p->vi[0].rss_base = val & 0xffff; } return 0; } /** * t4_read_cimq_cfg - read CIM queue configuration * @adap: the adapter * @base: holds the queue base addresses in bytes * @size: holds the queue sizes in bytes * @thres: holds the queue full thresholds in bytes * * Returns the current configuration of the CIM queues, starting with * the IBQs, then the OBQs. */ void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres) { unsigned int i, v; int cim_num_obq = adap->chip_params->cim_num_obq; for (i = 0; i < CIM_NUM_IBQ; i++) { t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_IBQSELECT | V_QUENUMSELECT(i)); v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL); /* value is in 256-byte units */ *base++ = G_CIMQBASE(v) * 256; *size++ = G_CIMQSIZE(v) * 256; *thres++ = G_QUEFULLTHRSH(v) * 8; /* 8-byte unit */ } for (i = 0; i < cim_num_obq; i++) { t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_OBQSELECT | V_QUENUMSELECT(i)); v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL); /* value is in 256-byte units */ *base++ = G_CIMQBASE(v) * 256; *size++ = G_CIMQSIZE(v) * 256; } } /** * t4_read_cim_ibq - read the contents of a CIM inbound queue * @adap: the adapter * @qid: the queue index * @data: where to store the queue contents * @n: capacity of @data in 32-bit words * * Reads the contents of the selected CIM queue starting at address 0 up * to the capacity of @data. @n must be a multiple of 4. Returns < 0 on * error and the number of 32-bit words actually read on success. */ int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n) { int i, err, attempts; unsigned int addr; const unsigned int nwords = CIM_IBQ_SIZE * 4; if (qid > 5 || (n & 3)) return -EINVAL; addr = qid * nwords; if (n > nwords) n = nwords; /* It might take 3-10ms before the IBQ debug read access is allowed. * Wait for 1 Sec with a delay of 1 usec. */ attempts = 1000000; for (i = 0; i < n; i++, addr++) { t4_write_reg(adap, A_CIM_IBQ_DBG_CFG, V_IBQDBGADDR(addr) | F_IBQDBGEN); err = t4_wait_op_done(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGBUSY, 0, attempts, 1); if (err) return err; *data++ = t4_read_reg(adap, A_CIM_IBQ_DBG_DATA); } t4_write_reg(adap, A_CIM_IBQ_DBG_CFG, 0); return i; } /** * t4_read_cim_obq - read the contents of a CIM outbound queue * @adap: the adapter * @qid: the queue index * @data: where to store the queue contents * @n: capacity of @data in 32-bit words * * Reads the contents of the selected CIM queue starting at address 0 up * to the capacity of @data. @n must be a multiple of 4. Returns < 0 on * error and the number of 32-bit words actually read on success. */ int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n) { int i, err; unsigned int addr, v, nwords; int cim_num_obq = adap->chip_params->cim_num_obq; if ((qid > (cim_num_obq - 1)) || (n & 3)) return -EINVAL; t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_OBQSELECT | V_QUENUMSELECT(qid)); v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL); addr = G_CIMQBASE(v) * 64; /* muliple of 256 -> muliple of 4 */ nwords = G_CIMQSIZE(v) * 64; /* same */ if (n > nwords) n = nwords; for (i = 0; i < n; i++, addr++) { t4_write_reg(adap, A_CIM_OBQ_DBG_CFG, V_OBQDBGADDR(addr) | F_OBQDBGEN); err = t4_wait_op_done(adap, A_CIM_OBQ_DBG_CFG, F_OBQDBGBUSY, 0, 2, 1); if (err) return err; *data++ = t4_read_reg(adap, A_CIM_OBQ_DBG_DATA); } t4_write_reg(adap, A_CIM_OBQ_DBG_CFG, 0); return i; } enum { CIM_QCTL_BASE = 0, CIM_CTL_BASE = 0x2000, CIM_PBT_ADDR_BASE = 0x2800, CIM_PBT_LRF_BASE = 0x3000, CIM_PBT_DATA_BASE = 0x3800 }; /** * t4_cim_read - read a block from CIM internal address space * @adap: the adapter * @addr: the start address within the CIM address space * @n: number of words to read * @valp: where to store the result * * Reads a block of 4-byte words from the CIM intenal address space. */ int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n, unsigned int *valp) { int ret = 0; if (t4_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY) return -EBUSY; for ( ; !ret && n--; addr += 4) { t4_write_reg(adap, A_CIM_HOST_ACC_CTRL, addr); ret = t4_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY, 0, 5, 2); if (!ret) *valp++ = t4_read_reg(adap, A_CIM_HOST_ACC_DATA); } return ret; } /** * t4_cim_write - write a block into CIM internal address space * @adap: the adapter * @addr: the start address within the CIM address space * @n: number of words to write * @valp: set of values to write * * Writes a block of 4-byte words into the CIM intenal address space. */ int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n, const unsigned int *valp) { int ret = 0; if (t4_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY) return -EBUSY; for ( ; !ret && n--; addr += 4) { t4_write_reg(adap, A_CIM_HOST_ACC_DATA, *valp++); t4_write_reg(adap, A_CIM_HOST_ACC_CTRL, addr | F_HOSTWRITE); ret = t4_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY, 0, 5, 2); } return ret; } static int t4_cim_write1(struct adapter *adap, unsigned int addr, unsigned int val) { return t4_cim_write(adap, addr, 1, &val); } /** * t4_cim_ctl_read - read a block from CIM control region * @adap: the adapter * @addr: the start address within the CIM control region * @n: number of words to read * @valp: where to store the result * * Reads a block of 4-byte words from the CIM control region. */ int t4_cim_ctl_read(struct adapter *adap, unsigned int addr, unsigned int n, unsigned int *valp) { return t4_cim_read(adap, addr + CIM_CTL_BASE, n, valp); } /** * t4_cim_read_la - read CIM LA capture buffer * @adap: the adapter * @la_buf: where to store the LA data * @wrptr: the HW write pointer within the capture buffer * * Reads the contents of the CIM LA buffer with the most recent entry at * the end of the returned data and with the entry at @wrptr first. * We try to leave the LA in the running state we find it in. */ int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr) { int i, ret; unsigned int cfg, val, idx; ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &cfg); if (ret) return ret; if (cfg & F_UPDBGLAEN) { /* LA is running, freeze it */ ret = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG, 0); if (ret) return ret; } ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &val); if (ret) goto restart; idx = G_UPDBGLAWRPTR(val); if (wrptr) *wrptr = idx; for (i = 0; i < adap->params.cim_la_size; i++) { ret = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG, V_UPDBGLARDPTR(idx) | F_UPDBGLARDEN); if (ret) break; ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &val); if (ret) break; if (val & F_UPDBGLARDEN) { ret = -ETIMEDOUT; break; } ret = t4_cim_read(adap, A_UP_UP_DBG_LA_DATA, 1, &la_buf[i]); if (ret) break; /* address can't exceed 0xfff (UpDbgLaRdPtr is of 12-bits) */ idx = (idx + 1) & M_UPDBGLARDPTR; /* * Bits 0-3 of UpDbgLaRdPtr can be between 0000 to 1001 to * identify the 32-bit portion of the full 312-bit data */ if (is_t6(adap)) while ((idx & 0xf) > 9) idx = (idx + 1) % M_UPDBGLARDPTR; } restart: if (cfg & F_UPDBGLAEN) { int r = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG, cfg & ~F_UPDBGLARDEN); if (!ret) ret = r; } return ret; } /** * t4_tp_read_la - read TP LA capture buffer * @adap: the adapter * @la_buf: where to store the LA data * @wrptr: the HW write pointer within the capture buffer * * Reads the contents of the TP LA buffer with the most recent entry at * the end of the returned data and with the entry at @wrptr first. * We leave the LA in the running state we find it in. */ void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr) { bool last_incomplete; unsigned int i, cfg, val, idx; cfg = t4_read_reg(adap, A_TP_DBG_LA_CONFIG) & 0xffff; if (cfg & F_DBGLAENABLE) /* freeze LA */ t4_write_reg(adap, A_TP_DBG_LA_CONFIG, adap->params.tp.la_mask | (cfg ^ F_DBGLAENABLE)); val = t4_read_reg(adap, A_TP_DBG_LA_CONFIG); idx = G_DBGLAWPTR(val); last_incomplete = G_DBGLAMODE(val) >= 2 && (val & F_DBGLAWHLF) == 0; if (last_incomplete) idx = (idx + 1) & M_DBGLARPTR; if (wrptr) *wrptr = idx; val &= 0xffff; val &= ~V_DBGLARPTR(M_DBGLARPTR); val |= adap->params.tp.la_mask; for (i = 0; i < TPLA_SIZE; i++) { t4_write_reg(adap, A_TP_DBG_LA_CONFIG, V_DBGLARPTR(idx) | val); la_buf[i] = t4_read_reg64(adap, A_TP_DBG_LA_DATAL); idx = (idx + 1) & M_DBGLARPTR; } /* Wipe out last entry if it isn't valid */ if (last_incomplete) la_buf[TPLA_SIZE - 1] = ~0ULL; if (cfg & F_DBGLAENABLE) /* restore running state */ t4_write_reg(adap, A_TP_DBG_LA_CONFIG, cfg | adap->params.tp.la_mask); } /* * SGE Hung Ingress DMA Warning Threshold time and Warning Repeat Rate (in * seconds). If we find one of the SGE Ingress DMA State Machines in the same * state for more than the Warning Threshold then we'll issue a warning about * a potential hang. We'll repeat the warning as the SGE Ingress DMA Channel * appears to be hung every Warning Repeat second till the situation clears. * If the situation clears, we'll note that as well. */ #define SGE_IDMA_WARN_THRESH 1 #define SGE_IDMA_WARN_REPEAT 300 /** * t4_idma_monitor_init - initialize SGE Ingress DMA Monitor * @adapter: the adapter * @idma: the adapter IDMA Monitor state * * Initialize the state of an SGE Ingress DMA Monitor. */ void t4_idma_monitor_init(struct adapter *adapter, struct sge_idma_monitor_state *idma) { /* Initialize the state variables for detecting an SGE Ingress DMA * hang. The SGE has internal counters which count up on each clock * tick whenever the SGE finds its Ingress DMA State Engines in the * same state they were on the previous clock tick. The clock used is * the Core Clock so we have a limit on the maximum "time" they can * record; typically a very small number of seconds. For instance, * with a 600MHz Core Clock, we can only count up to a bit more than * 7s. So we'll synthesize a larger counter in order to not run the * risk of having the "timers" overflow and give us the flexibility to * maintain a Hung SGE State Machine of our own which operates across * a longer time frame. */ idma->idma_1s_thresh = core_ticks_per_usec(adapter) * 1000000; /* 1s */ idma->idma_stalled[0] = idma->idma_stalled[1] = 0; } /** * t4_idma_monitor - monitor SGE Ingress DMA state * @adapter: the adapter * @idma: the adapter IDMA Monitor state * @hz: number of ticks/second * @ticks: number of ticks since the last IDMA Monitor call */ void t4_idma_monitor(struct adapter *adapter, struct sge_idma_monitor_state *idma, int hz, int ticks) { int i, idma_same_state_cnt[2]; /* Read the SGE Debug Ingress DMA Same State Count registers. These * are counters inside the SGE which count up on each clock when the * SGE finds its Ingress DMA State Engines in the same states they * were in the previous clock. The counters will peg out at * 0xffffffff without wrapping around so once they pass the 1s * threshold they'll stay above that till the IDMA state changes. */ t4_write_reg(adapter, A_SGE_DEBUG_INDEX, 13); idma_same_state_cnt[0] = t4_read_reg(adapter, A_SGE_DEBUG_DATA_HIGH); idma_same_state_cnt[1] = t4_read_reg(adapter, A_SGE_DEBUG_DATA_LOW); for (i = 0; i < 2; i++) { u32 debug0, debug11; /* If the Ingress DMA Same State Counter ("timer") is less * than 1s, then we can reset our synthesized Stall Timer and * continue. If we have previously emitted warnings about a * potential stalled Ingress Queue, issue a note indicating * that the Ingress Queue has resumed forward progress. */ if (idma_same_state_cnt[i] < idma->idma_1s_thresh) { if (idma->idma_stalled[i] >= SGE_IDMA_WARN_THRESH*hz) CH_WARN(adapter, "SGE idma%d, queue %u, " "resumed after %d seconds\n", i, idma->idma_qid[i], idma->idma_stalled[i]/hz); idma->idma_stalled[i] = 0; continue; } /* Synthesize an SGE Ingress DMA Same State Timer in the Hz * domain. The first time we get here it'll be because we * passed the 1s Threshold; each additional time it'll be * because the RX Timer Callback is being fired on its regular * schedule. * * If the stall is below our Potential Hung Ingress Queue * Warning Threshold, continue. */ if (idma->idma_stalled[i] == 0) { idma->idma_stalled[i] = hz; idma->idma_warn[i] = 0; } else { idma->idma_stalled[i] += ticks; idma->idma_warn[i] -= ticks; } if (idma->idma_stalled[i] < SGE_IDMA_WARN_THRESH*hz) continue; /* We'll issue a warning every SGE_IDMA_WARN_REPEAT seconds. */ if (idma->idma_warn[i] > 0) continue; idma->idma_warn[i] = SGE_IDMA_WARN_REPEAT*hz; /* Read and save the SGE IDMA State and Queue ID information. * We do this every time in case it changes across time ... * can't be too careful ... */ t4_write_reg(adapter, A_SGE_DEBUG_INDEX, 0); debug0 = t4_read_reg(adapter, A_SGE_DEBUG_DATA_LOW); idma->idma_state[i] = (debug0 >> (i * 9)) & 0x3f; t4_write_reg(adapter, A_SGE_DEBUG_INDEX, 11); debug11 = t4_read_reg(adapter, A_SGE_DEBUG_DATA_LOW); idma->idma_qid[i] = (debug11 >> (i * 16)) & 0xffff; CH_WARN(adapter, "SGE idma%u, queue %u, potentially stuck in " " state %u for %d seconds (debug0=%#x, debug11=%#x)\n", i, idma->idma_qid[i], idma->idma_state[i], idma->idma_stalled[i]/hz, debug0, debug11); t4_sge_decode_idma_state(adapter, idma->idma_state[i]); } } /** * t4_read_pace_tbl - read the pace table * @adap: the adapter * @pace_vals: holds the returned values * * Returns the values of TP's pace table in microseconds. */ void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED]) { unsigned int i, v; for (i = 0; i < NTX_SCHED; i++) { t4_write_reg(adap, A_TP_PACE_TABLE, 0xffff0000 + i); v = t4_read_reg(adap, A_TP_PACE_TABLE); pace_vals[i] = dack_ticks_to_usec(adap, v); } } /** * t4_get_tx_sched - get the configuration of a Tx HW traffic scheduler * @adap: the adapter * @sched: the scheduler index * @kbps: the byte rate in Kbps * @ipg: the interpacket delay in tenths of nanoseconds * * Return the current configuration of a HW Tx scheduler. */ void t4_get_tx_sched(struct adapter *adap, unsigned int sched, unsigned int *kbps, - unsigned int *ipg) + unsigned int *ipg, bool sleep_ok) { unsigned int v, addr, bpt, cpt; if (kbps) { addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2; - t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr); - v = t4_read_reg(adap, A_TP_TM_PIO_DATA); + t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok); if (sched & 1) v >>= 16; bpt = (v >> 8) & 0xff; cpt = v & 0xff; if (!cpt) *kbps = 0; /* scheduler disabled */ else { v = (adap->params.vpd.cclk * 1000) / cpt; /* ticks/s */ *kbps = (v * bpt) / 125; } } if (ipg) { addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2; - t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr); - v = t4_read_reg(adap, A_TP_TM_PIO_DATA); + t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok); if (sched & 1) v >>= 16; v &= 0xffff; *ipg = (10000 * v) / core_ticks_per_usec(adap); } } /** * t4_load_cfg - download config file * @adap: the adapter * @cfg_data: the cfg text file to write * @size: text file size * * Write the supplied config text file to the card's serial flash. */ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) { int ret, i, n, cfg_addr; unsigned int addr; unsigned int flash_cfg_start_sec; unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; cfg_addr = t4_flash_cfg_addr(adap); if (cfg_addr < 0) return cfg_addr; addr = cfg_addr; flash_cfg_start_sec = addr / SF_SEC_SIZE; if (size > FLASH_CFG_MAX_SIZE) { CH_ERR(adap, "cfg file too large, max is %u bytes\n", FLASH_CFG_MAX_SIZE); return -EFBIG; } i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE, /* # of sectors spanned */ sf_sec_size); ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec, flash_cfg_start_sec + i - 1); /* * If size == 0 then we're simply erasing the FLASH sectors associated * with the on-adapter Firmware Configuration File. */ if (ret || size == 0) goto out; /* this will write to the flash up to SF_PAGE_SIZE at a time */ for (i = 0; i< size; i+= SF_PAGE_SIZE) { if ( (size - i) < SF_PAGE_SIZE) n = size - i; else n = SF_PAGE_SIZE; ret = t4_write_flash(adap, addr, n, cfg_data, 1); if (ret) goto out; addr += SF_PAGE_SIZE; cfg_data += SF_PAGE_SIZE; } out: if (ret) CH_ERR(adap, "config file %s failed %d\n", (size == 0 ? "clear" : "download"), ret); return ret; } /** * t5_fw_init_extern_mem - initialize the external memory * @adap: the adapter * * Initializes the external memory on T5. */ int t5_fw_init_extern_mem(struct adapter *adap) { u32 params[1], val[1]; int ret; if (!is_t5(adap)) return 0; val[0] = 0xff; /* Initialize all MCs */ params[0] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_MCINIT)); ret = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, params, val, FW_CMD_MAX_TIMEOUT); return ret; } /* BIOS boot headers */ typedef struct pci_expansion_rom_header { u8 signature[2]; /* ROM Signature. Should be 0xaa55 */ u8 reserved[22]; /* Reserved per processor Architecture data */ u8 pcir_offset[2]; /* Offset to PCI Data Structure */ } pci_exp_rom_header_t; /* PCI_EXPANSION_ROM_HEADER */ /* Legacy PCI Expansion ROM Header */ typedef struct legacy_pci_expansion_rom_header { u8 signature[2]; /* ROM Signature. Should be 0xaa55 */ u8 size512; /* Current Image Size in units of 512 bytes */ u8 initentry_point[4]; u8 cksum; /* Checksum computed on the entire Image */ u8 reserved[16]; /* Reserved */ u8 pcir_offset[2]; /* Offset to PCI Data Struture */ } legacy_pci_exp_rom_header_t; /* LEGACY_PCI_EXPANSION_ROM_HEADER */ /* EFI PCI Expansion ROM Header */ typedef struct efi_pci_expansion_rom_header { u8 signature[2]; // ROM signature. The value 0xaa55 u8 initialization_size[2]; /* Units 512. Includes this header */ u8 efi_signature[4]; /* Signature from EFI image header. 0x0EF1 */ u8 efi_subsystem[2]; /* Subsystem value for EFI image header */ u8 efi_machine_type[2]; /* Machine type from EFI image header */ u8 compression_type[2]; /* Compression type. */ /* * Compression type definition * 0x0: uncompressed * 0x1: Compressed * 0x2-0xFFFF: Reserved */ u8 reserved[8]; /* Reserved */ u8 efi_image_header_offset[2]; /* Offset to EFI Image */ u8 pcir_offset[2]; /* Offset to PCI Data Structure */ } efi_pci_exp_rom_header_t; /* EFI PCI Expansion ROM Header */ /* PCI Data Structure Format */ typedef struct pcir_data_structure { /* PCI Data Structure */ u8 signature[4]; /* Signature. The string "PCIR" */ u8 vendor_id[2]; /* Vendor Identification */ u8 device_id[2]; /* Device Identification */ u8 vital_product[2]; /* Pointer to Vital Product Data */ u8 length[2]; /* PCIR Data Structure Length */ u8 revision; /* PCIR Data Structure Revision */ u8 class_code[3]; /* Class Code */ u8 image_length[2]; /* Image Length. Multiple of 512B */ u8 code_revision[2]; /* Revision Level of Code/Data */ u8 code_type; /* Code Type. */ /* * PCI Expansion ROM Code Types * 0x00: Intel IA-32, PC-AT compatible. Legacy * 0x01: Open Firmware standard for PCI. FCODE * 0x02: Hewlett-Packard PA RISC. HP reserved * 0x03: EFI Image. EFI * 0x04-0xFF: Reserved. */ u8 indicator; /* Indicator. Identifies the last image in the ROM */ u8 reserved[2]; /* Reserved */ } pcir_data_t; /* PCI__DATA_STRUCTURE */ /* BOOT constants */ enum { BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */ BOOT_SIGNATURE = 0xaa55, /* signature of BIOS boot ROM */ BOOT_SIZE_INC = 512, /* image size measured in 512B chunks */ BOOT_MIN_SIZE = sizeof(pci_exp_rom_header_t), /* basic header */ BOOT_MAX_SIZE = 1024*BOOT_SIZE_INC, /* 1 byte * length increment */ VENDOR_ID = 0x1425, /* Vendor ID */ PCIR_SIGNATURE = 0x52494350 /* PCIR signature */ }; /* * modify_device_id - Modifies the device ID of the Boot BIOS image * @adatper: the device ID to write. * @boot_data: the boot image to modify. * * Write the supplied device ID to the boot BIOS image. */ static void modify_device_id(int device_id, u8 *boot_data) { legacy_pci_exp_rom_header_t *header; pcir_data_t *pcir_header; u32 cur_header = 0; /* * Loop through all chained images and change the device ID's */ while (1) { header = (legacy_pci_exp_rom_header_t *) &boot_data[cur_header]; pcir_header = (pcir_data_t *) &boot_data[cur_header + le16_to_cpu(*(u16*)header->pcir_offset)]; /* * Only modify the Device ID if code type is Legacy or HP. * 0x00: Okay to modify * 0x01: FCODE. Do not be modify * 0x03: Okay to modify * 0x04-0xFF: Do not modify */ if (pcir_header->code_type == 0x00) { u8 csum = 0; int i; /* * Modify Device ID to match current adatper */ *(u16*) pcir_header->device_id = device_id; /* * Set checksum temporarily to 0. * We will recalculate it later. */ header->cksum = 0x0; /* * Calculate and update checksum */ for (i = 0; i < (header->size512 * 512); i++) csum += (u8)boot_data[cur_header + i]; /* * Invert summed value to create the checksum * Writing new checksum value directly to the boot data */ boot_data[cur_header + 7] = -csum; } else if (pcir_header->code_type == 0x03) { /* * Modify Device ID to match current adatper */ *(u16*) pcir_header->device_id = device_id; } /* * Check indicator element to identify if this is the last * image in the ROM. */ if (pcir_header->indicator & 0x80) break; /* * Move header pointer up to the next image in the ROM. */ cur_header += header->size512 * 512; } } /* * t4_load_boot - download boot flash * @adapter: the adapter * @boot_data: the boot image to write * @boot_addr: offset in flash to write boot_data * @size: image size * * Write the supplied boot image to the card's serial flash. * The boot image has the following sections: a 28-byte header and the * boot image. */ int t4_load_boot(struct adapter *adap, u8 *boot_data, unsigned int boot_addr, unsigned int size) { pci_exp_rom_header_t *header; int pcir_offset ; pcir_data_t *pcir_header; int ret, addr; uint16_t device_id; unsigned int i; unsigned int boot_sector = (boot_addr * 1024 ); unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; /* * Make sure the boot image does not encroach on the firmware region */ if ((boot_sector + size) >> 16 > FLASH_FW_START_SEC) { CH_ERR(adap, "boot image encroaching on firmware region\n"); return -EFBIG; } /* * The boot sector is comprised of the Expansion-ROM boot, iSCSI boot, * and Boot configuration data sections. These 3 boot sections span * sectors 0 to 7 in flash and live right before the FW image location. */ i = DIV_ROUND_UP(size ? size : FLASH_FW_START, sf_sec_size); ret = t4_flash_erase_sectors(adap, boot_sector >> 16, (boot_sector >> 16) + i - 1); /* * If size == 0 then we're simply erasing the FLASH sectors associated * with the on-adapter option ROM file */ if (ret || (size == 0)) goto out; /* Get boot header */ header = (pci_exp_rom_header_t *)boot_data; pcir_offset = le16_to_cpu(*(u16 *)header->pcir_offset); /* PCIR Data Structure */ pcir_header = (pcir_data_t *) &boot_data[pcir_offset]; /* * Perform some primitive sanity testing to avoid accidentally * writing garbage over the boot sectors. We ought to check for * more but it's not worth it for now ... */ if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) { CH_ERR(adap, "boot image too small/large\n"); return -EFBIG; } #ifndef CHELSIO_T4_DIAGS /* * Check BOOT ROM header signature */ if (le16_to_cpu(*(u16*)header->signature) != BOOT_SIGNATURE ) { CH_ERR(adap, "Boot image missing signature\n"); return -EINVAL; } /* * Check PCI header signature */ if (le32_to_cpu(*(u32*)pcir_header->signature) != PCIR_SIGNATURE) { CH_ERR(adap, "PCI header missing signature\n"); return -EINVAL; } /* * Check Vendor ID matches Chelsio ID */ if (le16_to_cpu(*(u16*)pcir_header->vendor_id) != VENDOR_ID) { CH_ERR(adap, "Vendor ID missing signature\n"); return -EINVAL; } #endif /* * Retrieve adapter's device ID */ t4_os_pci_read_cfg2(adap, PCI_DEVICE_ID, &device_id); /* Want to deal with PF 0 so I strip off PF 4 indicator */ device_id = device_id & 0xf0ff; /* * Check PCIE Device ID */ if (le16_to_cpu(*(u16*)pcir_header->device_id) != device_id) { /* * Change the device ID in the Boot BIOS image to match * the Device ID of the current adapter. */ modify_device_id(device_id, boot_data); } /* * Skip over the first SF_PAGE_SIZE worth of data and write it after * we finish copying the rest of the boot image. This will ensure * that the BIOS boot header will only be written if the boot image * was written in full. */ addr = boot_sector; for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { addr += SF_PAGE_SIZE; boot_data += SF_PAGE_SIZE; ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data, 0); if (ret) goto out; } ret = t4_write_flash(adap, boot_sector, SF_PAGE_SIZE, (const u8 *)header, 0); out: if (ret) CH_ERR(adap, "boot image download failed, error %d\n", ret); return ret; } /* * t4_flash_bootcfg_addr - return the address of the flash optionrom configuration * @adapter: the adapter * * Return the address within the flash where the OptionROM Configuration * is stored, or an error if the device FLASH is too small to contain * a OptionROM Configuration. */ static int t4_flash_bootcfg_addr(struct adapter *adapter) { /* * If the device FLASH isn't large enough to hold a Firmware * Configuration File, return an error. */ if (adapter->params.sf_size < FLASH_BOOTCFG_START + FLASH_BOOTCFG_MAX_SIZE) return -ENOSPC; return FLASH_BOOTCFG_START; } int t4_load_bootcfg(struct adapter *adap,const u8 *cfg_data, unsigned int size) { int ret, i, n, cfg_addr; unsigned int addr; unsigned int flash_cfg_start_sec; unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; cfg_addr = t4_flash_bootcfg_addr(adap); if (cfg_addr < 0) return cfg_addr; addr = cfg_addr; flash_cfg_start_sec = addr / SF_SEC_SIZE; if (size > FLASH_BOOTCFG_MAX_SIZE) { CH_ERR(adap, "bootcfg file too large, max is %u bytes\n", FLASH_BOOTCFG_MAX_SIZE); return -EFBIG; } i = DIV_ROUND_UP(FLASH_BOOTCFG_MAX_SIZE,/* # of sectors spanned */ sf_sec_size); ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec, flash_cfg_start_sec + i - 1); /* * If size == 0 then we're simply erasing the FLASH sectors associated * with the on-adapter OptionROM Configuration File. */ if (ret || size == 0) goto out; /* this will write to the flash up to SF_PAGE_SIZE at a time */ for (i = 0; i< size; i+= SF_PAGE_SIZE) { if ( (size - i) < SF_PAGE_SIZE) n = size - i; else n = SF_PAGE_SIZE; ret = t4_write_flash(adap, addr, n, cfg_data, 0); if (ret) goto out; addr += SF_PAGE_SIZE; cfg_data += SF_PAGE_SIZE; } out: if (ret) CH_ERR(adap, "boot config data %s failed %d\n", (size == 0 ? "clear" : "download"), ret); return ret; } /** * t4_set_filter_mode - configure the optional components of filter tuples * @adap: the adapter * @mode_map: a bitmap selcting which optional filter components to enable + * @sleep_ok: if true we may sleep while awaiting command completion * * Sets the filter mode by selecting the optional components to enable * in filter tuples. Returns 0 on success and a negative error if the * requested mode needs more bits than are available for optional * components. */ -int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map) +int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map, + bool sleep_ok) { static u8 width[] = { 1, 3, 17, 17, 8, 8, 16, 9, 3, 1 }; int i, nbits = 0; for (i = S_FCOE; i <= S_FRAGMENTATION; i++) if (mode_map & (1 << i)) nbits += width[i]; if (nbits > FILTER_OPT_LEN) return -EINVAL; - if (t4_use_ldst(adap)) - t4_fw_tp_pio_rw(adap, &mode_map, 1, A_TP_VLAN_PRI_MAP, 0); - else - t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, &mode_map, - 1, A_TP_VLAN_PRI_MAP); - read_filter_mode_and_ingress_config(adap); + t4_tp_pio_write(adap, &mode_map, 1, A_TP_VLAN_PRI_MAP, sleep_ok); + read_filter_mode_and_ingress_config(adap, sleep_ok); return 0; } /** * t4_clr_port_stats - clear port statistics * @adap: the adapter * @idx: the port index * * Clear HW statistics for the given port. */ void t4_clr_port_stats(struct adapter *adap, int idx) { unsigned int i; u32 bgmap = t4_get_mps_bg_map(adap, idx); u32 port_base_addr; if (is_t4(adap)) port_base_addr = PORT_BASE(idx); else port_base_addr = T5_PORT_BASE(idx); for (i = A_MPS_PORT_STAT_TX_PORT_BYTES_L; i <= A_MPS_PORT_STAT_TX_PORT_PPP7_H; i += 8) t4_write_reg(adap, port_base_addr + i, 0); for (i = A_MPS_PORT_STAT_RX_PORT_BYTES_L; i <= A_MPS_PORT_STAT_RX_PORT_LESS_64B_H; i += 8) t4_write_reg(adap, port_base_addr + i, 0); for (i = 0; i < 4; i++) if (bgmap & (1 << i)) { t4_write_reg(adap, A_MPS_STAT_RX_BG_0_MAC_DROP_FRAME_L + i * 8, 0); t4_write_reg(adap, A_MPS_STAT_RX_BG_0_MAC_TRUNC_FRAME_L + i * 8, 0); } } /** * t4_i2c_rd - read I2C data from adapter * @adap: the adapter * @port: Port number if per-port device; <0 if not * @devid: per-port device ID or absolute device ID * @offset: byte offset into device I2C space * @len: byte length of I2C space data * @buf: buffer in which to return I2C data * * Reads the I2C data from the indicated device and location. */ int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port, unsigned int devid, unsigned int offset, unsigned int len, u8 *buf) { u32 ldst_addrspace; struct fw_ldst_cmd ldst; int ret; if (port >= 4 || devid >= 256 || offset >= 256 || len > sizeof ldst.u.i2c.data) return -EINVAL; memset(&ldst, 0, sizeof ldst); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_I2C); ldst.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | ldst_addrspace); ldst.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst)); ldst.u.i2c.pid = (port < 0 ? 0xff : port); ldst.u.i2c.did = devid; ldst.u.i2c.boffset = offset; ldst.u.i2c.blen = len; ret = t4_wr_mbox(adap, mbox, &ldst, sizeof ldst, &ldst); if (!ret) memcpy(buf, ldst.u.i2c.data, len); return ret; } /** * t4_i2c_wr - write I2C data to adapter * @adap: the adapter * @port: Port number if per-port device; <0 if not * @devid: per-port device ID or absolute device ID * @offset: byte offset into device I2C space * @len: byte length of I2C space data * @buf: buffer containing new I2C data * * Write the I2C data to the indicated device and location. */ int t4_i2c_wr(struct adapter *adap, unsigned int mbox, int port, unsigned int devid, unsigned int offset, unsigned int len, u8 *buf) { u32 ldst_addrspace; struct fw_ldst_cmd ldst; if (port >= 4 || devid >= 256 || offset >= 256 || len > sizeof ldst.u.i2c.data) return -EINVAL; memset(&ldst, 0, sizeof ldst); ldst_addrspace = V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_I2C); ldst.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | ldst_addrspace); ldst.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst)); ldst.u.i2c.pid = (port < 0 ? 0xff : port); ldst.u.i2c.did = devid; ldst.u.i2c.boffset = offset; ldst.u.i2c.blen = len; memcpy(ldst.u.i2c.data, buf, len); return t4_wr_mbox(adap, mbox, &ldst, sizeof ldst, &ldst); } /** * t4_sge_ctxt_rd - read an SGE context through FW * @adap: the adapter * @mbox: mailbox to use for the FW command * @cid: the context id * @ctype: the context type * @data: where to store the context data * * Issues a FW command through the given mailbox to read an SGE context. */ int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid, enum ctxt_type ctype, u32 *data) { int ret; struct fw_ldst_cmd c; if (ctype == CTXT_EGRESS) ret = FW_LDST_ADDRSPC_SGE_EGRC; else if (ctype == CTXT_INGRESS) ret = FW_LDST_ADDRSPC_SGE_INGC; else if (ctype == CTXT_FLM) ret = FW_LDST_ADDRSPC_SGE_FLMC; else ret = FW_LDST_ADDRSPC_SGE_CONMC; memset(&c, 0, sizeof(c)); c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(ret)); c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); c.u.idctxt.physid = cpu_to_be32(cid); ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); if (ret == 0) { data[0] = be32_to_cpu(c.u.idctxt.ctxt_data0); data[1] = be32_to_cpu(c.u.idctxt.ctxt_data1); data[2] = be32_to_cpu(c.u.idctxt.ctxt_data2); data[3] = be32_to_cpu(c.u.idctxt.ctxt_data3); data[4] = be32_to_cpu(c.u.idctxt.ctxt_data4); data[5] = be32_to_cpu(c.u.idctxt.ctxt_data5); } return ret; } /** * t4_sge_ctxt_rd_bd - read an SGE context bypassing FW * @adap: the adapter * @cid: the context id * @ctype: the context type * @data: where to store the context data * * Reads an SGE context directly, bypassing FW. This is only for * debugging when FW is unavailable. */ int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, enum ctxt_type ctype, u32 *data) { int i, ret; t4_write_reg(adap, A_SGE_CTXT_CMD, V_CTXTQID(cid) | V_CTXTTYPE(ctype)); ret = t4_wait_op_done(adap, A_SGE_CTXT_CMD, F_BUSY, 0, 3, 1); if (!ret) for (i = A_SGE_CTXT_DATA0; i <= A_SGE_CTXT_DATA5; i += 4) *data++ = t4_read_reg(adap, i); return ret; } int t4_sched_config(struct adapter *adapter, int type, int minmaxen, - int sleep_ok) + int sleep_ok) { struct fw_sched_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); cmd.u.config.sc = FW_SCHED_SC_CONFIG; cmd.u.config.type = type; cmd.u.config.minmaxen = minmaxen; return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd), NULL, sleep_ok); } int t4_sched_params(struct adapter *adapter, int type, int level, int mode, int rateunit, int ratemode, int channel, int cl, int minrate, int maxrate, int weight, int pktsize, int sleep_ok) { struct fw_sched_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); cmd.u.params.sc = FW_SCHED_SC_PARAMS; cmd.u.params.type = type; cmd.u.params.level = level; cmd.u.params.mode = mode; cmd.u.params.ch = channel; cmd.u.params.cl = cl; cmd.u.params.unit = rateunit; cmd.u.params.rate = ratemode; cmd.u.params.min = cpu_to_be32(minrate); cmd.u.params.max = cpu_to_be32(maxrate); cmd.u.params.weight = cpu_to_be16(weight); cmd.u.params.pktsize = cpu_to_be16(pktsize); return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd), NULL, sleep_ok); } int t4_sched_params_ch_rl(struct adapter *adapter, int channel, int ratemode, unsigned int maxrate, int sleep_ok) { struct fw_sched_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); cmd.u.params.sc = FW_SCHED_SC_PARAMS; cmd.u.params.type = FW_SCHED_TYPE_PKTSCHED; cmd.u.params.level = FW_SCHED_PARAMS_LEVEL_CH_RL; cmd.u.params.ch = channel; cmd.u.params.rate = ratemode; /* REL or ABS */ cmd.u.params.max = cpu_to_be32(maxrate);/* % or kbps */ return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd), NULL, sleep_ok); } int t4_sched_params_cl_wrr(struct adapter *adapter, int channel, int cl, int weight, int sleep_ok) { struct fw_sched_cmd cmd; if (weight < 0 || weight > 100) return -EINVAL; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); cmd.u.params.sc = FW_SCHED_SC_PARAMS; cmd.u.params.type = FW_SCHED_TYPE_PKTSCHED; cmd.u.params.level = FW_SCHED_PARAMS_LEVEL_CL_WRR; cmd.u.params.ch = channel; cmd.u.params.cl = cl; cmd.u.params.weight = cpu_to_be16(weight); return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd), NULL, sleep_ok); } int t4_sched_params_cl_rl_kbps(struct adapter *adapter, int channel, int cl, int mode, unsigned int maxrate, int pktsize, int sleep_ok) { struct fw_sched_cmd cmd; memset(&cmd, 0, sizeof(cmd)); cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); cmd.u.params.sc = FW_SCHED_SC_PARAMS; cmd.u.params.type = FW_SCHED_TYPE_PKTSCHED; cmd.u.params.level = FW_SCHED_PARAMS_LEVEL_CL_RL; cmd.u.params.mode = mode; cmd.u.params.ch = channel; cmd.u.params.cl = cl; cmd.u.params.unit = FW_SCHED_PARAMS_UNIT_BITRATE; cmd.u.params.rate = FW_SCHED_PARAMS_RATE_ABS; cmd.u.params.max = cpu_to_be32(maxrate); cmd.u.params.pktsize = cpu_to_be16(pktsize); return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd), NULL, sleep_ok); } /* * t4_config_watchdog - configure (enable/disable) a watchdog timer * @adapter: the adapter * @mbox: mailbox to use for the FW command * @pf: the PF owning the queue * @vf: the VF owning the queue * @timeout: watchdog timeout in ms * @action: watchdog timer / action * * There are separate watchdog timers for each possible watchdog * action. Configure one of the watchdog timers by setting a non-zero * timeout. Disable a watchdog timer by using a timeout of zero. */ int t4_config_watchdog(struct adapter *adapter, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int timeout, unsigned int action) { struct fw_watchdog_cmd wdog; unsigned int ticks; /* * The watchdog command expects a timeout in units of 10ms so we need * to convert it here (via rounding) and force a minimum of one 10ms * "tick" if the timeout is non-zero but the conversion results in 0 * ticks. */ ticks = (timeout + 5)/10; if (timeout && !ticks) ticks = 1; memset(&wdog, 0, sizeof wdog); wdog.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_WATCHDOG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | V_FW_PARAMS_CMD_PFN(pf) | V_FW_PARAMS_CMD_VFN(vf)); wdog.retval_len16 = cpu_to_be32(FW_LEN16(wdog)); wdog.timeout = cpu_to_be32(ticks); wdog.action = cpu_to_be32(action); return t4_wr_mbox(adapter, mbox, &wdog, sizeof wdog, NULL); } int t4_get_devlog_level(struct adapter *adapter, unsigned int *level) { struct fw_devlog_cmd devlog_cmd; int ret; memset(&devlog_cmd, 0, sizeof(devlog_cmd)); devlog_cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_DEVLOG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd)); ret = t4_wr_mbox(adapter, adapter->mbox, &devlog_cmd, sizeof(devlog_cmd), &devlog_cmd); if (ret) return ret; *level = devlog_cmd.level; return 0; } int t4_set_devlog_level(struct adapter *adapter, unsigned int level) { struct fw_devlog_cmd devlog_cmd; memset(&devlog_cmd, 0, sizeof(devlog_cmd)); devlog_cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_DEVLOG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); devlog_cmd.level = level; devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd)); return t4_wr_mbox(adapter, adapter->mbox, &devlog_cmd, sizeof(devlog_cmd), &devlog_cmd); } Index: projects/runtime-coverage/sys/dev/cxgbe/t4_main.c =================================================================== --- projects/runtime-coverage/sys/dev/cxgbe/t4_main.c (revision 321584) +++ projects/runtime-coverage/sys/dev/cxgbe/t4_main.c (revision 321585) @@ -1,10078 +1,10078 @@ /*- * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_inet.h" #include "opt_inet6.h" #include "opt_rss.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RSS #include #endif #if defined(__i386__) || defined(__amd64__) #include #include #endif #ifdef DDB #include #include #endif #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_regs_values.h" #include "t4_ioctl.h" #include "t4_l2t.h" #include "t4_mp_ring.h" #include "t4_if.h" /* T4 bus driver interface */ static int t4_probe(device_t); static int t4_attach(device_t); static int t4_detach(device_t); static int t4_ready(device_t); static int t4_read_port_device(device_t, int, device_t *); static device_method_t t4_methods[] = { DEVMETHOD(device_probe, t4_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t4_driver = { "t4nex", t4_methods, sizeof(struct adapter) }; /* T4 port (cxgbe) interface */ static int cxgbe_probe(device_t); static int cxgbe_attach(device_t); static int cxgbe_detach(device_t); device_method_t cxgbe_methods[] = { DEVMETHOD(device_probe, cxgbe_probe), DEVMETHOD(device_attach, cxgbe_attach), DEVMETHOD(device_detach, cxgbe_detach), { 0, 0 } }; static driver_t cxgbe_driver = { "cxgbe", cxgbe_methods, sizeof(struct port_info) }; /* T4 VI (vcxgbe) interface */ static int vcxgbe_probe(device_t); static int vcxgbe_attach(device_t); static int vcxgbe_detach(device_t); static device_method_t vcxgbe_methods[] = { DEVMETHOD(device_probe, vcxgbe_probe), DEVMETHOD(device_attach, vcxgbe_attach), DEVMETHOD(device_detach, vcxgbe_detach), { 0, 0 } }; static driver_t vcxgbe_driver = { "vcxgbe", vcxgbe_methods, sizeof(struct vi_info) }; static d_ioctl_t t4_ioctl; static struct cdevsw t4_cdevsw = { .d_version = D_VERSION, .d_ioctl = t4_ioctl, .d_name = "t4nex", }; /* T5 bus driver interface */ static int t5_probe(device_t); static device_method_t t5_methods[] = { DEVMETHOD(device_probe, t5_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t5_driver = { "t5nex", t5_methods, sizeof(struct adapter) }; /* T5 port (cxl) interface */ static driver_t cxl_driver = { "cxl", cxgbe_methods, sizeof(struct port_info) }; /* T5 VI (vcxl) interface */ static driver_t vcxl_driver = { "vcxl", vcxgbe_methods, sizeof(struct vi_info) }; /* T6 bus driver interface */ static int t6_probe(device_t); static device_method_t t6_methods[] = { DEVMETHOD(device_probe, t6_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t6_driver = { "t6nex", t6_methods, sizeof(struct adapter) }; /* T6 port (cc) interface */ static driver_t cc_driver = { "cc", cxgbe_methods, sizeof(struct port_info) }; /* T6 VI (vcc) interface */ static driver_t vcc_driver = { "vcc", vcxgbe_methods, sizeof(struct vi_info) }; /* ifnet + media interface */ static void cxgbe_init(void *); static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); static int cxgbe_transmit(struct ifnet *, struct mbuf *); static void cxgbe_qflush(struct ifnet *); static int cxgbe_media_change(struct ifnet *); static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); /* * Correct lock order when you need to acquire multiple locks is t4_list_lock, * then ADAPTER_LOCK, then t4_uld_list_lock. */ static struct sx t4_list_lock; SLIST_HEAD(, adapter) t4_list; #ifdef TCP_OFFLOAD static struct sx t4_uld_list_lock; SLIST_HEAD(, uld_info) t4_uld_list; #endif /* * Tunables. See tweak_tunables() too. * * Each tunable is set to a default value here if it's known at compile-time. * Otherwise it is set to -n as an indication to tweak_tunables() that it should * provide a reasonable default (upto n) when the driver is loaded. * * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to * T5 are under hw.cxl. */ /* * Number of queues for tx and rx, 10G and 1G, NIC and offload. */ #define NTXQ_10G 16 int t4_ntxq10g = -NTXQ_10G; TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); #define NRXQ_10G 8 int t4_nrxq10g = -NRXQ_10G; TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); #define NTXQ_1G 4 int t4_ntxq1g = -NTXQ_1G; TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); #define NRXQ_1G 2 int t4_nrxq1g = -NRXQ_1G; TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); #define NTXQ_VI 1 static int t4_ntxq_vi = -NTXQ_VI; TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi); #define NRXQ_VI 1 static int t4_nrxq_vi = -NRXQ_VI; TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi); static int t4_rsrv_noflowq = 0; TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); #ifdef TCP_OFFLOAD #define NOFLDTXQ_10G 8 static int t4_nofldtxq10g = -NOFLDTXQ_10G; TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); #define NOFLDRXQ_10G 2 static int t4_nofldrxq10g = -NOFLDRXQ_10G; TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); #define NOFLDTXQ_1G 2 static int t4_nofldtxq1g = -NOFLDTXQ_1G; TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); #define NOFLDRXQ_1G 1 static int t4_nofldrxq1g = -NOFLDRXQ_1G; TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); #define NOFLDTXQ_VI 1 static int t4_nofldtxq_vi = -NOFLDTXQ_VI; TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi); #define NOFLDRXQ_VI 1 static int t4_nofldrxq_vi = -NOFLDRXQ_VI; TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi); #endif #ifdef DEV_NETMAP #define NNMTXQ_VI 2 static int t4_nnmtxq_vi = -NNMTXQ_VI; TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi); #define NNMRXQ_VI 2 static int t4_nnmrxq_vi = -NNMRXQ_VI; TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi); #endif /* * Holdoff parameters for 10G and 1G ports. */ #define TMR_IDX_10G 1 int t4_tmr_idx_10g = TMR_IDX_10G; TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); #define PKTC_IDX_10G (-1) int t4_pktc_idx_10g = PKTC_IDX_10G; TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); #define TMR_IDX_1G 1 int t4_tmr_idx_1g = TMR_IDX_1G; TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); #define PKTC_IDX_1G (-1) int t4_pktc_idx_1g = PKTC_IDX_1G; TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); /* * Size (# of entries) of each tx and rx queue. */ unsigned int t4_qsize_txq = TX_EQ_QSIZE; TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); unsigned int t4_qsize_rxq = RX_IQ_QSIZE; TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); /* * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). */ int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); /* * Configuration file. */ #define DEFAULT_CF "default" #define FLASH_CF "flash" #define UWIRE_CF "uwire" #define FPGA_CF "fpga" static char t4_cfg_file[32] = DEFAULT_CF; TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); /* * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water * mark or when signalled to do so, 0 to never emit PAUSE. */ static int t4_pause_settings = PAUSE_TX | PAUSE_RX; TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); /* * Forward Error Correction settings (bit 0, 1, 2 = FEC_RS, FEC_BASER_RS, * FEC_RESERVED respectively). * -1 to run with the firmware default. * 0 to disable FEC. */ static int t4_fec = -1; TUNABLE_INT("hw.cxgbe.fec", &t4_fec); /* * Link autonegotiation. * -1 to run with the firmware default. * 0 to disable. * 1 to enable. */ static int t4_autoneg = -1; TUNABLE_INT("hw.cxgbe.autoneg", &t4_autoneg); /* * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, * encouraged respectively). */ static unsigned int t4_fw_install = 1; TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); /* * ASIC features that will be used. Disable the ones you don't want so that the * chip resources aren't wasted on features that will not be used. */ static int t4_nbmcaps_allowed = 0; TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed); static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS | FW_CAPS_CONFIG_SWITCH_EGRESS; TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed); static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); static int t4_toecaps_allowed = -1; TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); static int t4_rdmacaps_allowed = -1; TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); static int t4_cryptocaps_allowed = 0; TUNABLE_INT("hw.cxgbe.cryptocaps_allowed", &t4_cryptocaps_allowed); static int t4_iscsicaps_allowed = -1; TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); static int t4_fcoecaps_allowed = 0; TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); static int t5_write_combine = 0; TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); static int t4_num_vis = 1; TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ static int vi_mac_funcs[] = { FW_VI_FUNC_OFLD, FW_VI_FUNC_IWARP, FW_VI_FUNC_OPENISCSI, FW_VI_FUNC_OPENFCOE, FW_VI_FUNC_FOISCSI, FW_VI_FUNC_FOFCOE, }; struct intrs_and_queues { uint16_t intr_type; /* INTx, MSI, or MSI-X */ uint16_t nirq; /* Total # of vectors */ uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ /* The vcxgbe/vcxl interfaces use these and not the ones above. */ uint16_t ntxq_vi; /* # of NIC txq's */ uint16_t nrxq_vi; /* # of NIC rxq's */ uint16_t nofldtxq_vi; /* # of TOE txq's */ uint16_t nofldrxq_vi; /* # of TOE rxq's */ uint16_t nnmtxq_vi; /* # of netmap txq's */ uint16_t nnmrxq_vi; /* # of netmap rxq's */ }; struct filter_entry { uint32_t valid:1; /* filter allocated and valid */ uint32_t locked:1; /* filter is administratively locked */ uint32_t pending:1; /* filter action is pending firmware reply */ uint32_t smtidx:8; /* Source MAC Table index for smac */ struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ struct t4_filter_specification fs; }; static void setup_memwin(struct adapter *); static void position_memwin(struct adapter *, int, uint32_t); static int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int); static inline int read_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int); static inline int write_via_memwin(struct adapter *, int, uint32_t, const uint32_t *, int); static int validate_mem_range(struct adapter *, uint32_t, int); static int fwmtype_to_hwmtype(int); static int validate_mt_off_len(struct adapter *, int, uint32_t, int, uint32_t *); static int fixup_devlog_params(struct adapter *); static int cfg_itype_and_nqueues(struct adapter *, int, int, int, struct intrs_and_queues *); static int prep_firmware(struct adapter *); static int partition_resources(struct adapter *, const struct firmware *, const char *); static int get_params__pre_init(struct adapter *); static int get_params__post_init(struct adapter *); static int set_params__post_init(struct adapter *); static void t4_set_desc(struct adapter *); static void build_medialist(struct port_info *, struct ifmedia *); static void init_l1cfg(struct port_info *); static int cxgbe_init_synchronized(struct vi_info *); static int cxgbe_uninit_synchronized(struct vi_info *); static void quiesce_txq(struct adapter *, struct sge_txq *); static void quiesce_wrq(struct adapter *, struct sge_wrq *); static void quiesce_iq(struct adapter *, struct sge_iq *); static void quiesce_fl(struct adapter *, struct sge_fl *); static int t4_alloc_irq(struct adapter *, struct irq *, int rid, driver_intr_t *, void *, char *); static int t4_free_irq(struct adapter *, struct irq *); static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *); static void vi_refresh_stats(struct adapter *, struct vi_info *); static void cxgbe_refresh_stats(struct adapter *, struct port_info *); static void cxgbe_tick(void *); static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); static void cxgbe_sysctls(struct port_info *); static int sysctl_int_array(SYSCTL_HANDLER_ARGS); static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); static int sysctl_btphy(SYSCTL_HANDLER_ARGS); static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); static int sysctl_fec(SYSCTL_HANDLER_ARGS); static int sysctl_autoneg(SYSCTL_HANDLER_ARGS); static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); static int sysctl_temperature(SYSCTL_HANDLER_ARGS); #ifdef SBUF_DRAIN static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS); static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); static int sysctl_devlog(SYSCTL_HANDLER_ARGS); static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tids(SYSCTL_HANDLER_ARGS); static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS); static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tc_params(SYSCTL_HANDLER_ARGS); #endif #ifdef TCP_OFFLOAD static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS); static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS); #endif static uint32_t fconf_iconf_to_mode(uint32_t, uint32_t); static uint32_t mode_to_fconf(uint32_t); static uint32_t mode_to_iconf(uint32_t); static int check_fspec_against_fconf_iconf(struct adapter *, struct t4_filter_specification *); static int get_filter_mode(struct adapter *, uint32_t *); static int set_filter_mode(struct adapter *, uint32_t); static inline uint64_t get_filter_hits(struct adapter *, uint32_t); static int get_filter(struct adapter *, struct t4_filter *); static int set_filter(struct adapter *, struct t4_filter *); static int del_filter(struct adapter *, struct t4_filter *); static void clear_filter(struct filter_entry *); static int set_filter_wr(struct adapter *, int); static int del_filter_wr(struct adapter *, int); static int set_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); static int get_sge_context(struct adapter *, struct t4_sge_context *); static int load_fw(struct adapter *, struct t4_data *); static int load_cfg(struct adapter *, struct t4_data *); static int load_boot(struct adapter *, struct t4_bootrom *); static int load_bootcfg(struct adapter *, struct t4_data *); static int read_card_mem(struct adapter *, int, struct t4_mem_range *); static int read_i2c(struct adapter *, struct t4_i2c_data *); #ifdef TCP_OFFLOAD static int toe_capability(struct vi_info *, int); #endif static int mod_event(module_t, int, void *); static int notify_siblings(device_t, int); struct { uint16_t device; char *desc; } t4_pciids[] = { {0xa000, "Chelsio Terminator 4 FPGA"}, {0x4400, "Chelsio T440-dbg"}, {0x4401, "Chelsio T420-CR"}, {0x4402, "Chelsio T422-CR"}, {0x4403, "Chelsio T440-CR"}, {0x4404, "Chelsio T420-BCH"}, {0x4405, "Chelsio T440-BCH"}, {0x4406, "Chelsio T440-CH"}, {0x4407, "Chelsio T420-SO"}, {0x4408, "Chelsio T420-CX"}, {0x4409, "Chelsio T420-BT"}, {0x440a, "Chelsio T404-BT"}, {0x440e, "Chelsio T440-LP-CR"}, }, t5_pciids[] = { {0xb000, "Chelsio Terminator 5 FPGA"}, {0x5400, "Chelsio T580-dbg"}, {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ #ifdef notyet {0x5404, "Chelsio T520-BCH"}, {0x5405, "Chelsio T540-BCH"}, {0x5406, "Chelsio T540-CH"}, {0x5408, "Chelsio T520-CX"}, {0x540b, "Chelsio B520-SR"}, {0x540c, "Chelsio B504-BT"}, {0x540f, "Chelsio Amsterdam"}, {0x5413, "Chelsio T580-CHR"}, #endif }, t6_pciids[] = { {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */ {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ {0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */ {0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */ {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ {0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */ {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ /* Custom */ {0x6480, "Chelsio T6225 80"}, {0x6481, "Chelsio T62100 81"}, }; #ifdef TCP_OFFLOAD /* * service_iq() has an iq and needs the fl. Offset of fl from the iq should be * exactly the same for both rxq and ofld_rxq. */ CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); #endif CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); static int t4_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); uint8_t f = pci_get_function(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); /* Attach only to PF0 of the FPGA */ if (d == 0xa000 && f != 0) return (ENXIO); for (i = 0; i < nitems(t4_pciids); i++) { if (d == t4_pciids[i].device) { device_set_desc(dev, t4_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t5_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); uint8_t f = pci_get_function(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); /* Attach only to PF0 of the FPGA */ if (d == 0xb000 && f != 0) return (ENXIO); for (i = 0; i < nitems(t5_pciids); i++) { if (d == t5_pciids[i].device) { device_set_desc(dev, t5_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t6_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); for (i = 0; i < nitems(t6_pciids); i++) { if (d == t6_pciids[i].device) { device_set_desc(dev, t6_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static void t5_attribute_workaround(device_t dev) { device_t root_port; uint32_t v; /* * The T5 chips do not properly echo the No Snoop and Relaxed * Ordering attributes when replying to a TLP from a Root * Port. As a workaround, find the parent Root Port and * disable No Snoop and Relaxed Ordering. Note that this * affects all devices under this root port. */ root_port = pci_find_pcie_root_port(dev); if (root_port == NULL) { device_printf(dev, "Unable to find parent root port\n"); return; } v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 0) device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", device_get_nameunit(root_port)); } static const struct devnames devnames[] = { { .nexus_name = "t4nex", .ifnet_name = "cxgbe", .vi_ifnet_name = "vcxgbe", .pf03_drv_name = "t4iov", .vf_nexus_name = "t4vf", .vf_ifnet_name = "cxgbev" }, { .nexus_name = "t5nex", .ifnet_name = "cxl", .vi_ifnet_name = "vcxl", .pf03_drv_name = "t5iov", .vf_nexus_name = "t5vf", .vf_ifnet_name = "cxlv" }, { .nexus_name = "t6nex", .ifnet_name = "cc", .vi_ifnet_name = "vcc", .pf03_drv_name = "t6iov", .vf_nexus_name = "t6vf", .vf_ifnet_name = "ccv" } }; void t4_init_devnames(struct adapter *sc) { int id; id = chip_id(sc); if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames)) sc->names = &devnames[id - CHELSIO_T4]; else { device_printf(sc->dev, "chip id %d is not supported.\n", id); sc->names = NULL; } } static int t4_attach(device_t dev) { struct adapter *sc; int rc = 0, i, j, n10g, n1g, rqidx, tqidx; struct make_dev_args mda; struct intrs_and_queues iaq; struct sge *s; uint8_t *buf; #ifdef TCP_OFFLOAD int ofld_rqidx, ofld_tqidx; #endif #ifdef DEV_NETMAP int nm_rqidx, nm_tqidx; #endif int num_vis; sc = device_get_softc(dev); sc->dev = dev; TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); if ((pci_get_device(dev) & 0xff00) == 0x5400) t5_attribute_workaround(dev); pci_enable_busmaster(dev); if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { uint32_t v; pci_set_max_read_req(dev, 4096); v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); v |= PCIEM_CTL_RELAXED_ORD_ENABLE; pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); } sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); sc->traceq = -1; mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", device_get_nameunit(dev)); snprintf(sc->lockname, sizeof(sc->lockname), "%s", device_get_nameunit(dev)); mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); t4_add_adapter(sc); mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); TAILQ_INIT(&sc->sfl); callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); rc = t4_map_bars_0_and_4(sc); if (rc != 0) goto done; /* error message displayed already */ memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); /* Prepare the adapter for operation. */ buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); rc = -t4_prep_adapter(sc, buf); free(buf, M_CXGBE); if (rc != 0) { device_printf(dev, "failed to prepare adapter: %d.\n", rc); goto done; } /* * This is the real PF# to which we're attaching. Works from within PCI * passthrough environments too, where pci_get_function() could return a * different PF# depending on the passthrough configuration. We need to * use the real PF# in all our communication with the firmware. */ j = t4_read_reg(sc, A_PL_WHOAMI); sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j); sc->mbox = sc->pf; t4_init_devnames(sc); if (sc->names == NULL) { rc = ENOTSUP; goto done; /* error message displayed already */ } /* * Do this really early, with the memory windows set up even before the * character device. The userland tool's register i/o and mem read * will work even in "recovery mode". */ setup_memwin(sc); if (t4_init_devlog_params(sc, 0) == 0) fixup_devlog_params(sc); make_dev_args_init(&mda); mda.mda_devsw = &t4_cdevsw; mda.mda_uid = UID_ROOT; mda.mda_gid = GID_WHEEL; mda.mda_mode = 0600; mda.mda_si_drv1 = sc; rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); if (rc != 0) device_printf(dev, "failed to create nexus char device: %d.\n", rc); /* Go no further if recovery mode has been requested. */ if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { device_printf(dev, "recovery mode.\n"); goto done; } #if defined(__i386__) if ((cpu_feature & CPUID_CX8) == 0) { device_printf(dev, "64 bit atomics not available.\n"); rc = ENOTSUP; goto done; } #endif /* Prepare the firmware for operation */ rc = prep_firmware(sc); if (rc != 0) goto done; /* error message displayed already */ rc = get_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = set_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_map_bar_2(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_create_dma_tag(sc); if (rc != 0) goto done; /* error message displayed already */ /* * Number of VIs to create per-port. The first VI is the "main" regular * VI for the port. The rest are additional virtual interfaces on the * same physical port. Note that the main VI does not have native * netmap support but the extra VIs do. * * Limit the number of VIs per port to the number of available * MAC addresses per port. */ if (t4_num_vis >= 1) num_vis = t4_num_vis; else num_vis = 1; if (num_vis > nitems(vi_mac_funcs)) { num_vis = nitems(vi_mac_funcs); device_printf(dev, "Number of VIs limited to %d\n", num_vis); } /* * First pass over all the ports - allocate VIs and initialize some * basic parameters like mac address, port type, etc. We also figure * out whether a port is 10G or 1G and use that information when * calculating how many interrupts to attempt to allocate. */ n10g = n1g = 0; for_each_port(sc, i) { struct port_info *pi; pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); sc->port[i] = pi; /* These must be set before t4_port_init */ pi->adapter = sc; pi->port_id = i; /* * XXX: vi[0] is special so we can't delay this allocation until * pi->nvi's final value is known. */ pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, M_ZERO | M_WAITOK); /* * Allocate the "main" VI and initialize parameters * like mac addr. */ rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); if (rc != 0) { device_printf(dev, "unable to initialize port %d: %d\n", i, rc); free(pi->vi, M_CXGBE); free(pi, M_CXGBE); sc->port[i] = NULL; goto done; } snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", device_get_nameunit(dev), i); mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); sc->chan_map[pi->tx_chan] = i; if (port_top_speed(pi) >= 10) { n10g++; } else { n1g++; } pi->dev = device_add_child(dev, sc->names->ifnet_name, -1); if (pi->dev == NULL) { device_printf(dev, "failed to add device for port %d.\n", i); rc = ENXIO; goto done; } pi->vi[0].dev = pi->dev; device_set_softc(pi->dev, pi); } /* * Interrupt type, # of interrupts, # of rx/tx queues, etc. */ rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); if (rc != 0) goto done; /* error message displayed already */ if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0) num_vis = 1; sc->intr_type = iaq.intr_type; sc->intr_count = iaq.nirq; s = &sc->sge; s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; if (num_vis > 1) { s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi; s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi; } s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ #ifdef TCP_OFFLOAD if (is_offload(sc)) { s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; if (num_vis > 1) { s->nofldrxq += (n10g + n1g) * (num_vis - 1) * iaq.nofldrxq_vi; s->nofldtxq += (n10g + n1g) * (num_vis - 1) * iaq.nofldtxq_vi; } s->neq += s->nofldtxq + s->nofldrxq; s->niq += s->nofldrxq; s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), M_CXGBE, M_ZERO | M_WAITOK); } #endif #ifdef DEV_NETMAP if (num_vis > 1) { s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi; s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi; } s->neq += s->nnmtxq + s->nnmrxq; s->niq += s->nnmrxq; s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), M_CXGBE, M_ZERO | M_WAITOK); #endif s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, M_ZERO | M_WAITOK); s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, M_ZERO | M_WAITOK); s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, M_ZERO | M_WAITOK); s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, M_ZERO | M_WAITOK); sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, M_ZERO | M_WAITOK); t4_init_l2t(sc, M_WAITOK); t4_init_tx_sched(sc); /* * Second pass over the ports. This time we know the number of rx and * tx queues that each port should get. */ rqidx = tqidx = 0; #ifdef TCP_OFFLOAD ofld_rqidx = ofld_tqidx = 0; #endif #ifdef DEV_NETMAP nm_rqidx = nm_tqidx = 0; #endif for_each_port(sc, i) { struct port_info *pi = sc->port[i]; struct vi_info *vi; if (pi == NULL) continue; pi->nvi = num_vis; for_each_vi(pi, j, vi) { vi->pi = pi; vi->qsize_rxq = t4_qsize_rxq; vi->qsize_txq = t4_qsize_txq; vi->first_rxq = rqidx; vi->first_txq = tqidx; if (port_top_speed(pi) >= 10) { vi->tmr_idx = t4_tmr_idx_10g; vi->pktc_idx = t4_pktc_idx_10g; vi->flags |= iaq.intr_flags_10g & INTR_RXQ; vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi; vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi; } else { vi->tmr_idx = t4_tmr_idx_1g; vi->pktc_idx = t4_pktc_idx_1g; vi->flags |= iaq.intr_flags_1g & INTR_RXQ; vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi; vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi; } rqidx += vi->nrxq; tqidx += vi->ntxq; if (j == 0 && vi->ntxq > 1) vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; else vi->rsrv_noflowq = 0; #ifdef TCP_OFFLOAD vi->first_ofld_rxq = ofld_rqidx; vi->first_ofld_txq = ofld_tqidx; if (port_top_speed(pi) >= 10) { vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : iaq.nofldrxq_vi; vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : iaq.nofldtxq_vi; } else { vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : iaq.nofldrxq_vi; vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : iaq.nofldtxq_vi; } ofld_rqidx += vi->nofldrxq; ofld_tqidx += vi->nofldtxq; #endif #ifdef DEV_NETMAP if (j > 0) { vi->first_nm_rxq = nm_rqidx; vi->first_nm_txq = nm_tqidx; vi->nnmrxq = iaq.nnmrxq_vi; vi->nnmtxq = iaq.nnmtxq_vi; nm_rqidx += vi->nnmrxq; nm_tqidx += vi->nnmtxq; } #endif } } rc = t4_setup_intr_handlers(sc); if (rc != 0) { device_printf(dev, "failed to setup interrupt handlers: %d\n", rc); goto done; } rc = bus_generic_probe(dev); if (rc != 0) { device_printf(dev, "failed to probe child drivers: %d\n", rc); goto done; } rc = bus_generic_attach(dev); if (rc != 0) { device_printf(dev, "failed to attach all child ports: %d\n", rc); goto done; } device_printf(dev, "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", sc->params.pci.speed, sc->params.pci.width, sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); t4_set_desc(sc); notify_siblings(dev, 0); done: if (rc != 0 && sc->cdev) { /* cdev was created and so cxgbetool works; recover that way. */ device_printf(dev, "error during attach, adapter is now in recovery mode.\n"); rc = 0; } if (rc != 0) t4_detach_common(dev); else t4_sysctls(sc); return (rc); } static int t4_ready(device_t dev) { struct adapter *sc; sc = device_get_softc(dev); if (sc->flags & FW_OK) return (0); return (ENXIO); } static int t4_read_port_device(device_t dev, int port, device_t *child) { struct adapter *sc; struct port_info *pi; sc = device_get_softc(dev); if (port < 0 || port >= MAX_NPORTS) return (EINVAL); pi = sc->port[port]; if (pi == NULL || pi->dev == NULL) return (ENXIO); *child = pi->dev; return (0); } static int notify_siblings(device_t dev, int detaching) { device_t sibling; int error, i; error = 0; for (i = 0; i < PCI_FUNCMAX; i++) { if (i == pci_get_function(dev)) continue; sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), i); if (sibling == NULL || !device_is_attached(sibling)) continue; if (detaching) error = T4_DETACH_CHILD(sibling); else (void)T4_ATTACH_CHILD(sibling); if (error) break; } return (error); } /* * Idempotent */ static int t4_detach(device_t dev) { struct adapter *sc; int rc; sc = device_get_softc(dev); rc = notify_siblings(dev, 1); if (rc) { device_printf(dev, "failed to detach sibling devices: %d\n", rc); return (rc); } return (t4_detach_common(dev)); } int t4_detach_common(device_t dev) { struct adapter *sc; struct port_info *pi; int i, rc; sc = device_get_softc(dev); if (sc->flags & FULL_INIT_DONE) { if (!(sc->flags & IS_VF)) t4_intr_disable(sc); } if (sc->cdev) { destroy_dev(sc->cdev); sc->cdev = NULL; } if (device_is_attached(dev)) { rc = bus_generic_detach(dev); if (rc) { device_printf(dev, "failed to detach child devices: %d\n", rc); return (rc); } } for (i = 0; i < sc->intr_count; i++) t4_free_irq(sc, &sc->irq[i]); if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) t4_free_tx_sched(sc); for (i = 0; i < MAX_NPORTS; i++) { pi = sc->port[i]; if (pi) { t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); if (pi->dev) device_delete_child(dev, pi->dev); mtx_destroy(&pi->pi_lock); free(pi->vi, M_CXGBE); free(pi, M_CXGBE); } } device_delete_children(dev); if (sc->flags & FULL_INIT_DONE) adapter_full_uninit(sc); if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) t4_fw_bye(sc, sc->mbox); if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) pci_release_msi(dev); if (sc->regs_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, sc->regs_res); if (sc->udbs_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, sc->udbs_res); if (sc->msix_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, sc->msix_res); if (sc->l2t) t4_free_l2t(sc->l2t); #ifdef TCP_OFFLOAD free(sc->sge.ofld_rxq, M_CXGBE); free(sc->sge.ofld_txq, M_CXGBE); #endif #ifdef DEV_NETMAP free(sc->sge.nm_rxq, M_CXGBE); free(sc->sge.nm_txq, M_CXGBE); #endif free(sc->irq, M_CXGBE); free(sc->sge.rxq, M_CXGBE); free(sc->sge.txq, M_CXGBE); free(sc->sge.ctrlq, M_CXGBE); free(sc->sge.iqmap, M_CXGBE); free(sc->sge.eqmap, M_CXGBE); free(sc->tids.ftid_tab, M_CXGBE); t4_destroy_dma_tag(sc); if (mtx_initialized(&sc->sc_lock)) { sx_xlock(&t4_list_lock); SLIST_REMOVE(&t4_list, sc, adapter, link); sx_xunlock(&t4_list_lock); mtx_destroy(&sc->sc_lock); } callout_drain(&sc->sfl_callout); if (mtx_initialized(&sc->tids.ftid_lock)) mtx_destroy(&sc->tids.ftid_lock); if (mtx_initialized(&sc->sfl_lock)) mtx_destroy(&sc->sfl_lock); if (mtx_initialized(&sc->ifp_lock)) mtx_destroy(&sc->ifp_lock); if (mtx_initialized(&sc->reg_lock)) mtx_destroy(&sc->reg_lock); for (i = 0; i < NUM_MEMWIN; i++) { struct memwin *mw = &sc->memwin[i]; if (rw_initialized(&mw->mw_lock)) rw_destroy(&mw->mw_lock); } bzero(sc, sizeof(*sc)); return (0); } static int cxgbe_probe(device_t dev) { char buf[128]; struct port_info *pi = device_get_softc(dev); snprintf(buf, sizeof(buf), "port %d", pi->port_id); device_set_desc_copy(dev, buf); return (BUS_PROBE_DEFAULT); } #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) #define T4_CAP_ENABLE (T4_CAP) static int cxgbe_vi_attach(device_t dev, struct vi_info *vi) { struct ifnet *ifp; struct sbuf *sb; vi->xact_addr_filt = -1; callout_init(&vi->tick, 1); /* Allocate an ifnet and set it up */ ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "Cannot allocate ifnet\n"); return (ENOMEM); } vi->ifp = ifp; ifp->if_softc = vi; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_init = cxgbe_init; ifp->if_ioctl = cxgbe_ioctl; ifp->if_transmit = cxgbe_transmit; ifp->if_qflush = cxgbe_qflush; ifp->if_get_counter = cxgbe_get_counter; ifp->if_capabilities = T4_CAP; #ifdef TCP_OFFLOAD if (vi->nofldrxq != 0) ifp->if_capabilities |= IFCAP_TOE; #endif #ifdef DEV_NETMAP if (vi->nnmrxq != 0) ifp->if_capabilities |= IFCAP_NETMAP; #endif ifp->if_capenable = T4_CAP_ENABLE; ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | CSUM_UDP_IPV6 | CSUM_TCP_IPV6; ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; ifp->if_hw_tsomaxsegsize = 65536; /* Initialize ifmedia for this VI */ ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, cxgbe_media_status); vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, EVENTHANDLER_PRI_ANY); ether_ifattach(ifp, vi->hw_addr); #ifdef DEV_NETMAP if (ifp->if_capabilities & IFCAP_NETMAP) cxgbe_nm_attach(vi); #endif sb = sbuf_new_auto(); sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); #ifdef TCP_OFFLOAD if (ifp->if_capabilities & IFCAP_TOE) sbuf_printf(sb, "; %d txq, %d rxq (TOE)", vi->nofldtxq, vi->nofldrxq); #endif #ifdef DEV_NETMAP if (ifp->if_capabilities & IFCAP_NETMAP) sbuf_printf(sb, "; %d txq, %d rxq (netmap)", vi->nnmtxq, vi->nnmrxq); #endif sbuf_finish(sb); device_printf(dev, "%s\n", sbuf_data(sb)); sbuf_delete(sb); vi_sysctls(vi); return (0); } static int cxgbe_attach(device_t dev) { struct port_info *pi = device_get_softc(dev); struct adapter *sc = pi->adapter; struct vi_info *vi; int i, rc; callout_init_mtx(&pi->tick, &pi->pi_lock, 0); rc = cxgbe_vi_attach(dev, &pi->vi[0]); if (rc) return (rc); for_each_vi(pi, i, vi) { if (i == 0) continue; vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1); if (vi->dev == NULL) { device_printf(dev, "failed to add VI %d\n", i); continue; } device_set_softc(vi->dev, vi); } cxgbe_sysctls(pi); bus_generic_attach(dev); return (0); } static void cxgbe_vi_detach(struct vi_info *vi) { struct ifnet *ifp = vi->ifp; ether_ifdetach(ifp); if (vi->vlan_c) EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); /* Let detach proceed even if these fail. */ #ifdef DEV_NETMAP if (ifp->if_capabilities & IFCAP_NETMAP) cxgbe_nm_detach(vi); #endif cxgbe_uninit_synchronized(vi); callout_drain(&vi->tick); vi_full_uninit(vi); ifmedia_removeall(&vi->media); if_free(vi->ifp); vi->ifp = NULL; } static int cxgbe_detach(device_t dev) { struct port_info *pi = device_get_softc(dev); struct adapter *sc = pi->adapter; int rc; /* Detach the extra VIs first. */ rc = bus_generic_detach(dev); if (rc) return (rc); device_delete_children(dev); doom_vi(sc, &pi->vi[0]); if (pi->flags & HAS_TRACEQ) { sc->traceq = -1; /* cloner should not create ifnet */ t4_tracer_port_detach(sc); } cxgbe_vi_detach(&pi->vi[0]); callout_drain(&pi->tick); end_synchronized_op(sc, 0); return (0); } static void cxgbe_init(void *arg) { struct vi_info *vi = arg; struct adapter *sc = vi->pi->adapter; if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) return; cxgbe_init_synchronized(vi); end_synchronized_op(sc, 0); } static int cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) { int rc = 0, mtu, flags, can_sleep; struct vi_info *vi = ifp->if_softc; struct adapter *sc = vi->pi->adapter; struct ifreq *ifr = (struct ifreq *)data; uint32_t mask; switch (cmd) { case SIOCSIFMTU: mtu = ifr->ifr_mtu; if (mtu < ETHERMIN || mtu > MAX_MTU) return (EINVAL); rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); if (rc) return (rc); ifp->if_mtu = mtu; if (vi->flags & VI_INIT_DONE) { t4_update_fl_bufsize(ifp); if (ifp->if_drv_flags & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_MTU); } end_synchronized_op(sc, 0); break; case SIOCSIFFLAGS: can_sleep = 0; redo_sifflags: rc = begin_synchronized_op(sc, vi, can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); if (rc) return (rc); if (ifp->if_flags & IFF_UP) { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { flags = vi->if_flags; if ((ifp->if_flags ^ flags) & (IFF_PROMISC | IFF_ALLMULTI)) { if (can_sleep == 1) { end_synchronized_op(sc, 0); can_sleep = 0; goto redo_sifflags; } rc = update_mac_settings(ifp, XGMAC_PROMISC | XGMAC_ALLMULTI); } } else { if (can_sleep == 0) { end_synchronized_op(sc, LOCK_HELD); can_sleep = 1; goto redo_sifflags; } rc = cxgbe_init_synchronized(vi); } vi->if_flags = ifp->if_flags; } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (can_sleep == 0) { end_synchronized_op(sc, LOCK_HELD); can_sleep = 1; goto redo_sifflags; } rc = cxgbe_uninit_synchronized(vi); } end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); break; case SIOCADDMULTI: case SIOCDELMULTI: /* these two are called with a mutex held :-( */ rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); if (rc) return (rc); if (ifp->if_drv_flags & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_MCADDRS); end_synchronized_op(sc, LOCK_HELD); break; case SIOCSIFCAP: rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); if (rc) return (rc); mask = ifr->ifr_reqcap ^ ifp->if_capenable; if (mask & IFCAP_TXCSUM) { ifp->if_capenable ^= IFCAP_TXCSUM; ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); if (IFCAP_TSO4 & ifp->if_capenable && !(IFCAP_TXCSUM & ifp->if_capenable)) { ifp->if_capenable &= ~IFCAP_TSO4; if_printf(ifp, "tso4 disabled due to -txcsum.\n"); } } if (mask & IFCAP_TXCSUM_IPV6) { ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); if (IFCAP_TSO6 & ifp->if_capenable && !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { ifp->if_capenable &= ~IFCAP_TSO6; if_printf(ifp, "tso6 disabled due to -txcsum6.\n"); } } if (mask & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; if (mask & IFCAP_RXCSUM_IPV6) ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; /* * Note that we leave CSUM_TSO alone (it is always set). The * kernel takes both IFCAP_TSOx and CSUM_TSO into account before * sending a TSO request our way, so it's sufficient to toggle * IFCAP_TSOx only. */ if (mask & IFCAP_TSO4) { if (!(IFCAP_TSO4 & ifp->if_capenable) && !(IFCAP_TXCSUM & ifp->if_capenable)) { if_printf(ifp, "enable txcsum first.\n"); rc = EAGAIN; goto fail; } ifp->if_capenable ^= IFCAP_TSO4; } if (mask & IFCAP_TSO6) { if (!(IFCAP_TSO6 & ifp->if_capenable) && !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { if_printf(ifp, "enable txcsum6 first.\n"); rc = EAGAIN; goto fail; } ifp->if_capenable ^= IFCAP_TSO6; } if (mask & IFCAP_LRO) { #if defined(INET) || defined(INET6) int i; struct sge_rxq *rxq; ifp->if_capenable ^= IFCAP_LRO; for_each_rxq(vi, i, rxq) { if (ifp->if_capenable & IFCAP_LRO) rxq->iq.flags |= IQ_LRO_ENABLED; else rxq->iq.flags &= ~IQ_LRO_ENABLED; } #endif } #ifdef TCP_OFFLOAD if (mask & IFCAP_TOE) { int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; rc = toe_capability(vi, enable); if (rc != 0) goto fail; ifp->if_capenable ^= mask; } #endif if (mask & IFCAP_VLAN_HWTAGGING) { ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (ifp->if_drv_flags & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_VLANEX); } if (mask & IFCAP_VLAN_MTU) { ifp->if_capenable ^= IFCAP_VLAN_MTU; /* Need to find out how to disable auto-mtu-inflation */ } if (mask & IFCAP_VLAN_HWTSO) ifp->if_capenable ^= IFCAP_VLAN_HWTSO; if (mask & IFCAP_VLAN_HWCSUM) ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; #ifdef VLAN_CAPABILITIES VLAN_CAPABILITIES(ifp); #endif fail: end_synchronized_op(sc, 0); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: case SIOCGIFXMEDIA: ifmedia_ioctl(ifp, ifr, &vi->media, cmd); break; case SIOCGI2C: { struct ifi2creq i2c; rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); if (rc != 0) break; if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { rc = EPERM; break; } if (i2c.len > sizeof(i2c.data)) { rc = EINVAL; break; } rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); if (rc) return (rc); rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, i2c.offset, i2c.len, &i2c.data[0]); end_synchronized_op(sc, 0); if (rc == 0) rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); break; } default: rc = ether_ioctl(ifp, cmd, data); } return (rc); } static int cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) { struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_txq *txq; void *items[1]; int rc; M_ASSERTPKTHDR(m); MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ if (__predict_false(pi->link_cfg.link_ok == 0)) { m_freem(m); return (ENETDOWN); } rc = parse_pkt(sc, &m); if (__predict_false(rc != 0)) { MPASS(m == NULL); /* was freed already */ atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ return (rc); } /* Select a txq. */ txq = &sc->sge.txq[vi->first_txq]; if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + vi->rsrv_noflowq); items[0] = m; rc = mp_ring_enqueue(txq->r, items, 1, 4096); if (__predict_false(rc != 0)) m_freem(m); return (rc); } static void cxgbe_qflush(struct ifnet *ifp) { struct vi_info *vi = ifp->if_softc; struct sge_txq *txq; int i; /* queues do not exist if !VI_INIT_DONE. */ if (vi->flags & VI_INIT_DONE) { for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags |= EQ_QFLUSH; TXQ_UNLOCK(txq); while (!mp_ring_is_idle(txq->r)) { mp_ring_check_drainage(txq->r, 0); pause("qflush", 1); } TXQ_LOCK(txq); txq->eq.flags &= ~EQ_QFLUSH; TXQ_UNLOCK(txq); } } if_qflush(ifp); } static uint64_t vi_get_counter(struct ifnet *ifp, ift_counter c) { struct vi_info *vi = ifp->if_softc; struct fw_vi_stats_vf *s = &vi->stats; vi_refresh_stats(vi->pi->adapter, vi); switch (c) { case IFCOUNTER_IPACKETS: return (s->rx_bcast_frames + s->rx_mcast_frames + s->rx_ucast_frames); case IFCOUNTER_IERRORS: return (s->rx_err_frames); case IFCOUNTER_OPACKETS: return (s->tx_bcast_frames + s->tx_mcast_frames + s->tx_ucast_frames + s->tx_offload_frames); case IFCOUNTER_OERRORS: return (s->tx_drop_frames); case IFCOUNTER_IBYTES: return (s->rx_bcast_bytes + s->rx_mcast_bytes + s->rx_ucast_bytes); case IFCOUNTER_OBYTES: return (s->tx_bcast_bytes + s->tx_mcast_bytes + s->tx_ucast_bytes + s->tx_offload_bytes); case IFCOUNTER_IMCASTS: return (s->rx_mcast_frames); case IFCOUNTER_OMCASTS: return (s->tx_mcast_frames); case IFCOUNTER_OQDROPS: { uint64_t drops; drops = 0; if (vi->flags & VI_INIT_DONE) { int i; struct sge_txq *txq; for_each_txq(vi, i, txq) drops += counter_u64_fetch(txq->r->drops); } return (drops); } default: return (if_get_counter_default(ifp, c)); } } uint64_t cxgbe_get_counter(struct ifnet *ifp, ift_counter c) { struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct port_stats *s = &pi->stats; if (pi->nvi > 1 || sc->flags & IS_VF) return (vi_get_counter(ifp, c)); cxgbe_refresh_stats(sc, pi); switch (c) { case IFCOUNTER_IPACKETS: return (s->rx_frames); case IFCOUNTER_IERRORS: return (s->rx_jabber + s->rx_runt + s->rx_too_long + s->rx_fcs_err + s->rx_len_err); case IFCOUNTER_OPACKETS: return (s->tx_frames); case IFCOUNTER_OERRORS: return (s->tx_error_frames); case IFCOUNTER_IBYTES: return (s->rx_octets); case IFCOUNTER_OBYTES: return (s->tx_octets); case IFCOUNTER_IMCASTS: return (s->rx_mcast_frames); case IFCOUNTER_OMCASTS: return (s->tx_mcast_frames); case IFCOUNTER_IQDROPS: return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + s->rx_trunc3 + pi->tnl_cong_drops); case IFCOUNTER_OQDROPS: { uint64_t drops; drops = s->tx_drop; if (vi->flags & VI_INIT_DONE) { int i; struct sge_txq *txq; for_each_txq(vi, i, txq) drops += counter_u64_fetch(txq->r->drops); } return (drops); } default: return (if_get_counter_default(ifp, c)); } } static int cxgbe_media_change(struct ifnet *ifp) { struct vi_info *vi = ifp->if_softc; device_printf(vi->dev, "%s unimplemented.\n", __func__); return (EOPNOTSUPP); } static void cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct ifmedia_entry *cur; struct link_config *lc = &pi->link_cfg; /* * If all the interfaces are administratively down the firmware does not * report transceiver changes. Refresh port info here so that ifconfig * displays accurate information at all times. */ if (begin_synchronized_op(pi->adapter, NULL, SLEEP_OK | INTR_OK, "t4med") == 0) { PORT_LOCK(pi); if (pi->up_vis == 0) { t4_update_port_info(pi); build_medialist(pi, &vi->media); } PORT_UNLOCK(pi); end_synchronized_op(pi->adapter, 0); } cur = vi->media.ifm_cur; ifmr->ifm_status = IFM_AVALID; if (lc->link_ok == 0) return; ifmr->ifm_status |= IFM_ACTIVE; ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE); if (lc->fc & PAUSE_RX) ifmr->ifm_active |= IFM_ETH_RXPAUSE; if (lc->fc & PAUSE_TX) ifmr->ifm_active |= IFM_ETH_TXPAUSE; /* active and current will differ iff current media is autoselect. */ if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) return; ifmr->ifm_active = IFM_ETHER | IFM_FDX; if (lc->fc & PAUSE_RX) ifmr->ifm_active |= IFM_ETH_RXPAUSE; if (lc->fc & PAUSE_TX) ifmr->ifm_active |= IFM_ETH_TXPAUSE; switch (lc->speed) { case 10000: ifmr->ifm_active |= IFM_10G_T; break; case 1000: ifmr->ifm_active |= IFM_1000_T; break; case 100: ifmr->ifm_active |= IFM_100_TX; break; case 10: ifmr->ifm_active |= IFM_10_T; break; default: device_printf(vi->dev, "link up but speed unknown (%u)\n", lc->speed); } } static int vcxgbe_probe(device_t dev) { char buf[128]; struct vi_info *vi = device_get_softc(dev); snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, vi - vi->pi->vi); device_set_desc_copy(dev, buf); return (BUS_PROBE_DEFAULT); } static int vcxgbe_attach(device_t dev) { struct vi_info *vi; struct port_info *pi; struct adapter *sc; int func, index, rc; u32 param, val; vi = device_get_softc(dev); pi = vi->pi; sc = pi->adapter; index = vi - pi->vi; KASSERT(index < nitems(vi_mac_funcs), ("%s: VI %s doesn't have a MAC func", __func__, device_get_nameunit(dev))); func = vi_mac_funcs[index]; rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, vi->hw_addr, &vi->rss_size, func, 0); if (rc < 0) { device_printf(dev, "Failed to allocate virtual interface " "for port %d: %d\n", pi->port_id, -rc); return (-rc); } vi->viid = rc; if (chip_id(sc) <= CHELSIO_T5) vi->smt_idx = (rc & 0x7f) << 1; else vi->smt_idx = (rc & 0x7f); param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | V_FW_PARAMS_PARAM_YZ(vi->viid); rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc) vi->rss_base = 0xffff; else { /* MPASS((val >> 16) == rss_size); */ vi->rss_base = val & 0xffff; } rc = cxgbe_vi_attach(dev, vi); if (rc) { t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); return (rc); } return (0); } static int vcxgbe_detach(device_t dev) { struct vi_info *vi; struct adapter *sc; vi = device_get_softc(dev); sc = vi->pi->adapter; doom_vi(sc, vi); cxgbe_vi_detach(vi); t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); end_synchronized_op(sc, 0); return (0); } void t4_fatal_err(struct adapter *sc) { t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); t4_intr_disable(sc); log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", device_get_nameunit(sc->dev)); } void t4_add_adapter(struct adapter *sc) { sx_xlock(&t4_list_lock); SLIST_INSERT_HEAD(&t4_list, sc, link); sx_xunlock(&t4_list_lock); } int t4_map_bars_0_and_4(struct adapter *sc) { sc->regs_rid = PCIR_BAR(0); sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->regs_rid, RF_ACTIVE); if (sc->regs_res == NULL) { device_printf(sc->dev, "cannot map registers.\n"); return (ENXIO); } sc->bt = rman_get_bustag(sc->regs_res); sc->bh = rman_get_bushandle(sc->regs_res); sc->mmio_len = rman_get_size(sc->regs_res); setbit(&sc->doorbells, DOORBELL_KDB); sc->msix_rid = PCIR_BAR(4); sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->msix_rid, RF_ACTIVE); if (sc->msix_res == NULL) { device_printf(sc->dev, "cannot map MSI-X BAR.\n"); return (ENXIO); } return (0); } int t4_map_bar_2(struct adapter *sc) { /* * T4: only iWARP driver uses the userspace doorbells. There is no need * to map it if RDMA is disabled. */ if (is_t4(sc) && sc->rdmacaps == 0) return (0); sc->udbs_rid = PCIR_BAR(2); sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->udbs_rid, RF_ACTIVE); if (sc->udbs_res == NULL) { device_printf(sc->dev, "cannot map doorbell BAR.\n"); return (ENXIO); } sc->udbs_base = rman_get_virtual(sc->udbs_res); if (chip_id(sc) >= CHELSIO_T5) { setbit(&sc->doorbells, DOORBELL_UDB); #if defined(__i386__) || defined(__amd64__) if (t5_write_combine) { int rc, mode; /* * Enable write combining on BAR2. This is the * userspace doorbell BAR and is split into 128B * (UDBS_SEG_SIZE) doorbell regions, each associated * with an egress queue. The first 64B has the doorbell * and the second 64B can be used to submit a tx work * request with an implicit doorbell. */ rc = pmap_change_attr((vm_offset_t)sc->udbs_base, rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); if (rc == 0) { clrbit(&sc->doorbells, DOORBELL_UDB); setbit(&sc->doorbells, DOORBELL_WCWR); setbit(&sc->doorbells, DOORBELL_UDBWC); } else { device_printf(sc->dev, "couldn't enable write combining: %d\n", rc); } mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0); t4_write_reg(sc, A_SGE_STAT_CFG, V_STATSOURCE_T5(7) | mode); } #endif } return (0); } struct memwin_init { uint32_t base; uint32_t aperture; }; static const struct memwin_init t4_memwin[NUM_MEMWIN] = { { MEMWIN0_BASE, MEMWIN0_APERTURE }, { MEMWIN1_BASE, MEMWIN1_APERTURE }, { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } }; static const struct memwin_init t5_memwin[NUM_MEMWIN] = { { MEMWIN0_BASE, MEMWIN0_APERTURE }, { MEMWIN1_BASE, MEMWIN1_APERTURE }, { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, }; static void setup_memwin(struct adapter *sc) { const struct memwin_init *mw_init; struct memwin *mw; int i; uint32_t bar0; if (is_t4(sc)) { /* * Read low 32b of bar0 indirectly via the hardware backdoor * mechanism. Works from within PCI passthrough environments * too, where rman_get_start() can return a different value. We * need to program the T4 memory window decoders with the actual * addresses that will be coming across the PCIe link. */ bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; mw_init = &t4_memwin[0]; } else { /* T5+ use the relative offset inside the PCIe BAR */ bar0 = 0; mw_init = &t5_memwin[0]; } for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) { rw_init(&mw->mw_lock, "memory window access"); mw->mw_base = mw_init->base; mw->mw_aperture = mw_init->aperture; mw->mw_curpos = 0; t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), (mw->mw_base + bar0) | V_BIR(0) | V_WINDOW(ilog2(mw->mw_aperture) - 10)); rw_wlock(&mw->mw_lock); position_memwin(sc, i, 0); rw_wunlock(&mw->mw_lock); } /* flush */ t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); } /* * Positions the memory window at the given address in the card's address space. * There are some alignment requirements and the actual position may be at an * address prior to the requested address. mw->mw_curpos always has the actual * position of the window. */ static void position_memwin(struct adapter *sc, int idx, uint32_t addr) { struct memwin *mw; uint32_t pf; uint32_t reg; MPASS(idx >= 0 && idx < NUM_MEMWIN); mw = &sc->memwin[idx]; rw_assert(&mw->mw_lock, RA_WLOCKED); if (is_t4(sc)) { pf = 0; mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */ } else { pf = V_PFNUM(sc->pf); mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */ } reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx); t4_write_reg(sc, reg, mw->mw_curpos | pf); t4_read_reg(sc, reg); /* flush */ } static int rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len, int rw) { struct memwin *mw; uint32_t mw_end, v; MPASS(idx >= 0 && idx < NUM_MEMWIN); /* Memory can only be accessed in naturally aligned 4 byte units */ if (addr & 3 || len & 3 || len <= 0) return (EINVAL); mw = &sc->memwin[idx]; while (len > 0) { rw_rlock(&mw->mw_lock); mw_end = mw->mw_curpos + mw->mw_aperture; if (addr >= mw_end || addr < mw->mw_curpos) { /* Will need to reposition the window */ if (!rw_try_upgrade(&mw->mw_lock)) { rw_runlock(&mw->mw_lock); rw_wlock(&mw->mw_lock); } rw_assert(&mw->mw_lock, RA_WLOCKED); position_memwin(sc, idx, addr); rw_downgrade(&mw->mw_lock); mw_end = mw->mw_curpos + mw->mw_aperture; } rw_assert(&mw->mw_lock, RA_RLOCKED); while (addr < mw_end && len > 0) { if (rw == 0) { v = t4_read_reg(sc, mw->mw_base + addr - mw->mw_curpos); *val++ = le32toh(v); } else { v = *val++; t4_write_reg(sc, mw->mw_base + addr - mw->mw_curpos, htole32(v)); } addr += 4; len -= 4; } rw_runlock(&mw->mw_lock); } return (0); } static inline int read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, val, len, 0)); } static inline int write_via_memwin(struct adapter *sc, int idx, uint32_t addr, const uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1)); } static int t4_range_cmp(const void *a, const void *b) { return ((const struct t4_range *)a)->start - ((const struct t4_range *)b)->start; } /* * Verify that the memory range specified by the addr/len pair is valid within * the card's address space. */ static int validate_mem_range(struct adapter *sc, uint32_t addr, int len) { struct t4_range mem_ranges[4], *r, *next; uint32_t em, addr_len; int i, n, remaining; /* Memory can only be accessed in naturally aligned 4 byte units */ if (addr & 3 || len & 3 || len <= 0) return (EINVAL); /* Enabled memories */ em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); r = &mem_ranges[0]; n = 0; bzero(r, sizeof(mem_ranges)); if (em & F_EDRAM0_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); r->size = G_EDRAM0_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EDRAM0_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (em & F_EDRAM1_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); r->size = G_EDRAM1_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EDRAM1_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (em & F_EXT_MEM_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); r->size = G_EXT_MEM_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EXT_MEM_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); r->size = G_EXT_MEM1_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EXT_MEM1_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } MPASS(n <= nitems(mem_ranges)); if (n > 1) { /* Sort and merge the ranges. */ qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp); /* Start from index 0 and examine the next n - 1 entries. */ r = &mem_ranges[0]; for (remaining = n - 1; remaining > 0; remaining--, r++) { MPASS(r->size > 0); /* r is a valid entry. */ next = r + 1; MPASS(next->size > 0); /* and so is the next one. */ while (r->start + r->size >= next->start) { /* Merge the next one into the current entry. */ r->size = max(r->start + r->size, next->start + next->size) - r->start; n--; /* One fewer entry in total. */ if (--remaining == 0) goto done; /* short circuit */ next++; } if (next != r + 1) { /* * Some entries were merged into r and next * points to the first valid entry that couldn't * be merged. */ MPASS(next->size > 0); /* must be valid */ memcpy(r + 1, next, remaining * sizeof(*r)); #ifdef INVARIANTS /* * This so that the foo->size assertion in the * next iteration of the loop do the right * thing for entries that were pulled up and are * no longer valid. */ MPASS(n < nitems(mem_ranges)); bzero(&mem_ranges[n], (nitems(mem_ranges) - n) * sizeof(struct t4_range)); #endif } } done: /* Done merging the ranges. */ MPASS(n > 0); r = &mem_ranges[0]; for (i = 0; i < n; i++, r++) { if (addr >= r->start && addr + len <= r->start + r->size) return (0); } } return (EFAULT); } static int fwmtype_to_hwmtype(int mtype) { switch (mtype) { case FW_MEMTYPE_EDC0: return (MEM_EDC0); case FW_MEMTYPE_EDC1: return (MEM_EDC1); case FW_MEMTYPE_EXTMEM: return (MEM_MC0); case FW_MEMTYPE_EXTMEM1: return (MEM_MC1); default: panic("%s: cannot translate fw mtype %d.", __func__, mtype); } } /* * Verify that the memory range specified by the memtype/offset/len pair is * valid and lies entirely within the memtype specified. The global address of * the start of the range is returned in addr. */ static int validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, uint32_t *addr) { uint32_t em, addr_len, maddr; /* Memory can only be accessed in naturally aligned 4 byte units */ if (off & 3 || len & 3 || len == 0) return (EINVAL); em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); switch (fwmtype_to_hwmtype(mtype)) { case MEM_EDC0: if (!(em & F_EDRAM0_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); maddr = G_EDRAM0_BASE(addr_len) << 20; break; case MEM_EDC1: if (!(em & F_EDRAM1_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); maddr = G_EDRAM1_BASE(addr_len) << 20; break; case MEM_MC: if (!(em & F_EXT_MEM_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); maddr = G_EXT_MEM_BASE(addr_len) << 20; break; case MEM_MC1: if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); maddr = G_EXT_MEM1_BASE(addr_len) << 20; break; default: return (EINVAL); } *addr = maddr + off; /* global address */ return (validate_mem_range(sc, *addr, len)); } static int fixup_devlog_params(struct adapter *sc) { struct devlog_params *dparams = &sc->params.devlog; int rc; rc = validate_mt_off_len(sc, dparams->memtype, dparams->start, dparams->size, &dparams->addr); return (rc); } static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, struct intrs_and_queues *iaq) { int rc, itype, navail, nrxq10g, nrxq1g, n; int nofldrxq10g = 0, nofldrxq1g = 0; bzero(iaq, sizeof(*iaq)); iaq->ntxq10g = t4_ntxq10g; iaq->ntxq1g = t4_ntxq1g; iaq->ntxq_vi = t4_ntxq_vi; iaq->nrxq10g = nrxq10g = t4_nrxq10g; iaq->nrxq1g = nrxq1g = t4_nrxq1g; iaq->nrxq_vi = t4_nrxq_vi; iaq->rsrv_noflowq = t4_rsrv_noflowq; #ifdef TCP_OFFLOAD if (is_offload(sc)) { iaq->nofldtxq10g = t4_nofldtxq10g; iaq->nofldtxq1g = t4_nofldtxq1g; iaq->nofldtxq_vi = t4_nofldtxq_vi; iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; iaq->nofldrxq_vi = t4_nofldrxq_vi; } #endif #ifdef DEV_NETMAP iaq->nnmtxq_vi = t4_nnmtxq_vi; iaq->nnmrxq_vi = t4_nnmrxq_vi; #endif for (itype = INTR_MSIX; itype; itype >>= 1) { if ((itype & t4_intr_types) == 0) continue; /* not allowed */ if (itype == INTR_MSIX) navail = pci_msix_count(sc->dev); else if (itype == INTR_MSI) navail = pci_msi_count(sc->dev); else navail = 1; restart: if (navail == 0) continue; iaq->intr_type = itype; iaq->intr_flags_10g = 0; iaq->intr_flags_1g = 0; /* * Best option: an interrupt vector for errors, one for the * firmware event queue, and one for every rxq (NIC and TOE) of * every VI. The VIs that support netmap use the same * interrupts for the NIC rx queues and the netmap rx queues * because only one set of queues is active at a time. */ iaq->nirq = T4_EXTRA_INTR; iaq->nirq += n10g * (nrxq10g + nofldrxq10g); iaq->nirq += n1g * (nrxq1g + nofldrxq1g); iaq->nirq += (n10g + n1g) * (num_vis - 1) * max(iaq->nrxq_vi, iaq->nnmrxq_vi); /* See comment above. */ iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi; if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) { iaq->intr_flags_10g = INTR_ALL; iaq->intr_flags_1g = INTR_ALL; goto allocate; } /* Disable the VIs (and netmap) if there aren't enough intrs */ if (num_vis > 1) { device_printf(sc->dev, "virtual interfaces disabled " "because num_vis=%u with current settings " "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, " "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, " "nnmrxq_vi=%u) would need %u interrupts but " "only %u are available.\n", num_vis, nrxq10g, nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq, navail); num_vis = 1; iaq->ntxq_vi = iaq->nrxq_vi = 0; iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; goto restart; } /* * Second best option: a vector for errors, one for the firmware * event queue, and vectors for either all the NIC rx queues or * all the TOE rx queues. The queues that don't get vectors * will forward their interrupts to those that do. */ iaq->nirq = T4_EXTRA_INTR; if (nrxq10g >= nofldrxq10g) { iaq->intr_flags_10g = INTR_RXQ; iaq->nirq += n10g * nrxq10g; } else { iaq->intr_flags_10g = INTR_OFLD_RXQ; iaq->nirq += n10g * nofldrxq10g; } if (nrxq1g >= nofldrxq1g) { iaq->intr_flags_1g = INTR_RXQ; iaq->nirq += n1g * nrxq1g; } else { iaq->intr_flags_1g = INTR_OFLD_RXQ; iaq->nirq += n1g * nofldrxq1g; } if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) goto allocate; /* * Next best option: an interrupt vector for errors, one for the * firmware event queue, and at least one per main-VI. At this * point we know we'll have to downsize nrxq and/or nofldrxq to * fit what's available to us. */ iaq->nirq = T4_EXTRA_INTR; iaq->nirq += n10g + n1g; if (iaq->nirq <= navail) { int leftover = navail - iaq->nirq; if (n10g > 0) { int target = max(nrxq10g, nofldrxq10g); iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? INTR_RXQ : INTR_OFLD_RXQ; n = 1; while (n < target && leftover >= n10g) { leftover -= n10g; iaq->nirq += n10g; n++; } iaq->nrxq10g = min(n, nrxq10g); #ifdef TCP_OFFLOAD iaq->nofldrxq10g = min(n, nofldrxq10g); #endif } if (n1g > 0) { int target = max(nrxq1g, nofldrxq1g); iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? INTR_RXQ : INTR_OFLD_RXQ; n = 1; while (n < target && leftover >= n1g) { leftover -= n1g; iaq->nirq += n1g; n++; } iaq->nrxq1g = min(n, nrxq1g); #ifdef TCP_OFFLOAD iaq->nofldrxq1g = min(n, nofldrxq1g); #endif } if (itype != INTR_MSI || powerof2(iaq->nirq)) goto allocate; } /* * Least desirable option: one interrupt vector for everything. */ iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; iaq->intr_flags_10g = iaq->intr_flags_1g = 0; #ifdef TCP_OFFLOAD if (is_offload(sc)) iaq->nofldrxq10g = iaq->nofldrxq1g = 1; #endif allocate: navail = iaq->nirq; rc = 0; if (itype == INTR_MSIX) rc = pci_alloc_msix(sc->dev, &navail); else if (itype == INTR_MSI) rc = pci_alloc_msi(sc->dev, &navail); if (rc == 0) { if (navail == iaq->nirq) return (0); /* * Didn't get the number requested. Use whatever number * the kernel is willing to allocate (it's in navail). */ device_printf(sc->dev, "fewer vectors than requested, " "type=%d, req=%d, rcvd=%d; will downshift req.\n", itype, iaq->nirq, navail); pci_release_msi(sc->dev); goto restart; } device_printf(sc->dev, "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", itype, rc, iaq->nirq, navail); } device_printf(sc->dev, "failed to find a usable interrupt type. " "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, pci_msix_count(sc->dev), pci_msi_count(sc->dev)); return (ENXIO); } #define FW_VERSION(chip) ( \ V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) struct fw_info { uint8_t chip; char *kld_name; char *fw_mod_name; struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ } fw_info[] = { { .chip = CHELSIO_T4, .kld_name = "t4fw_cfg", .fw_mod_name = "t4fw", .fw_hdr = { .chip = FW_HDR_CHIP_T4, .fw_ver = htobe32_const(FW_VERSION(T4)), .intfver_nic = FW_INTFVER(T4, NIC), .intfver_vnic = FW_INTFVER(T4, VNIC), .intfver_ofld = FW_INTFVER(T4, OFLD), .intfver_ri = FW_INTFVER(T4, RI), .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T4, ISCSI), .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), .intfver_fcoe = FW_INTFVER(T4, FCOE), }, }, { .chip = CHELSIO_T5, .kld_name = "t5fw_cfg", .fw_mod_name = "t5fw", .fw_hdr = { .chip = FW_HDR_CHIP_T5, .fw_ver = htobe32_const(FW_VERSION(T5)), .intfver_nic = FW_INTFVER(T5, NIC), .intfver_vnic = FW_INTFVER(T5, VNIC), .intfver_ofld = FW_INTFVER(T5, OFLD), .intfver_ri = FW_INTFVER(T5, RI), .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T5, ISCSI), .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), .intfver_fcoe = FW_INTFVER(T5, FCOE), }, }, { .chip = CHELSIO_T6, .kld_name = "t6fw_cfg", .fw_mod_name = "t6fw", .fw_hdr = { .chip = FW_HDR_CHIP_T6, .fw_ver = htobe32_const(FW_VERSION(T6)), .intfver_nic = FW_INTFVER(T6, NIC), .intfver_vnic = FW_INTFVER(T6, VNIC), .intfver_ofld = FW_INTFVER(T6, OFLD), .intfver_ri = FW_INTFVER(T6, RI), .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T6, ISCSI), .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), .intfver_fcoe = FW_INTFVER(T6, FCOE), }, } }; static struct fw_info * find_fw_info(int chip) { int i; for (i = 0; i < nitems(fw_info); i++) { if (fw_info[i].chip == chip) return (&fw_info[i]); } return (NULL); } /* * Is the given firmware API compatible with the one the driver was compiled * with? */ static int fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) { /* short circuit if it's the exact same firmware version */ if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) return (1); /* * XXX: Is this too conservative? Perhaps I should limit this to the * features that are supported in the driver. */ #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) return (1); #undef SAME_INTF return (0); } /* * The firmware in the KLD is usable, but should it be installed? This routine * explains itself in detail if it indicates the KLD firmware should be * installed. */ static int should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) { const char *reason; if (!card_fw_usable) { reason = "incompatible or unusable"; goto install; } if (k > c) { reason = "older than the version bundled with this driver"; goto install; } if (t4_fw_install == 2 && k != c) { reason = "different than the version bundled with this driver"; goto install; } return (0); install: if (t4_fw_install == 0) { device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " "but the driver is prohibited from installing a different " "firmware on the card.\n", G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); return (0); } device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " "installing firmware %u.%u.%u.%u on card.\n", G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); return (1); } /* * Establish contact with the firmware and determine if we are the master driver * or not, and whether we are responsible for chip initialization. */ static int prep_firmware(struct adapter *sc) { const struct firmware *fw = NULL, *default_cfg; int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; enum dev_state state; struct fw_info *fw_info; struct fw_hdr *card_fw; /* fw on the card */ const struct fw_hdr *kld_fw; /* fw in the KLD */ const struct fw_hdr *drv_fw; /* fw header the driver was compiled against */ /* This is the firmware whose headers the driver was compiled against */ fw_info = find_fw_info(chip_id(sc)); if (fw_info == NULL) { device_printf(sc->dev, "unable to look up firmware information for chip %d.\n", chip_id(sc)); return (EINVAL); } drv_fw = &fw_info->fw_hdr; /* * The firmware KLD contains many modules. The KLD name is also the * name of the module that contains the default config file. */ default_cfg = firmware_get(fw_info->kld_name); /* This is the firmware in the KLD */ fw = firmware_get(fw_info->fw_mod_name); if (fw != NULL) { kld_fw = (const void *)fw->data; kld_fw_usable = fw_compatible(drv_fw, kld_fw); } else { kld_fw = NULL; kld_fw_usable = 0; } /* Read the header of the firmware on the card */ card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); rc = -t4_read_flash(sc, FLASH_FW_START, sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); if (rc == 0) { card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); if (card_fw->fw_ver == be32toh(0xffffffff)) { uint32_t d = be32toh(kld_fw->fw_ver); if (!kld_fw_usable) { device_printf(sc->dev, "no firmware on the card and no usable " "firmware bundled with the driver.\n"); rc = EIO; goto done; } else if (t4_fw_install == 0) { device_printf(sc->dev, "no firmware on the card and the driver " "is prohibited from installing new " "firmware.\n"); rc = EIO; goto done; } device_printf(sc->dev, "no firmware on the card, " "installing firmware %d.%d.%d.%d\n", G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); rc = t4_fw_forceinstall(sc, fw->data, fw->datasize); if (rc < 0) { rc = -rc; device_printf(sc->dev, "firmware install failed: %d.\n", rc); goto done; } memcpy(card_fw, kld_fw, sizeof(*card_fw)); card_fw_usable = 1; need_fw_reset = 0; } } else { device_printf(sc->dev, "Unable to read card's firmware header: %d\n", rc); card_fw_usable = 0; } /* Contact firmware. */ rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); if (rc < 0 || state == DEV_STATE_ERR) { rc = -rc; device_printf(sc->dev, "failed to connect to the firmware: %d, %d.\n", rc, state); goto done; } pf = rc; if (pf == sc->mbox) sc->flags |= MASTER_PF; else if (state == DEV_STATE_UNINIT) { /* * We didn't get to be the master so we definitely won't be * configuring the chip. It's a bug if someone else hasn't * configured it already. */ device_printf(sc->dev, "couldn't be master(%d), " "device not already initialized either(%d).\n", rc, state); rc = EPROTO; goto done; } if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { /* * Common case: the firmware on the card is an exact match and * the KLD is an exact match too, or the KLD is * absent/incompatible. Note that t4_fw_install = 2 is ignored * here -- use cxgbetool loadfw if you want to reinstall the * same firmware as the one on the card. */ } else if (kld_fw_usable && state == DEV_STATE_UNINIT && should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), be32toh(card_fw->fw_ver))) { rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); if (rc != 0) { device_printf(sc->dev, "failed to install firmware: %d\n", rc); goto done; } /* Installed successfully, update the cached header too. */ memcpy(card_fw, kld_fw, sizeof(*card_fw)); card_fw_usable = 1; need_fw_reset = 0; /* already reset as part of load_fw */ } if (!card_fw_usable) { uint32_t d, c, k; d = ntohl(drv_fw->fw_ver); c = ntohl(card_fw->fw_ver); k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; device_printf(sc->dev, "Cannot find a usable firmware: " "fw_install %d, chip state %d, " "driver compiled with %d.%d.%d.%d, " "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", t4_fw_install, state, G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); rc = EINVAL; goto done; } /* Reset device */ if (need_fw_reset && (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { device_printf(sc->dev, "firmware reset failed: %d.\n", rc); if (rc != ETIMEDOUT && rc != EIO) t4_fw_bye(sc, sc->mbox); goto done; } sc->flags |= FW_OK; rc = get_params__pre_init(sc); if (rc != 0) goto done; /* error message displayed already */ /* Partition adapter resources as specified in the config file. */ if (state == DEV_STATE_UNINIT) { KASSERT(sc->flags & MASTER_PF, ("%s: trying to change chip settings when not master.", __func__)); rc = partition_resources(sc, default_cfg, fw_info->kld_name); if (rc != 0) goto done; /* error message displayed already */ t4_tweak_chip_settings(sc); /* get basic stuff going */ rc = -t4_fw_initialize(sc, sc->mbox); if (rc != 0) { device_printf(sc->dev, "fw init failed: %d.\n", rc); goto done; } } else { snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); sc->cfcsum = 0; } done: free(card_fw, M_CXGBE); if (fw != NULL) firmware_put(fw, FIRMWARE_UNLOAD); if (default_cfg != NULL) firmware_put(default_cfg, FIRMWARE_UNLOAD); return (rc); } #define FW_PARAM_DEV(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) #define FW_PARAM_PFVF(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) /* * Partition chip resources for use between various PFs, VFs, etc. */ static int partition_resources(struct adapter *sc, const struct firmware *default_cfg, const char *name_prefix) { const struct firmware *cfg = NULL; int rc = 0; struct fw_caps_config_cmd caps; uint32_t mtype, moff, finicsum, cfcsum; /* * Figure out what configuration file to use. Pick the default config * file for the card if the user hasn't specified one explicitly. */ snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { /* Card specific overrides go here. */ if (pci_get_device(sc->dev) == 0x440a) snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); if (is_fpga(sc)) snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); } /* * We need to load another module if the profile is anything except * "default" or "flash". */ if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { char s[32]; snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); cfg = firmware_get(s); if (cfg == NULL) { if (default_cfg != NULL) { device_printf(sc->dev, "unable to load module \"%s\" for " "configuration profile \"%s\", will use " "the default config file instead.\n", s, sc->cfg_file); snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", DEFAULT_CF); } else { device_printf(sc->dev, "unable to load module \"%s\" for " "configuration profile \"%s\", will use " "the config file on the card's flash " "instead.\n", s, sc->cfg_file); snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); } } } if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && default_cfg == NULL) { device_printf(sc->dev, "default config file not available, will use the config " "file on the card's flash instead.\n"); snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); } if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { u_int cflen; const uint32_t *cfdata; uint32_t param, val, addr; KASSERT(cfg != NULL || default_cfg != NULL, ("%s: no config to upload", __func__)); /* * Ask the firmware where it wants us to upload the config file. */ param = FW_PARAM_DEV(CF); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc != 0) { /* No support for config file? Shouldn't happen. */ device_printf(sc->dev, "failed to query config file location: %d.\n", rc); goto done; } mtype = G_FW_PARAMS_PARAM_Y(val); moff = G_FW_PARAMS_PARAM_Z(val) << 16; /* * XXX: sheer laziness. We deliberately added 4 bytes of * useless stuffing/comments at the end of the config file so * it's ok to simply throw away the last remaining bytes when * the config file is not an exact multiple of 4. This also * helps with the validate_mt_off_len check. */ if (cfg != NULL) { cflen = cfg->datasize & ~3; cfdata = cfg->data; } else { cflen = default_cfg->datasize & ~3; cfdata = default_cfg->data; } if (cflen > FLASH_CFG_MAX_SIZE) { device_printf(sc->dev, "config file too long (%d, max allowed is %d). " "Will try to use the config on the card, if any.\n", cflen, FLASH_CFG_MAX_SIZE); goto use_config_on_flash; } rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); if (rc != 0) { device_printf(sc->dev, "%s: addr (%d/0x%x) or len %d is not valid: %d. " "Will try to use the config on the card, if any.\n", __func__, mtype, moff, cflen, rc); goto use_config_on_flash; } write_via_memwin(sc, 2, addr, cfdata, cflen); } else { use_config_on_flash: mtype = FW_MEMTYPE_FLASH; moff = t4_flash_cfg_addr(sc); } bzero(&caps, sizeof(caps)); caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); if (rc != 0) { device_printf(sc->dev, "failed to pre-process config file: %d " "(mtype %d, moff 0x%x).\n", rc, mtype, moff); goto done; } finicsum = be32toh(caps.finicsum); cfcsum = be32toh(caps.cfcsum); if (finicsum != cfcsum) { device_printf(sc->dev, "WARNING: config file checksum mismatch: %08x %08x\n", finicsum, cfcsum); } sc->cfcsum = cfcsum; #define LIMIT_CAPS(x) do { \ caps.x &= htobe16(t4_##x##_allowed); \ } while (0) /* * Let the firmware know what features will (not) be used so it can tune * things accordingly. */ LIMIT_CAPS(nbmcaps); LIMIT_CAPS(linkcaps); LIMIT_CAPS(switchcaps); LIMIT_CAPS(niccaps); LIMIT_CAPS(toecaps); LIMIT_CAPS(rdmacaps); LIMIT_CAPS(cryptocaps); LIMIT_CAPS(iscsicaps); LIMIT_CAPS(fcoecaps); #undef LIMIT_CAPS caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); if (rc != 0) { device_printf(sc->dev, "failed to process config file: %d.\n", rc); } done: if (cfg != NULL) firmware_put(cfg, FIRMWARE_UNLOAD); return (rc); } /* * Retrieve parameters that are needed (or nice to have) very early. */ static int get_params__pre_init(struct adapter *sc) { int rc; uint32_t param[2], val[2]; t4_get_version_info(sc); snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers), G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers), G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers), G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers)); snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers), G_FW_HDR_FW_VER_MINOR(sc->params.er_vers), G_FW_HDR_FW_VER_MICRO(sc->params.er_vers), G_FW_HDR_FW_VER_BUILD(sc->params.er_vers)); param[0] = FW_PARAM_DEV(PORTVEC); param[1] = FW_PARAM_DEV(CCLK); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (pre_init): %d.\n", rc); return (rc); } sc->params.portvec = val[0]; sc->params.nports = bitcount32(val[0]); sc->params.vpd.cclk = val[1]; /* Read device log parameters. */ rc = -t4_init_devlog_params(sc, 1); if (rc == 0) fixup_devlog_params(sc); else { device_printf(sc->dev, "failed to get devlog parameters: %d.\n", rc); rc = 0; /* devlog isn't critical for device operation */ } return (rc); } /* * Retrieve various parameters that are of interest to the driver. The device * has been initialized by the firmware at this point. */ static int get_params__post_init(struct adapter *sc) { int rc; uint32_t param[7], val[7]; struct fw_caps_config_cmd caps; param[0] = FW_PARAM_PFVF(IQFLINT_START); param[1] = FW_PARAM_PFVF(EQ_START); param[2] = FW_PARAM_PFVF(FILTER_START); param[3] = FW_PARAM_PFVF(FILTER_END); param[4] = FW_PARAM_PFVF(L2T_START); param[5] = FW_PARAM_PFVF(L2T_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (post_init): %d.\n", rc); return (rc); } sc->sge.iq_start = val[0]; sc->sge.eq_start = val[1]; sc->tids.ftid_base = val[2]; sc->tids.nftids = val[3] - val[2] + 1; sc->params.ftid_min = val[2]; sc->params.ftid_max = val[3]; sc->vres.l2t.start = val[4]; sc->vres.l2t.size = val[5] - val[4] + 1; KASSERT(sc->vres.l2t.size <= L2T_SIZE, ("%s: L2 table size (%u) larger than expected (%u)", __func__, sc->vres.l2t.size, L2T_SIZE)); /* get capabilites */ bzero(&caps, sizeof(caps)); caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); if (rc != 0) { device_printf(sc->dev, "failed to get card capabilities: %d.\n", rc); return (rc); } #define READ_CAPS(x) do { \ sc->x = htobe16(caps.x); \ } while (0) READ_CAPS(nbmcaps); READ_CAPS(linkcaps); READ_CAPS(switchcaps); READ_CAPS(niccaps); READ_CAPS(toecaps); READ_CAPS(rdmacaps); READ_CAPS(cryptocaps); READ_CAPS(iscsicaps); READ_CAPS(fcoecaps); if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { param[0] = FW_PARAM_PFVF(ETHOFLD_START); param[1] = FW_PARAM_PFVF(ETHOFLD_END); param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query NIC parameters: %d.\n", rc); return (rc); } sc->tids.etid_base = val[0]; sc->params.etid_min = val[0]; sc->tids.netids = val[1] - val[0] + 1; sc->params.netids = sc->tids.netids; sc->params.eo_wr_cred = val[2]; sc->params.ethoffload = 1; } if (sc->toecaps) { /* query offload-related parameters */ param[0] = FW_PARAM_DEV(NTID); param[1] = FW_PARAM_PFVF(SERVER_START); param[2] = FW_PARAM_PFVF(SERVER_END); param[3] = FW_PARAM_PFVF(TDDP_START); param[4] = FW_PARAM_PFVF(TDDP_END); param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query TOE parameters: %d.\n", rc); return (rc); } sc->tids.ntids = val[0]; sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); sc->tids.stid_base = val[1]; sc->tids.nstids = val[2] - val[1] + 1; sc->vres.ddp.start = val[3]; sc->vres.ddp.size = val[4] - val[3] + 1; sc->params.ofldq_wr_cred = val[5]; sc->params.offload = 1; } if (sc->rdmacaps) { param[0] = FW_PARAM_PFVF(STAG_START); param[1] = FW_PARAM_PFVF(STAG_END); param[2] = FW_PARAM_PFVF(RQ_START); param[3] = FW_PARAM_PFVF(RQ_END); param[4] = FW_PARAM_PFVF(PBL_START); param[5] = FW_PARAM_PFVF(PBL_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(1): %d.\n", rc); return (rc); } sc->vres.stag.start = val[0]; sc->vres.stag.size = val[1] - val[0] + 1; sc->vres.rq.start = val[2]; sc->vres.rq.size = val[3] - val[2] + 1; sc->vres.pbl.start = val[4]; sc->vres.pbl.size = val[5] - val[4] + 1; param[0] = FW_PARAM_PFVF(SQRQ_START); param[1] = FW_PARAM_PFVF(SQRQ_END); param[2] = FW_PARAM_PFVF(CQ_START); param[3] = FW_PARAM_PFVF(CQ_END); param[4] = FW_PARAM_PFVF(OCQ_START); param[5] = FW_PARAM_PFVF(OCQ_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(2): %d.\n", rc); return (rc); } sc->vres.qp.start = val[0]; sc->vres.qp.size = val[1] - val[0] + 1; sc->vres.cq.start = val[2]; sc->vres.cq.size = val[3] - val[2] + 1; sc->vres.ocq.start = val[4]; sc->vres.ocq.size = val[5] - val[4] + 1; param[0] = FW_PARAM_PFVF(SRQ_START); param[1] = FW_PARAM_PFVF(SRQ_END); param[2] = FW_PARAM_DEV(MAXORDIRD_QP); param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(3): %d.\n", rc); return (rc); } sc->vres.srq.start = val[0]; sc->vres.srq.size = val[1] - val[0] + 1; sc->params.max_ordird_qp = val[2]; sc->params.max_ird_adapter = val[3]; } if (sc->iscsicaps) { param[0] = FW_PARAM_PFVF(ISCSI_START); param[1] = FW_PARAM_PFVF(ISCSI_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query iSCSI parameters: %d.\n", rc); return (rc); } sc->vres.iscsi.start = val[0]; sc->vres.iscsi.size = val[1] - val[0] + 1; } t4_init_sge_params(sc); /* * We've got the params we wanted to query via the firmware. Now grab * some others directly from the chip. */ rc = t4_read_chip_settings(sc); return (rc); } static int set_params__post_init(struct adapter *sc) { uint32_t param, val; int i, v, shift; char s[32]; /* ask for encapsulated CPLs */ param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); val = 1; (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); /* * Override the TOE timers with user provided tunables. This is not the * recommended way to change the timers (the firmware config file is) so * these tunables are not documented. * * All the timer tunables are in milliseconds. */ if (TUNABLE_INT_FETCH("hw.cxgbe.toe.keepalive_idle", &v)) { t4_set_reg_field(sc, A_TP_KEEP_IDLE, V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(ms_to_tcp_ticks(sc, v))); } if (TUNABLE_INT_FETCH("hw.cxgbe.toe.keepalive_interval", &v)) { t4_set_reg_field(sc, A_TP_KEEP_INTVL, V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(ms_to_tcp_ticks(sc, v))); } if (TUNABLE_INT_FETCH("hw.cxgbe.toe.keepalive_count", &v)) { v &= M_KEEPALIVEMAXR1; t4_set_reg_field(sc, A_TP_SHIFT_CNT, V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) | V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2), V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v)); } if (TUNABLE_INT_FETCH("hw.cxgbe.toe.rexmt_min", &v)) { t4_set_reg_field(sc, A_TP_RXT_MIN, V_RXTMIN(M_RXTMIN), V_RXTMIN(ms_to_tcp_ticks(sc, v))); } if (TUNABLE_INT_FETCH("hw.cxgbe.toe.rexmt_max", &v)) { t4_set_reg_field(sc, A_TP_RXT_MAX, V_RXTMAX(M_RXTMAX), V_RXTMAX(ms_to_tcp_ticks(sc, v))); } if (TUNABLE_INT_FETCH("hw.cxgbe.toe.rexmt_count", &v)) { v &= M_RXTSHIFTMAXR1; t4_set_reg_field(sc, A_TP_SHIFT_CNT, V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) | V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2), V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v)); } for (i = 0; i < 16; i++) { snprintf(s, sizeof(s), "hw.cxgbe.toe.rexmt_backoff.%d", i); if (TUNABLE_INT_FETCH(s, &v)) { v &= M_TIMERBACKOFFINDEX0; shift = (i & 3) << 3; t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), M_TIMERBACKOFFINDEX0 << shift, v << shift); } } return (0); } #undef FW_PARAM_PFVF #undef FW_PARAM_DEV static void t4_set_desc(struct adapter *sc) { char buf[128]; struct adapter_params *p = &sc->params; snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id); device_set_desc_copy(sc->dev, buf); } static void build_medialist(struct port_info *pi, struct ifmedia *media) { int m; PORT_LOCK_ASSERT_OWNED(pi); ifmedia_removeall(media); /* * XXX: Would it be better to ifmedia_add all 4 combinations of pause * settings for every speed instead of just txpause|rxpause? ifconfig * media display looks much better if autoselect is the only case where * ifm_current is different from ifm_active. If the user picks anything * except txpause|rxpause the display is ugly. */ m = IFM_ETHER | IFM_FDX | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; switch(pi->port_type) { case FW_PORT_TYPE_BT_XFI: case FW_PORT_TYPE_BT_XAUI: ifmedia_add(media, m | IFM_10G_T, 0, NULL); /* fall through */ case FW_PORT_TYPE_BT_SGMII: ifmedia_add(media, m | IFM_1000_T, 0, NULL); ifmedia_add(media, m | IFM_100_TX, 0, NULL); ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(media, IFM_ETHER | IFM_AUTO); break; case FW_PORT_TYPE_CX4: ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); ifmedia_set(media, m | IFM_10G_CX4); break; case FW_PORT_TYPE_QSFP_10G: case FW_PORT_TYPE_SFP: case FW_PORT_TYPE_FIBER_XFI: case FW_PORT_TYPE_FIBER_XAUI: switch (pi->mod_type) { case FW_PORT_MOD_TYPE_LR: ifmedia_add(media, m | IFM_10G_LR, 0, NULL); ifmedia_set(media, m | IFM_10G_LR); break; case FW_PORT_MOD_TYPE_SR: ifmedia_add(media, m | IFM_10G_SR, 0, NULL); ifmedia_set(media, m | IFM_10G_SR); break; case FW_PORT_MOD_TYPE_LRM: ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); ifmedia_set(media, m | IFM_10G_LRM); break; case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); ifmedia_set(media, m | IFM_10G_TWINAX); break; case FW_PORT_MOD_TYPE_NONE: m &= ~IFM_FDX; ifmedia_add(media, m | IFM_NONE, 0, NULL); ifmedia_set(media, m | IFM_NONE); break; case FW_PORT_MOD_TYPE_NA: case FW_PORT_MOD_TYPE_ER: default: device_printf(pi->dev, "unknown port_type (%d), mod_type (%d)\n", pi->port_type, pi->mod_type); ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); ifmedia_set(media, m | IFM_UNKNOWN); break; } break; case FW_PORT_TYPE_CR_QSFP: case FW_PORT_TYPE_SFP28: case FW_PORT_TYPE_KR_SFP28: switch (pi->mod_type) { case FW_PORT_MOD_TYPE_SR: ifmedia_add(media, m | IFM_25G_SR, 0, NULL); ifmedia_set(media, m | IFM_25G_SR); break; case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: ifmedia_add(media, m | IFM_25G_CR, 0, NULL); ifmedia_set(media, m | IFM_25G_CR); break; case FW_PORT_MOD_TYPE_NONE: m &= ~IFM_FDX; ifmedia_add(media, m | IFM_NONE, 0, NULL); ifmedia_set(media, m | IFM_NONE); break; default: device_printf(pi->dev, "unknown port_type (%d), mod_type (%d)\n", pi->port_type, pi->mod_type); ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); ifmedia_set(media, m | IFM_UNKNOWN); break; } break; case FW_PORT_TYPE_QSFP: switch (pi->mod_type) { case FW_PORT_MOD_TYPE_LR: ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); ifmedia_set(media, m | IFM_40G_LR4); break; case FW_PORT_MOD_TYPE_SR: ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); ifmedia_set(media, m | IFM_40G_SR4); break; case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); ifmedia_set(media, m | IFM_40G_CR4); break; case FW_PORT_MOD_TYPE_NONE: m &= ~IFM_FDX; ifmedia_add(media, m | IFM_NONE, 0, NULL); ifmedia_set(media, m | IFM_NONE); break; default: device_printf(pi->dev, "unknown port_type (%d), mod_type (%d)\n", pi->port_type, pi->mod_type); ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); ifmedia_set(media, m | IFM_UNKNOWN); break; } break; case FW_PORT_TYPE_KR4_100G: case FW_PORT_TYPE_CR4_QSFP: switch (pi->mod_type) { case FW_PORT_MOD_TYPE_LR: ifmedia_add(media, m | IFM_100G_LR4, 0, NULL); ifmedia_set(media, m | IFM_100G_LR4); break; case FW_PORT_MOD_TYPE_SR: ifmedia_add(media, m | IFM_100G_SR4, 0, NULL); ifmedia_set(media, m | IFM_100G_SR4); break; case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: ifmedia_add(media, m | IFM_100G_CR4, 0, NULL); ifmedia_set(media, m | IFM_100G_CR4); break; case FW_PORT_MOD_TYPE_NONE: m &= ~IFM_FDX; ifmedia_add(media, m | IFM_NONE, 0, NULL); ifmedia_set(media, m | IFM_NONE); break; default: device_printf(pi->dev, "unknown port_type (%d), mod_type (%d)\n", pi->port_type, pi->mod_type); ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); ifmedia_set(media, m | IFM_UNKNOWN); break; } break; default: device_printf(pi->dev, "unknown port_type (%d), mod_type (%d)\n", pi->port_type, pi->mod_type); ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); ifmedia_set(media, m | IFM_UNKNOWN); break; } } /* * Update all the requested_* fields in the link config and then send a mailbox * command to apply the settings. */ static void init_l1cfg(struct port_info *pi) { struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; ASSERT_SYNCHRONIZED_OP(sc); if (t4_autoneg != 0 && lc->supported & FW_PORT_CAP_ANEG) { lc->requested_aneg = AUTONEG_ENABLE; lc->requested_speed = 0; } else { lc->requested_aneg = AUTONEG_DISABLE; lc->requested_speed = port_top_speed(pi); /* in Gbps */ } lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX); if (t4_fec != -1) { lc->requested_fec = t4_fec & (FEC_RS | FEC_BASER_RS | FEC_RESERVED); } else { /* Use the suggested value provided by the firmware in acaps */ if (lc->advertising & FW_PORT_CAP_FEC_RS) lc->requested_fec = FEC_RS; else if (lc->advertising & FW_PORT_CAP_FEC_BASER_RS) lc->requested_fec = FEC_BASER_RS; else if (lc->advertising & FW_PORT_CAP_FEC_RESERVED) lc->requested_fec = FEC_RESERVED; else lc->requested_fec = 0; } rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); if (rc != 0) { device_printf(pi->dev, "l1cfg failed: %d\n", rc); } else { lc->fc = lc->requested_fc; lc->fec = lc->requested_fec; } } #define FW_MAC_EXACT_CHUNK 7 /* * Program the port's XGMAC based on parameters in ifnet. The caller also * indicates which parameters should be programmed (the rest are left alone). */ int update_mac_settings(struct ifnet *ifp, int flags) { int rc = 0; struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; ASSERT_SYNCHRONIZED_OP(sc); KASSERT(flags, ("%s: not told what to update.", __func__)); if (flags & XGMAC_MTU) mtu = ifp->if_mtu; if (flags & XGMAC_PROMISC) promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; if (flags & XGMAC_ALLMULTI) allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; if (flags & XGMAC_VLANEX) vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, allmulti, 1, vlanex, false); if (rc) { if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); return (rc); } } if (flags & XGMAC_UCADDR) { uint8_t ucaddr[ETHER_ADDR_LEN]; bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, ucaddr, true, true); if (rc < 0) { rc = -rc; if_printf(ifp, "change_mac failed: %d\n", rc); return (rc); } else { vi->xact_addr_filt = rc; rc = 0; } } if (flags & XGMAC_MCADDRS) { const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; int del = 1; uint64_t hash = 0; struct ifmultiaddr *ifma; int i = 0, j; if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; mcaddr[i] = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); MPASS(ETHER_IS_MULTICAST(mcaddr[i])); i++; if (i == FW_MAC_EXACT_CHUNK) { rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, mcaddr, NULL, &hash, 0); if (rc < 0) { rc = -rc; for (j = 0; j < i; j++) { if_printf(ifp, "failed to add mc address" " %02x:%02x:%02x:" "%02x:%02x:%02x rc=%d\n", mcaddr[j][0], mcaddr[j][1], mcaddr[j][2], mcaddr[j][3], mcaddr[j][4], mcaddr[j][5], rc); } goto mcfail; } del = 0; i = 0; } } if (i > 0) { rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, mcaddr, NULL, &hash, 0); if (rc < 0) { rc = -rc; for (j = 0; j < i; j++) { if_printf(ifp, "failed to add mc address" " %02x:%02x:%02x:" "%02x:%02x:%02x rc=%d\n", mcaddr[j][0], mcaddr[j][1], mcaddr[j][2], mcaddr[j][3], mcaddr[j][4], mcaddr[j][5], rc); } goto mcfail; } } rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); if (rc != 0) if_printf(ifp, "failed to set mc address hash: %d", rc); mcfail: if_maddr_runlock(ifp); } return (rc); } /* * {begin|end}_synchronized_op must be called from the same thread. */ int begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, char *wmesg) { int rc, pri; #ifdef WITNESS /* the caller thinks it's ok to sleep, but is it really? */ if (flags & SLEEP_OK) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "begin_synchronized_op"); #endif if (INTR_OK) pri = PCATCH; else pri = 0; ADAPTER_LOCK(sc); for (;;) { if (vi && IS_DOOMED(vi)) { rc = ENXIO; goto done; } if (!IS_BUSY(sc)) { rc = 0; break; } if (!(flags & SLEEP_OK)) { rc = EBUSY; goto done; } if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { rc = EINTR; goto done; } } KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); SET_BUSY(sc); #ifdef INVARIANTS sc->last_op = wmesg; sc->last_op_thr = curthread; sc->last_op_flags = flags; #endif done: if (!(flags & HOLD_LOCK) || rc) ADAPTER_UNLOCK(sc); return (rc); } /* * Tell if_ioctl and if_init that the VI is going away. This is * special variant of begin_synchronized_op and must be paired with a * call to end_synchronized_op. */ void doom_vi(struct adapter *sc, struct vi_info *vi) { ADAPTER_LOCK(sc); SET_DOOMED(vi); wakeup(&sc->flags); while (IS_BUSY(sc)) mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); SET_BUSY(sc); #ifdef INVARIANTS sc->last_op = "t4detach"; sc->last_op_thr = curthread; sc->last_op_flags = 0; #endif ADAPTER_UNLOCK(sc); } /* * {begin|end}_synchronized_op must be called from the same thread. */ void end_synchronized_op(struct adapter *sc, int flags) { if (flags & LOCK_HELD) ADAPTER_LOCK_ASSERT_OWNED(sc); else ADAPTER_LOCK(sc); KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); CLR_BUSY(sc); wakeup(&sc->flags); ADAPTER_UNLOCK(sc); } static int cxgbe_init_synchronized(struct vi_info *vi) { struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct ifnet *ifp = vi->ifp; int rc = 0, i; struct sge_txq *txq; ASSERT_SYNCHRONIZED_OP(sc); if (ifp->if_drv_flags & IFF_DRV_RUNNING) return (0); /* already running */ if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_full_init(sc)) != 0)) return (rc); /* error message displayed already */ if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_full_init(vi)) != 0)) return (rc); /* error message displayed already */ rc = update_mac_settings(ifp, XGMAC_ALL); if (rc) goto done; /* error message displayed already */ rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); if (rc != 0) { if_printf(ifp, "enable_vi failed: %d\n", rc); goto done; } /* * Can't fail from this point onwards. Review cxgbe_uninit_synchronized * if this changes. */ for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags |= EQ_ENABLED; TXQ_UNLOCK(txq); } /* * The first iq of the first port to come up is used for tracing. */ if (sc->traceq < 0 && IS_MAIN_VI(vi)) { sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | V_QUEUENUMBER(sc->traceq)); pi->flags |= HAS_TRACEQ; } /* all ok */ PORT_LOCK(pi); if (pi->up_vis++ == 0) { t4_update_port_info(pi); build_medialist(vi->pi, &vi->media); init_l1cfg(pi); } ifp->if_drv_flags |= IFF_DRV_RUNNING; if (pi->nvi > 1 || sc->flags & IS_VF) callout_reset(&vi->tick, hz, vi_tick, vi); else callout_reset(&pi->tick, hz, cxgbe_tick, pi); PORT_UNLOCK(pi); done: if (rc != 0) cxgbe_uninit_synchronized(vi); return (rc); } /* * Idempotent. */ static int cxgbe_uninit_synchronized(struct vi_info *vi) { struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct ifnet *ifp = vi->ifp; int rc, i; struct sge_txq *txq; ASSERT_SYNCHRONIZED_OP(sc); if (!(vi->flags & VI_INIT_DONE)) { KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), ("uninited VI is running")); return (0); } /* * Disable the VI so that all its data in either direction is discarded * by the MPS. Leave everything else (the queues, interrupts, and 1Hz * tick) intact as the TP can deliver negative advice or data that it's * holding in its RAM (for an offloaded connection) even after the VI is * disabled. */ rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); if (rc) { if_printf(ifp, "disable_vi failed: %d\n", rc); return (rc); } for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags &= ~EQ_ENABLED; TXQ_UNLOCK(txq); } PORT_LOCK(pi); if (pi->nvi > 1 || sc->flags & IS_VF) callout_stop(&vi->tick); else callout_stop(&pi->tick); if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { PORT_UNLOCK(pi); return (0); } ifp->if_drv_flags &= ~IFF_DRV_RUNNING; pi->up_vis--; if (pi->up_vis > 0) { PORT_UNLOCK(pi); return (0); } PORT_UNLOCK(pi); pi->link_cfg.link_ok = 0; pi->link_cfg.speed = 0; pi->link_cfg.link_down_rc = 255; t4_os_link_changed(pi, NULL); return (0); } /* * It is ok for this function to fail midway and return right away. t4_detach * will walk the entire sc->irq list and clean up whatever is valid. */ int t4_setup_intr_handlers(struct adapter *sc) { int rc, rid, p, q, v; char s[8]; struct irq *irq; struct port_info *pi; struct vi_info *vi; struct sge *sge = &sc->sge; struct sge_rxq *rxq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif #ifdef DEV_NETMAP struct sge_nm_rxq *nm_rxq; #endif #ifdef RSS int nbuckets = rss_getnumbuckets(); #endif /* * Setup interrupts. */ irq = &sc->irq[0]; rid = sc->intr_type == INTR_INTX ? 0 : 1; if (sc->intr_count == 1) return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); /* Multiple interrupts. */ if (sc->flags & IS_VF) KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports, ("%s: too few intr.", __func__)); else KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, ("%s: too few intr.", __func__)); /* The first one is always error intr on PFs */ if (!(sc->flags & IS_VF)) { rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); if (rc != 0) return (rc); irq++; rid++; } /* The second one is always the firmware event queue (first on VFs) */ rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); if (rc != 0) return (rc); irq++; rid++; for_each_port(sc, p) { pi = sc->port[p]; for_each_vi(pi, v, vi) { vi->first_intr = rid - 1; if (vi->nnmrxq > 0) { int n = max(vi->nrxq, vi->nnmrxq); MPASS(vi->flags & INTR_RXQ); rxq = &sge->rxq[vi->first_rxq]; #ifdef DEV_NETMAP nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; #endif for (q = 0; q < n; q++) { snprintf(s, sizeof(s), "%x%c%x", p, 'a' + v, q); if (q < vi->nrxq) irq->rxq = rxq++; #ifdef DEV_NETMAP if (q < vi->nnmrxq) irq->nm_rxq = nm_rxq++; #endif rc = t4_alloc_irq(sc, irq, rid, t4_vi_intr, irq, s); if (rc != 0) return (rc); irq++; rid++; vi->nintr++; } } else if (vi->flags & INTR_RXQ) { for_each_rxq(vi, q, rxq) { snprintf(s, sizeof(s), "%x%c%x", p, 'a' + v, q); rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, s); if (rc != 0) return (rc); #ifdef RSS bus_bind_intr(sc->dev, irq->res, rss_getcpu(q % nbuckets)); #endif irq++; rid++; vi->nintr++; } } #ifdef TCP_OFFLOAD if (vi->flags & INTR_OFLD_RXQ) { for_each_ofld_rxq(vi, q, ofld_rxq) { snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q); rc = t4_alloc_irq(sc, irq, rid, t4_intr, ofld_rxq, s); if (rc != 0) return (rc); irq++; rid++; vi->nintr++; } } #endif } } MPASS(irq == &sc->irq[sc->intr_count]); return (0); } int adapter_full_init(struct adapter *sc) { int rc, i; #ifdef RSS uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; #endif ASSERT_SYNCHRONIZED_OP(sc); ADAPTER_LOCK_ASSERT_NOTOWNED(sc); KASSERT((sc->flags & FULL_INIT_DONE) == 0, ("%s: FULL_INIT_DONE already", __func__)); /* * queues that belong to the adapter (not any particular port). */ rc = t4_setup_adapter_queues(sc); if (rc != 0) goto done; for (i = 0; i < nitems(sc->tq); i++) { sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, taskqueue_thread_enqueue, &sc->tq[i]); if (sc->tq[i] == NULL) { device_printf(sc->dev, "failed to allocate task queue %d\n", i); rc = ENOMEM; goto done; } taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", device_get_nameunit(sc->dev), i); } #ifdef RSS MPASS(RSS_KEYSIZE == 40); rss_getkey((void *)&raw_rss_key[0]); for (i = 0; i < nitems(rss_key); i++) { rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); } - t4_write_rss_key(sc, &rss_key[0], -1); + t4_write_rss_key(sc, &rss_key[0], -1, 1); #endif if (!(sc->flags & IS_VF)) t4_intr_enable(sc); sc->flags |= FULL_INIT_DONE; done: if (rc != 0) adapter_full_uninit(sc); return (rc); } int adapter_full_uninit(struct adapter *sc) { int i; ADAPTER_LOCK_ASSERT_NOTOWNED(sc); t4_teardown_adapter_queues(sc); for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { taskqueue_free(sc->tq[i]); sc->tq[i] = NULL; } sc->flags &= ~FULL_INIT_DONE; return (0); } #ifdef RSS #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ RSS_HASHTYPE_RSS_UDP_IPV6) /* Translates kernel hash types to hardware. */ static int hashconfig_to_hashen(int hashconfig) { int hashen = 0; if (hashconfig & RSS_HASHTYPE_RSS_IPV4) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_IPV6) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; } if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; } if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; return (hashen); } /* Translates hardware hash types to kernel. */ static int hashen_to_hashconfig(int hashen) { int hashconfig = 0; if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { /* * If UDP hashing was enabled it must have been enabled for * either IPv4 or IPv6 (inclusive or). Enabling UDP without * enabling any 4-tuple hash is nonsense configuration. */ MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; } if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) hashconfig |= RSS_HASHTYPE_RSS_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) hashconfig |= RSS_HASHTYPE_RSS_IPV6; return (hashconfig); } #endif int vi_full_init(struct vi_info *vi) { struct adapter *sc = vi->pi->adapter; struct ifnet *ifp = vi->ifp; uint16_t *rss; struct sge_rxq *rxq; int rc, i, j, hashen; #ifdef RSS int nbuckets = rss_getnumbuckets(); int hashconfig = rss_gethashconfig(); int extra; #endif ASSERT_SYNCHRONIZED_OP(sc); KASSERT((vi->flags & VI_INIT_DONE) == 0, ("%s: VI_INIT_DONE already", __func__)); sysctl_ctx_init(&vi->ctx); vi->flags |= VI_SYSCTL_CTX; /* * Allocate tx/rx/fl queues for this VI. */ rc = t4_setup_vi_queues(vi); if (rc != 0) goto done; /* error message displayed already */ /* * Setup RSS for this VI. Save a copy of the RSS table for later use. */ if (vi->nrxq > vi->rss_size) { if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " "some queues will never receive traffic.\n", vi->nrxq, vi->rss_size); } else if (vi->rss_size % vi->nrxq) { if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " "expect uneven traffic distribution.\n", vi->nrxq, vi->rss_size); } #ifdef RSS if (vi->nrxq != nbuckets) { if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" "performance will be impacted.\n", vi->nrxq, nbuckets); } #endif rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); for (i = 0; i < vi->rss_size;) { #ifdef RSS j = rss_get_indirection_to_bucket(i); j %= vi->nrxq; rxq = &sc->sge.rxq[vi->first_rxq + j]; rss[i++] = rxq->iq.abs_id; #else for_each_rxq(vi, j, rxq) { rss[i++] = rxq->iq.abs_id; if (i == vi->rss_size) break; } #endif } rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, vi->rss_size); if (rc != 0) { if_printf(ifp, "rss_config failed: %d\n", rc); goto done; } #ifdef RSS hashen = hashconfig_to_hashen(hashconfig); /* * We may have had to enable some hashes even though the global config * wants them disabled. This is a potential problem that must be * reported to the user. */ extra = hashen_to_hashconfig(hashen) ^ hashconfig; /* * If we consider only the supported hash types, then the enabled hashes * are a superset of the requested hashes. In other words, there cannot * be any supported hash that was requested but not enabled, but there * can be hashes that were not requested but had to be enabled. */ extra &= SUPPORTED_RSS_HASHTYPES; MPASS((extra & hashconfig) == 0); if (extra) { if_printf(ifp, "global RSS config (0x%x) cannot be accommodated.\n", hashconfig); } if (extra & RSS_HASHTYPE_RSS_IPV4) if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_IPV6) if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); #else hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; #endif rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0], 0, 0); if (rc != 0) { if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); goto done; } vi->rss = rss; vi->flags |= VI_INIT_DONE; done: if (rc != 0) vi_full_uninit(vi); return (rc); } /* * Idempotent. */ int vi_full_uninit(struct vi_info *vi) { struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; int i; struct sge_rxq *rxq; struct sge_txq *txq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ofld_txq; #endif if (vi->flags & VI_INIT_DONE) { /* Need to quiesce queues. */ /* XXX: Only for the first VI? */ if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF)) quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); for_each_txq(vi, i, txq) { quiesce_txq(sc, txq); } #ifdef TCP_OFFLOAD for_each_ofld_txq(vi, i, ofld_txq) { quiesce_wrq(sc, ofld_txq); } #endif for_each_rxq(vi, i, rxq) { quiesce_iq(sc, &rxq->iq); quiesce_fl(sc, &rxq->fl); } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { quiesce_iq(sc, &ofld_rxq->iq); quiesce_fl(sc, &ofld_rxq->fl); } #endif free(vi->rss, M_CXGBE); free(vi->nm_rss, M_CXGBE); } t4_teardown_vi_queues(vi); vi->flags &= ~VI_INIT_DONE; return (0); } static void quiesce_txq(struct adapter *sc, struct sge_txq *txq) { struct sge_eq *eq = &txq->eq; struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; (void) sc; /* unused */ #ifdef INVARIANTS TXQ_LOCK(txq); MPASS((eq->flags & EQ_ENABLED) == 0); TXQ_UNLOCK(txq); #endif /* Wait for the mp_ring to empty. */ while (!mp_ring_is_idle(txq->r)) { mp_ring_check_drainage(txq->r, 0); pause("rquiesce", 1); } /* Then wait for the hardware to finish. */ while (spg->cidx != htobe16(eq->pidx)) pause("equiesce", 1); /* Finally, wait for the driver to reclaim all descriptors. */ while (eq->cidx != eq->pidx) pause("dquiesce", 1); } static void quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) { /* XXXTX */ } static void quiesce_iq(struct adapter *sc, struct sge_iq *iq) { (void) sc; /* unused */ /* Synchronize with the interrupt handler */ while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) pause("iqfree", 1); } static void quiesce_fl(struct adapter *sc, struct sge_fl *fl) { mtx_lock(&sc->sfl_lock); FL_LOCK(fl); fl->flags |= FL_DOOMED; FL_UNLOCK(fl); callout_stop(&sc->sfl_callout); mtx_unlock(&sc->sfl_lock); KASSERT((fl->flags & FL_STARVING) == 0, ("%s: still starving", __func__)); } static int t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, driver_intr_t *handler, void *arg, char *name) { int rc; irq->rid = rid; irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, RF_SHAREABLE | RF_ACTIVE); if (irq->res == NULL) { device_printf(sc->dev, "failed to allocate IRQ for rid %d, name %s.\n", rid, name); return (ENOMEM); } rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, NULL, handler, arg, &irq->tag); if (rc != 0) { device_printf(sc->dev, "failed to setup interrupt for rid %d, name %s: %d\n", rid, name, rc); } else if (name) bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name); return (rc); } static int t4_free_irq(struct adapter *sc, struct irq *irq) { if (irq->tag) bus_teardown_intr(sc->dev, irq->res, irq->tag); if (irq->res) bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); bzero(irq, sizeof(*irq)); return (0); } static void get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) { regs->version = chip_id(sc) | chip_rev(sc) << 10; t4_get_regs(sc, buf, regs->len); } #define A_PL_INDIR_CMD 0x1f8 #define S_PL_AUTOINC 31 #define M_PL_AUTOINC 0x1U #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) #define S_PL_VFID 20 #define M_PL_VFID 0xffU #define V_PL_VFID(x) ((x) << S_PL_VFID) #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) #define S_PL_ADDR 0 #define M_PL_ADDR 0xfffffU #define V_PL_ADDR(x) ((x) << S_PL_ADDR) #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) #define A_PL_INDIR_DATA 0x1fc static uint64_t read_vf_stat(struct adapter *sc, unsigned int viid, int reg) { u32 stats[2]; mtx_assert(&sc->reg_lock, MA_OWNED); if (sc->flags & IS_VF) { stats[0] = t4_read_reg(sc, VF_MPS_REG(reg)); stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4)); } else { t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); } return (((uint64_t)stats[1]) << 32 | stats[0]); } static void t4_get_vi_stats(struct adapter *sc, unsigned int viid, struct fw_vi_stats_vf *stats) { #define GET_STAT(name) \ read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); #undef GET_STAT } static void t4_clr_vi_stats(struct adapter *sc, unsigned int viid) { int reg; t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) t4_write_reg(sc, A_PL_INDIR_DATA, 0); } static void vi_refresh_stats(struct adapter *sc, struct vi_info *vi) { struct timeval tv; const struct timeval interval = {0, 250000}; /* 250ms */ if (!(vi->flags & VI_INIT_DONE)) return; getmicrotime(&tv); timevalsub(&tv, &interval); if (timevalcmp(&tv, &vi->last_refreshed, <)) return; mtx_lock(&sc->reg_lock); t4_get_vi_stats(sc, vi->viid, &vi->stats); getmicrotime(&vi->last_refreshed); mtx_unlock(&sc->reg_lock); } static void cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) { int i; u_int v, tnl_cong_drops; struct timeval tv; const struct timeval interval = {0, 250000}; /* 250ms */ getmicrotime(&tv); timevalsub(&tv, &interval); if (timevalcmp(&tv, &pi->last_refreshed, <)) return; tnl_cong_drops = 0; t4_get_port_stats(sc, pi->tx_chan, &pi->stats); for (i = 0; i < sc->chip_params->nchan; i++) { if (pi->rx_chan_map & (1 << i)) { mtx_lock(&sc->reg_lock); t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1, A_TP_MIB_TNL_CNG_DROP_0 + i); mtx_unlock(&sc->reg_lock); tnl_cong_drops += v; } } pi->tnl_cong_drops = tnl_cong_drops; getmicrotime(&pi->last_refreshed); } static void cxgbe_tick(void *arg) { struct port_info *pi = arg; struct adapter *sc = pi->adapter; PORT_LOCK_ASSERT_OWNED(pi); cxgbe_refresh_stats(sc, pi); callout_schedule(&pi->tick, hz); } void vi_tick(void *arg) { struct vi_info *vi = arg; struct adapter *sc = vi->pi->adapter; vi_refresh_stats(sc, vi); callout_schedule(&vi->tick, hz); } static void cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) { struct ifnet *vlan; if (arg != ifp || ifp->if_type != IFT_ETHER) return; vlan = VLAN_DEVAT(ifp, vid); VLAN_SETCOOKIE(vlan, ifp); } /* * Should match fw_caps_config_ enums in t4fw_interface.h */ static char *caps_decoder[] = { "\20\001IPMI\002NCSI", /* 0: NBM */ "\20\001PPP\002QFC\003DCBX", /* 1: link */ "\20\001INGRESS\002EGRESS", /* 2: switch */ "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */ "\006HASHFILTER\007ETHOFLD", "\20\001TOE", /* 4: TOE */ "\20\001RDDP\002RDMAC", /* 5: RDMA */ "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */ "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD" "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD" "\007T10DIF" "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD", "\20\001LOOKASIDE\002TLSKEYS", /* 7: Crypto */ "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */ "\004PO_INITIATOR\005PO_TARGET", }; void t4_sysctls(struct adapter *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children, *c0; static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; ctx = device_get_sysctl_ctx(sc->dev); /* * dev.t4nex.X. */ oid = device_get_sysctl_tree(sc->dev); c0 = children = SYSCTL_CHILDREN(oid); sc->sc_do_rxcopy = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, sc->params.nports, "# of ports"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, sysctl_bitfield, "A", "available doorbells"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, sc->params.vpd.cclk, "core clock frequency (in KHz)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val), sysctl_int_array, "A", "interrupt holdoff timer values (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val), sysctl_int_array, "A", "interrupt holdoff packet counter values"); t4_sge_sysctls(sc, ctx, children); sc->lro_timeout = 100; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW, &sc->debug_flags, 0, "flags to enable runtime debugging"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version", CTLFLAG_RD, sc->tp_version, 0, "TP microcode version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", CTLFLAG_RD, sc->fw_version, 0, "firmware version"); if (sc->flags & IS_VF) return; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, NULL, chip_rev(sc), "chip hardware revision"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn", CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn", CTLFLAG_RD, sc->params.vpd.pn, 0, "part number"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec", CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na", CTLFLAG_RD, sc->params.vpd.na, 0, "network address"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD, sc->er_version, 0, "expansion ROM version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD, sc->bs_version, 0, "bootstrap firmware version"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD, NULL, sc->params.scfg_vers, "serial config version"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD, NULL, sc->params.vpd_vers, "VPD version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, sc->cfcsum, "config file checksum"); #define SYSCTL_CAP(name, n, text) \ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \ CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], sc->name, \ sysctl_bitfield, "A", "available " text " capabilities") SYSCTL_CAP(nbmcaps, 0, "NBM"); SYSCTL_CAP(linkcaps, 1, "link"); SYSCTL_CAP(switchcaps, 2, "switch"); SYSCTL_CAP(niccaps, 3, "NIC"); SYSCTL_CAP(toecaps, 4, "TCP offload"); SYSCTL_CAP(rdmacaps, 5, "RDMA"); SYSCTL_CAP(iscsicaps, 6, "iSCSI"); SYSCTL_CAP(cryptocaps, 7, "crypto"); SYSCTL_CAP(fcoecaps, 8, "FCoE"); #undef SYSCTL_CAP SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, NULL, sc->tids.nftids, "number of filters"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | CTLFLAG_RD, sc, 0, sysctl_temperature, "I", "chip temperature (in Celsius)"); #ifdef SBUF_DRAIN /* * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. */ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", CTLFLAG_RD | CTLFLAG_SKIP, NULL, "logs and miscellaneous information"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cctrl, "A", "congestion control"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", CTLTYPE_STRING | CTLFLAG_RD, sc, 3, sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", CTLTYPE_STRING | CTLFLAG_RD, sc, 4, sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", CTLTYPE_STRING | CTLFLAG_RD, sc, 5, sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6, "A", "CIM logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); if (chip_id(sc) > CHELSIO_T4) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); } SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cim_qcfg, "A", "CIM queue configuration"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cpl_stats, "A", "CPL statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_ddp_stats, "A", "non-TCP DDP statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_devlog, "A", "firmware's device log"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_fcoe_stats, "A", "FCoE statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_hw_sched, "A", "hardware scheduler "); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_l2t, "A", "hardware L2 table"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_lb_stats, "A", "loopback statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_meminfo, "A", "memory regions"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6, "A", "MPS TCAM entries"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_path_mtus, "A", "path MTUs"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_pm_stats, "A", "PM statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_rdma_stats, "A", "RDMA statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tcp_stats, "A", "TCP statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tids, "A", "TID information"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_err_stats, "A", "TP error statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask", CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I", "TP logic analyzer event capture mask"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_la, "A", "TP logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tx_rate, "A", "Tx rate"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_ulprx_la, "A", "ULPRX logic analyzer"); if (chip_id(sc) >= CHELSIO_T5) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_wcwr_stats, "A", "write combined work requests"); } #endif #ifdef TCP_OFFLOAD if (is_offload(sc)) { int i; char s[4]; /* * dev.t4nex.X.toe. */ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, NULL, "TOE parameters"); children = SYSCTL_CHILDREN(oid); sc->tt.sndbuf = 256 * 1024; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, &sc->tt.sndbuf, 0, "max hardware send buffer size"); sc->tt.ddp = 0; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, &sc->tt.ddp, 0, "DDP allowed"); sc->tt.rx_coalesce = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); sc->tt.tx_align = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); sc->tt.tx_zcopy = 0; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy", CTLFLAG_RW, &sc->tt.tx_zcopy, 0, "Enable zero-copy aio_write(2)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A", "TP timer tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick", CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A", "TCP timestamp tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick", CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A", "DACK tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer", CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer, "IU", "DACK timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN, sysctl_tp_timer, "LU", "Minimum retransmit interval (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX, sysctl_tp_timer, "LU", "Maximum retransmit interval (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN, sysctl_tp_timer, "LU", "Persist timer min (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX, sysctl_tp_timer, "LU", "Persist timer max (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE, sysctl_tp_timer, "LU", "Keepalive idle timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL, sysctl_tp_timer, "LU", "Keepalive interval timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT, sysctl_tp_timer, "LU", "Initial SRTT (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer", CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER, sysctl_tp_timer, "LU", "FINWAIT2 timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count", CTLTYPE_UINT | CTLFLAG_RD, sc, S_SYNSHIFTMAX, sysctl_tp_shift_cnt, "IU", "Number of SYN retransmissions before abort"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count", CTLTYPE_UINT | CTLFLAG_RD, sc, S_RXTSHIFTMAXR2, sysctl_tp_shift_cnt, "IU", "Number of retransmissions before abort"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count", CTLTYPE_UINT | CTLFLAG_RD, sc, S_KEEPALIVEMAXR2, sysctl_tp_shift_cnt, "IU", "Number of keepalive probes before abort"); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff", CTLFLAG_RD, NULL, "TOE retransmit backoffs"); children = SYSCTL_CHILDREN(oid); for (i = 0; i < 16; i++) { snprintf(s, sizeof(s), "%u", i); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s, CTLTYPE_UINT | CTLFLAG_RD, sc, i, sysctl_tp_backoff, "IU", "TOE retransmit backoff"); } } #endif } void vi_sysctls(struct vi_info *vi) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; ctx = device_get_sysctl_ctx(vi->dev); /* * dev.v?(cxgbe|cxl).X. */ oid = device_get_sysctl_tree(vi->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, vi->viid, "VI identifer"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, &vi->nrxq, 0, "# of rx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, &vi->ntxq, 0, "# of tx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, &vi->first_rxq, 0, "index of first rx queue"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, &vi->first_txq, 0, "index of first tx queue"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL, vi->rss_size, "size of RSS indirection table"); if (IS_MAIN_VI(vi)) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", "Reserve queue 0 for non-flowid packets"); } #ifdef TCP_OFFLOAD if (vi->nofldrxq != 0) { SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, &vi->nofldrxq, 0, "# of rx queues for offloaded TCP connections"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, &vi->nofldtxq, 0, "# of tx queues for offloaded TCP connections"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", CTLFLAG_RD, &vi->first_ofld_rxq, 0, "index of first TOE rx queue"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", CTLFLAG_RD, &vi->first_ofld_txq, 0, "index of first TOE tx queue"); } #endif #ifdef DEV_NETMAP if (vi->nnmrxq != 0) { SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, &vi->nnmrxq, 0, "# of netmap rx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, &vi->nnmtxq, 0, "# of netmap tx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", CTLFLAG_RD, &vi->first_nm_rxq, 0, "index of first netmap rx queue"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", CTLFLAG_RD, &vi->first_nm_txq, 0, "index of first netmap tx queue"); } #endif SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", "holdoff timer index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", "holdoff packet counter index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", "rx queue size"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", "tx queue size"); } static void cxgbe_sysctls(struct port_info *pi) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children, *children2; struct adapter *sc = pi->adapter; int i; char name[16]; ctx = device_get_sysctl_ctx(pi->dev); /* * dev.cxgbe.X. */ oid = device_get_sysctl_tree(pi->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", "PHY temperature (in Celsius)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", "PHY firmware version"); } SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_pause_settings, "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec", CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_fec, "A", "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg", CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_autoneg, "I", "autonegotiation (-1 = not supported)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, port_top_speed(pi), "max speed (in Gbps)"); if (sc->flags & IS_VF) return; /* * dev.(cxgbe|cxl).X.tc. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL, "Tx scheduler traffic classes (cl_rl)"); for (i = 0; i < sc->chip_params->nsched_cls; i++) { struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i]; snprintf(name, sizeof(name), "%d", i); children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL, "traffic class")); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "flags", CTLFLAG_RD, &tc->flags, 0, "flags"); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount", CTLFLAG_RD, &tc->refcount, 0, "references to this class"); #ifdef SBUF_DRAIN SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params", CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i, sysctl_tc_params, "A", "traffic class parameters"); #endif } /* * dev.cxgbe.X.stats. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, NULL, "port statistics"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, &pi->tx_parse_error, 0, "# of tx packets with invalid length or # of segments"); #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ sysctl_handle_t4_reg64, "QU", desc) SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", "# of tx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", "# of frames received with bad FCS", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "rx_len_err", "# of frames received with length error", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", "# of rx frames in this range", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); #undef SYSCTL_ADD_T4_REG64 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ &pi->stats.name, desc) /* We get these from port_stats and they may be stale by up to 1s */ SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, "# drops due to buffer-group 0 overflows"); SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, "# drops due to buffer-group 1 overflows"); SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, "# drops due to buffer-group 2 overflows"); SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, "# drops due to buffer-group 3 overflows"); SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, "# of buffer-group 0 truncated packets"); SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, "# of buffer-group 1 truncated packets"); SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, "# of buffer-group 2 truncated packets"); SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, "# of buffer-group 3 truncated packets"); #undef SYSCTL_ADD_T4_PORTSTAT } static int sysctl_int_array(SYSCTL_HANDLER_ARGS) { int rc, *i, space = 0; struct sbuf sb; sbuf_new_for_sysctl(&sb, NULL, 64, req); for (i = arg1; arg2; arg2 -= sizeof(int), i++) { if (space) sbuf_printf(&sb, " "); sbuf_printf(&sb, "%d", *i); space = 1; } rc = sbuf_finish(&sb); sbuf_delete(&sb); return (rc); } static int sysctl_bitfield(SYSCTL_HANDLER_ARGS) { int rc; struct sbuf *sb; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return(rc); sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_btphy(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; int op = arg2; struct adapter *sc = pi->adapter; u_int v; int rc; rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); if (rc) return (rc); /* XXX: magic numbers */ rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, &v); end_synchronized_op(sc, 0); if (rc) return (rc); if (op == 0) v /= 256; rc = sysctl_handle_int(oidp, &v, 0, req); return (rc); } static int sysctl_noflowq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; int rc, val; val = vi->rsrv_noflowq; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if ((val >= 1) && (vi->ntxq > 1)) vi->rsrv_noflowq = 1; else vi->rsrv_noflowq = 0; return (rc); } static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->pi->adapter; int idx, rc, i; struct sge_rxq *rxq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif uint8_t v; idx = vi->tmr_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < 0 || idx >= SGE_NTIMERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4tmr"); if (rc) return (rc); v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); for_each_rxq(vi, i, rxq) { #ifdef atomic_store_rel_8 atomic_store_rel_8(&rxq->iq.intr_params, v); #else rxq->iq.intr_params = v; #endif } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { #ifdef atomic_store_rel_8 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); #else ofld_rxq->iq.intr_params = v; #endif } #endif vi->tmr_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (0); } static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->pi->adapter; int idx, rc; idx = vi->pktc_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < -1 || idx >= SGE_NCOUNTERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4pktc"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->pktc_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->pi->adapter; int qsize, rc; qsize = vi->qsize_rxq; rc = sysctl_handle_int(oidp, &qsize, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (qsize < 128 || (qsize & 7)) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4rxqs"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->qsize_rxq = qsize; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->pi->adapter; int qsize, rc; qsize = vi->qsize_txq; rc = sysctl_handle_int(oidp, &qsize, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (qsize < 128 || qsize > 65536) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4txqs"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->qsize_txq = qsize; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; if (req->newptr == NULL) { struct sbuf *sb; static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return(rc); sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); rc = sbuf_finish(sb); sbuf_delete(sb); } else { char s[2]; int n; s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); s[1] = 0; rc = sysctl_handle_string(oidp, s, sizeof(s), req); if (rc != 0) return(rc); if (s[1] != 0) return (EINVAL); if (s[0] < '0' || s[0] > '9') return (EINVAL); /* not a number */ n = s[0] - '0'; if (n & ~(PAUSE_TX | PAUSE_RX)) return (EINVAL); /* some other bit is set too */ rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4PAUSE"); if (rc) return (rc); if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); lc->requested_fc |= n; rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); if (rc == 0) { lc->fc = lc->requested_fc; } } end_synchronized_op(sc, 0); } return (rc); } static int sysctl_fec(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; if (req->newptr == NULL) { struct sbuf *sb; static char *bits = "\20\1RS\2BASER_RS\3RESERVED"; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return(rc); sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", lc->fec & M_FW_PORT_CAP_FEC, bits); rc = sbuf_finish(sb); sbuf_delete(sb); } else { char s[2]; int n; s[0] = '0' + (lc->requested_fec & M_FW_PORT_CAP_FEC); s[1] = 0; rc = sysctl_handle_string(oidp, s, sizeof(s), req); if (rc != 0) return(rc); if (s[1] != 0) return (EINVAL); if (s[0] < '0' || s[0] > '9') return (EINVAL); /* not a number */ n = s[0] - '0'; if (n & ~M_FW_PORT_CAP_FEC) return (EINVAL); /* some other bit is set too */ rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4fec"); if (rc) return (rc); if ((lc->requested_fec & M_FW_PORT_CAP_FEC) != n) { lc->requested_fec = n & G_FW_PORT_CAP_FEC(lc->supported); rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); if (rc == 0) { lc->fec = lc->requested_fec; } } end_synchronized_op(sc, 0); } return (rc); } static int sysctl_autoneg(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc, val, old; if (lc->supported & FW_PORT_CAP_ANEG) val = lc->requested_aneg == AUTONEG_ENABLE ? 1 : 0; else val = -1; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if ((lc->supported & FW_PORT_CAP_ANEG) == 0) return (ENOTSUP); if (val == 0) val = AUTONEG_DISABLE; else if (val == 1) val = AUTONEG_ENABLE; else return (EINVAL); if (lc->requested_aneg == val) return (0); /* no change */ rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4aneg"); if (rc) return (rc); old = lc->requested_aneg; lc->requested_aneg = val; rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); if (rc != 0) lc->requested_aneg = old; end_synchronized_op(sc, 0); return (rc); } static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int reg = arg2; uint64_t val; val = t4_read_reg64(sc, reg); return (sysctl_handle_64(oidp, &val, 0, req)); } static int sysctl_temperature(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, t; uint32_t param, val; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); if (rc) return (rc); param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); end_synchronized_op(sc, 0); if (rc) return (rc); /* unknown is returned as 0 but we display -1 in that case */ t = val == 0 ? -1 : val; rc = sysctl_handle_int(oidp, &t, 0, req); return (rc); } #ifdef SBUF_DRAIN static int sysctl_cctrl(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint16_t incr[NMTUS][NCCTRL_WIN]; static const char *dec_fac[] = { "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", "0.9375" }; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); t4_read_cong_tbl(sc, incr); for (i = 0; i < NCCTRL_WIN; ++i) { sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], incr[5][i], incr[6][i], incr[7][i]); sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", incr[8][i], incr[9][i], incr[10][i], incr[11][i], incr[12][i], incr[13][i], incr[14][i], incr[15][i], sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ }; static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, n, qid = arg2; uint32_t *buf, *p; char *qtype; u_int cim_num_obq = sc->chip_params->cim_num_obq; KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, ("%s: bad qid %d\n", __func__, qid)); if (qid < CIM_NUM_IBQ) { /* inbound queue */ qtype = "IBQ"; n = 4 * CIM_IBQ_SIZE; buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = t4_read_cim_ibq(sc, qid, buf, n); } else { /* outbound queue */ qtype = "OBQ"; qid -= CIM_NUM_IBQ; n = 4 * cim_num_obq * CIM_OBQ_SIZE; buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = t4_read_cim_obq(sc, qid, buf, n); } if (rc < 0) { rc = -rc; goto done; } n = rc * sizeof(uint32_t); /* rc has # of words actually read */ rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) goto done; sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); if (sb == NULL) { rc = ENOMEM; goto done; } sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); for (i = 0, p = buf; i < n; i += 16, p += 4) sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], p[2], p[3]); rc = sbuf_finish(sb); sbuf_delete(sb); done: free(buf, M_CXGBE); return (rc); } static int sysctl_cim_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int cfg; struct sbuf *sb; uint32_t *buf, *p; int rc; MPASS(chip_id(sc) <= CHELSIO_T5); rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); if (rc != 0) return (rc); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = -t4_cim_read_la(sc, buf, NULL); if (rc != 0) goto done; sbuf_printf(sb, "Status Data PC%s", cfg & F_UPDBGLACAPTPCONLY ? "" : " LS0Stat LS0Addr LS0Data"); for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { if (cfg & F_UPDBGLACAPTPCONLY) { sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, p[6], p[7]); sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, p[4] & 0xff, p[5] >> 8); sbuf_printf(sb, "\n %02x %x%07x %x%07x", (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, p[1] & 0xf, p[2] >> 4); } else { sbuf_printf(sb, "\n %02x %x%07x %x%07x %08x %08x " "%08x%08x%08x%08x", (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], p[6], p[7]); } } rc = sbuf_finish(sb); sbuf_delete(sb); done: free(buf, M_CXGBE); return (rc); } static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int cfg; struct sbuf *sb; uint32_t *buf, *p; int rc; MPASS(chip_id(sc) > CHELSIO_T5); rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); if (rc != 0) return (rc); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = -t4_cim_read_la(sc, buf, NULL); if (rc != 0) goto done; sbuf_printf(sb, "Status Inst Data PC%s", cfg & F_UPDBGLACAPTPCONLY ? "" : " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { if (cfg & F_UPDBGLACAPTPCONLY) { sbuf_printf(sb, "\n %02x %08x %08x %08x", p[3] & 0xff, p[2], p[1], p[0]); sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, p[6] >> 16); } else { sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " "%08x %08x %08x %08x %08x %08x", (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, p[6] >> 16, p[2], p[1], p[0], p[5], p[4], p[3]); } } rc = sbuf_finish(sb); sbuf_delete(sb); done: free(buf, M_CXGBE); return (rc); } static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int i; struct sbuf *sb; uint32_t *buf, *p; int rc; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); p = buf; for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], p[1], p[0]); } sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, (p[1] >> 2) | ((p[2] & 3) << 30), (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, p[0] & 1); } rc = sbuf_finish(sb); sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int i; struct sbuf *sb; uint32_t *buf, *p; int rc; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); p = buf; sbuf_printf(sb, "Cntl ID DataBE Addr Data"); for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, p[4], p[3], p[2], p[1], p[0]); } sbuf_printf(sb, "\n\nCntl ID Data"); for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); } rc = sbuf_finish(sb); sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; uint16_t thres[CIM_NUM_IBQ]; uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; cim_num_obq = sc->chip_params->cim_num_obq; if (is_t4(sc)) { ibq_rdaddr = A_UP_IBQ_0_RDADDR; obq_rdaddr = A_UP_OBQ_0_REALADDR; } else { ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; } nq = CIM_NUM_IBQ + cim_num_obq; rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); if (rc == 0) rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); if (rc != 0) return (rc); t4_read_cimq_cfg(sc, base, size, thres); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), G_QUEREMFLITS(p[2]) * 16); for ( ; i < nq; i++, p += 4, wr += 2) sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), G_QUEREMFLITS(p[2]) * 16); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_cpl_stats stats; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); - t4_tp_get_cpl_stats(sc, &stats); + t4_tp_get_cpl_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3"); sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", stats.req[0], stats.req[1], stats.req[2], stats.req[3]); sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); } else { sbuf_printf(sb, " channel 0 channel 1"); sbuf_printf(sb, "\nCPL requests: %10u %10u", stats.req[0], stats.req[1]); sbuf_printf(sb, "\nCPL responses: %10u %10u", stats.rsp[0], stats.rsp[1]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_usm_stats stats; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return(rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); - t4_get_usm_stats(sc, &stats); + t4_get_usm_stats(sc, &stats, 1); sbuf_printf(sb, "Frames: %u\n", stats.frames); sbuf_printf(sb, "Octets: %ju\n", stats.octets); sbuf_printf(sb, "Drops: %u", stats.drops); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static const char * const devlog_level_strings[] = { [FW_DEVLOG_LEVEL_EMERG] = "EMERG", [FW_DEVLOG_LEVEL_CRIT] = "CRIT", [FW_DEVLOG_LEVEL_ERR] = "ERR", [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", [FW_DEVLOG_LEVEL_INFO] = "INFO", [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" }; static const char * const devlog_facility_strings[] = { [FW_DEVLOG_FACILITY_CORE] = "CORE", [FW_DEVLOG_FACILITY_CF] = "CF", [FW_DEVLOG_FACILITY_SCHED] = "SCHED", [FW_DEVLOG_FACILITY_TIMER] = "TIMER", [FW_DEVLOG_FACILITY_RES] = "RES", [FW_DEVLOG_FACILITY_HW] = "HW", [FW_DEVLOG_FACILITY_FLR] = "FLR", [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", [FW_DEVLOG_FACILITY_PHY] = "PHY", [FW_DEVLOG_FACILITY_MAC] = "MAC", [FW_DEVLOG_FACILITY_PORT] = "PORT", [FW_DEVLOG_FACILITY_VI] = "VI", [FW_DEVLOG_FACILITY_FILTER] = "FILTER", [FW_DEVLOG_FACILITY_ACL] = "ACL", [FW_DEVLOG_FACILITY_TM] = "TM", [FW_DEVLOG_FACILITY_QFC] = "QFC", [FW_DEVLOG_FACILITY_DCB] = "DCB", [FW_DEVLOG_FACILITY_ETH] = "ETH", [FW_DEVLOG_FACILITY_OFLD] = "OFLD", [FW_DEVLOG_FACILITY_RI] = "RI", [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", [FW_DEVLOG_FACILITY_FCOE] = "FCOE", [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE", [FW_DEVLOG_FACILITY_CHNET] = "CHNET", }; static int sysctl_devlog(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct devlog_params *dparams = &sc->params.devlog; struct fw_devlog_e *buf, *e; int i, j, rc, nentries, first = 0; struct sbuf *sb; uint64_t ftstamp = UINT64_MAX; if (dparams->addr == 0) return (ENXIO); buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); if (buf == NULL) return (ENOMEM); rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size); if (rc != 0) goto done; nentries = dparams->size / sizeof(struct fw_devlog_e); for (i = 0; i < nentries; i++) { e = &buf[i]; if (e->timestamp == 0) break; /* end */ e->timestamp = be64toh(e->timestamp); e->seqno = be32toh(e->seqno); for (j = 0; j < 8; j++) e->params[j] = be32toh(e->params[j]); if (e->timestamp < ftstamp) { ftstamp = e->timestamp; first = i; } } if (buf[first].timestamp == 0) goto done; /* nothing in the log */ rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) goto done; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) { rc = ENOMEM; goto done; } sbuf_printf(sb, "%10s %15s %8s %8s %s\n", "Seq#", "Tstamp", "Level", "Facility", "Message"); i = first; do { e = &buf[i]; if (e->timestamp == 0) break; /* end */ sbuf_printf(sb, "%10d %15ju %8s %8s ", e->seqno, e->timestamp, (e->level < nitems(devlog_level_strings) ? devlog_level_strings[e->level] : "UNKNOWN"), (e->facility < nitems(devlog_facility_strings) ? devlog_facility_strings[e->facility] : "UNKNOWN")); sbuf_printf(sb, e->fmt, e->params[0], e->params[1], e->params[2], e->params[3], e->params[4], e->params[5], e->params[6], e->params[7]); if (++i == nentries) i = 0; } while (i != first); rc = sbuf_finish(sb); sbuf_delete(sb); done: free(buf, M_CXGBE); return (rc); } static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_fcoe_stats stats[MAX_NCHAN]; int i, nchan = sc->chip_params->nchan; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); for (i = 0; i < nchan; i++) - t4_get_fcoe_stats(sc, i, &stats[i]); + t4_get_fcoe_stats(sc, i, &stats[i], 1); if (nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3"); sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", stats[0].octets_ddp, stats[1].octets_ddp, stats[2].octets_ddp, stats[3].octets_ddp); sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", stats[0].frames_ddp, stats[1].frames_ddp, stats[2].frames_ddp, stats[3].frames_ddp); sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", stats[0].frames_drop, stats[1].frames_drop, stats[2].frames_drop, stats[3].frames_drop); } else { sbuf_printf(sb, " channel 0 channel 1"); sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", stats[0].octets_ddp, stats[1].octets_ddp); sbuf_printf(sb, "\nframesDDP: %16u %16u", stats[0].frames_ddp, stats[1].frames_ddp); sbuf_printf(sb, "\nframesDrop: %16u %16u", stats[0].frames_drop, stats[1].frames_drop); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; unsigned int map, kbps, ipg, mode; unsigned int pace_tab[NTX_SCHED]; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); t4_read_pace_tbl(sc, pace_tab); sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " "Class IPG (0.1 ns) Flow IPG (us)"); for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { - t4_get_tx_sched(sc, i, &kbps, &ipg); + t4_get_tx_sched(sc, i, &kbps, &ipg, 1); sbuf_printf(sb, "\n %u %-5s %u ", i, (mode & (1 << i)) ? "flow" : "class", map & 3); if (kbps) sbuf_printf(sb, "%9u ", kbps); else sbuf_printf(sb, " disabled "); if (ipg) sbuf_printf(sb, "%13u ", ipg); else sbuf_printf(sb, " disabled "); if (pace_tab[i]) sbuf_printf(sb, "%10u", pace_tab[i]); else sbuf_printf(sb, " disabled"); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, j; uint64_t *p0, *p1; struct lb_port_stats s[2]; static const char *stat_name[] = { "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", "Frames128To255:", "Frames256To511:", "Frames512To1023:", "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", "BG2FramesTrunc:", "BG3FramesTrunc:" }; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); memset(s, 0, sizeof(s)); for (i = 0; i < sc->chip_params->nchan; i += 2) { t4_get_lb_stats(sc, i, &s[0]); t4_get_lb_stats(sc, i + 1, &s[1]); p0 = &s[0].octets; p1 = &s[1].octets; sbuf_printf(sb, "%s Loopback %u" " Loopback %u", i == 0 ? "" : "\n", i, i + 1); for (j = 0; j < nitems(stat_name); j++) sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], *p0++, *p1++); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) { int rc = 0; struct port_info *pi = arg1; struct link_config *lc = &pi->link_cfg; struct sbuf *sb; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return(rc); sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); if (sb == NULL) return (ENOMEM); if (lc->link_ok || lc->link_down_rc == 255) sbuf_printf(sb, "n/a"); else sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc)); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } struct mem_desc { unsigned int base; unsigned int limit; unsigned int idx; }; static int mem_desc_cmp(const void *a, const void *b) { return ((const struct mem_desc *)a)->base - ((const struct mem_desc *)b)->base; } static void mem_region_show(struct sbuf *sb, const char *name, unsigned int from, unsigned int to) { unsigned int size; if (from == to) return; size = to - from + 1; if (size == 0) return; /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); } static int sysctl_meminfo(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, n; uint32_t lo, hi, used, alloc; static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; static const char *region[] = { "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", "TDDP region:", "TPT region:", "STAG region:", "RQ region:", "RQUDP region:", "PBL region:", "TXPBL region:", "DBVFIFO region:", "ULPRX state:", "ULPTX state:", "On-chip queues:" }; struct mem_desc avail[4]; struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ struct mem_desc *md = mem; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); for (i = 0; i < nitems(mem); i++) { mem[i].limit = 0; mem[i].idx = i; } /* Find and sort the populated memory ranges */ i = 0; lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); if (lo & F_EDRAM0_ENABLE) { hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); avail[i].base = G_EDRAM0_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); avail[i].idx = 0; i++; } if (lo & F_EDRAM1_ENABLE) { hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); avail[i].base = G_EDRAM1_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); avail[i].idx = 1; i++; } if (lo & F_EXT_MEM_ENABLE) { hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); avail[i].base = G_EXT_MEM_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */ i++; } if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) { hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); avail[i].base = G_EXT_MEM1_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EXT_MEM1_SIZE(hi) << 20); avail[i].idx = 4; i++; } if (!i) /* no memory available */ return 0; qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); /* the next few have explicit upper bounds */ md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); md->limit = md->base - 1 + t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); md++; md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); md->limit = md->base - 1 + t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); md++; if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { if (chip_id(sc) <= CHELSIO_T5) md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); else md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); md->limit = 0; } else { md->base = 0; md->idx = nitems(region); /* hide it */ } md++; #define ulp_region(reg) \ md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) ulp_region(RX_ISCSI); ulp_region(RX_TDDP); ulp_region(TX_TPT); ulp_region(RX_STAG); ulp_region(RX_RQ); ulp_region(RX_RQUDP); ulp_region(RX_PBL); ulp_region(TX_PBL); #undef ulp_region md->base = 0; md->idx = nitems(region); if (!is_t4(sc)) { uint32_t size = 0; uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); if (is_t5(sc)) { if (sge_ctrl & F_VFIFO_ENABLE) size = G_DBVFIFO_SIZE(fifo_size); } else size = G_T6_DBVFIFO_SIZE(fifo_size); if (size) { md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); md->limit = md->base + (size << 2) - 1; } } md++; md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); md->limit = 0; md++; md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); md->limit = 0; md++; md->base = sc->vres.ocq.start; if (sc->vres.ocq.size) md->limit = md->base + sc->vres.ocq.size - 1; else md->idx = nitems(region); /* hide it */ md++; /* add any address-space holes, there can be up to 3 */ for (n = 0; n < i - 1; n++) if (avail[n].limit < avail[n + 1].base) (md++)->base = avail[n].limit; if (avail[n].limit) (md++)->base = avail[n].limit; n = md - mem; qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); for (lo = 0; lo < i; lo++) mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, avail[lo].limit - 1); sbuf_printf(sb, "\n"); for (i = 0; i < n; i++) { if (mem[i].idx >= nitems(region)) continue; /* skip holes */ if (!mem[i].limit) mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; mem_region_show(sb, region[mem[i].idx], mem[i].base, mem[i].limit); } sbuf_printf(sb, "\n"); lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; mem_region_show(sb, "uP RAM:", lo, hi); lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; mem_region_show(sb, "uP Extmem2:", lo, hi); lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", G_PMRXMAXPAGE(lo), t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, (lo & F_PMRXNUMCHN) ? 2 : 1); lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", G_PMTXMAXPAGE(lo), hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); sbuf_printf(sb, "%u p-structs\n", t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); for (i = 0; i < 4; i++) { if (chip_id(sc) > CHELSIO_T5) lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); else lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); if (is_t5(sc)) { used = G_T5_USED(lo); alloc = G_T5_ALLOC(lo); } else { used = G_USED(lo); alloc = G_ALLOC(lo); } /* For T6 these are MAC buffer groups */ sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", i, used, alloc); } for (i = 0; i < sc->chip_params->nchan; i++) { if (chip_id(sc) > CHELSIO_T5) lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); else lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); if (is_t5(sc)) { used = G_T5_USED(lo); alloc = G_T5_ALLOC(lo); } else { used = G_USED(lo); alloc = G_ALLOC(lo); } /* For T6 these are MAC buffer groups */ sbuf_printf(sb, "\nLoopback %d using %u pages out of %u allocated", i, used, alloc); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static inline void tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) { *mask = x | y; y = htobe64(y); memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); } static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; MPASS(chip_id(sc) <= CHELSIO_T5); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "Idx Ethernet address Mask Vld Ports PF" " VF Replication P0 P1 P2 P3 ML"); for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { uint64_t tcamx, tcamy, mask; uint32_t cls_lo, cls_hi; uint8_t addr[ETHER_ADDR_LEN]; tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); if (tcamx & tcamy) continue; tcamxy2valmask(tcamx, tcamy, addr, &mask); cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask, (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', G_PORTMAP(cls_hi), G_PF(cls_lo), (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); if (cls_lo & F_REPLICATE) { struct fw_ldst_cmd ldst_cmd; memset(&ldst_cmd, 0, sizeof(ldst_cmd)); ldst_cmd.op_to_addrspace = htobe32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); ldst_cmd.u.mps.rplc.fid_idx = htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | V_FW_LDST_CMD_IDX(i)); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mps"); if (rc) break; rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, sizeof(ldst_cmd), &ldst_cmd); end_synchronized_op(sc, 0); if (rc != 0) { sbuf_printf(sb, "%36d", rc); rc = 0; } else { sbuf_printf(sb, " %08x %08x %08x %08x", be32toh(ldst_cmd.u.mps.rplc.rplc127_96), be32toh(ldst_cmd.u.mps.rplc.rplc95_64), be32toh(ldst_cmd.u.mps.rplc.rplc63_32), be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); } } else sbuf_printf(sb, "%36s", ""); sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); } if (rc) (void) sbuf_finish(sb); else rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; MPASS(chip_id(sc) > CHELSIO_T5); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" " Replication" " P0 P1 P2 P3 ML\n"); for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { uint8_t dip_hit, vlan_vld, lookup_type, port_num; uint16_t ivlan; uint64_t tcamx, tcamy, val, mask; uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; uint8_t addr[ETHER_ADDR_LEN]; ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); if (i < 256) ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); else ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); tcamy = G_DMACH(val) << 32; tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); lookup_type = G_DATALKPTYPE(data2); port_num = G_DATAPORTNUM(data2); if (lookup_type && lookup_type != M_DATALKPTYPE) { /* Inner header VNI */ vniy = ((data2 & F_DATAVIDH2) << 23) | (G_DATAVIDH1(data2) << 16) | G_VIDL(val); dip_hit = data2 & F_DATADIPHIT; vlan_vld = 0; } else { vniy = 0; dip_hit = 0; vlan_vld = data2 & F_DATAVIDH2; ivlan = G_VIDL(val); } ctl |= V_CTLXYBITSEL(1); t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); tcamx = G_DMACH(val) << 32; tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); if (lookup_type && lookup_type != M_DATALKPTYPE) { /* Inner header VNI mask */ vnix = ((data2 & F_DATAVIDH2) << 23) | (G_DATAVIDH1(data2) << 16) | G_VIDL(val); } else vnix = 0; if (tcamx & tcamy) continue; tcamxy2valmask(tcamx, tcamy, addr, &mask); cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); if (lookup_type && lookup_type != M_DATALKPTYPE) { sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " "%012jx %06x %06x - - %3c" " 'I' %4x %3c %#x%4u%4d", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', G_PORTMAP(cls_hi), G_T6_PF(cls_lo), cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); } else { sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " "%012jx - - ", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask); if (vlan_vld) sbuf_printf(sb, "%4u Y ", ivlan); else sbuf_printf(sb, " - N "); sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", lookup_type ? 'I' : 'O', port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', G_PORTMAP(cls_hi), G_T6_PF(cls_lo), cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); } if (cls_lo & F_T6_REPLICATE) { struct fw_ldst_cmd ldst_cmd; memset(&ldst_cmd, 0, sizeof(ldst_cmd)); ldst_cmd.op_to_addrspace = htobe32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); ldst_cmd.u.mps.rplc.fid_idx = htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | V_FW_LDST_CMD_IDX(i)); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t6mps"); if (rc) break; rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, sizeof(ldst_cmd), &ldst_cmd); end_synchronized_op(sc, 0); if (rc != 0) { sbuf_printf(sb, "%72d", rc); rc = 0; } else { sbuf_printf(sb, " %08x %08x %08x %08x" " %08x %08x %08x %08x", be32toh(ldst_cmd.u.mps.rplc.rplc255_224), be32toh(ldst_cmd.u.mps.rplc.rplc223_192), be32toh(ldst_cmd.u.mps.rplc.rplc191_160), be32toh(ldst_cmd.u.mps.rplc.rplc159_128), be32toh(ldst_cmd.u.mps.rplc.rplc127_96), be32toh(ldst_cmd.u.mps.rplc.rplc95_64), be32toh(ldst_cmd.u.mps.rplc.rplc63_32), be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); } } else sbuf_printf(sb, "%72s", ""); sbuf_printf(sb, "%4u%3u%3u%3u %#x", G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), (cls_lo >> S_T6_MULTILISTEN0) & 0xf); } if (rc) (void) sbuf_finish(sb); else rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; uint16_t mtus[NMTUS]; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); t4_read_mtu_tbl(sc, mtus, NULL); sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], mtus[14], mtus[15]); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; static const char *tx_stats[MAX_PM_NSTATS] = { "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", "Tx FIFO wait", NULL, "Tx latency" }; static const char *rx_stats[MAX_PM_NSTATS] = { "Read:", "Write bypass:", "Write mem:", "Flush:", "Rx FIFO wait", NULL, "Rx latency" }; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); sbuf_printf(sb, " Tx pcmds Tx bytes"); for (i = 0; i < 4; i++) { sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); } sbuf_printf(sb, "\n Rx pcmds Rx bytes"); for (i = 0; i < 4; i++) { sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); } if (chip_id(sc) > CHELSIO_T5) { sbuf_printf(sb, "\n Total wait Total occupancy"); sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); i += 2; MPASS(i < nitems(tx_stats)); sbuf_printf(sb, "\n Reads Total wait"); sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_rdma_stats stats; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); - t4_tp_get_rdma_stats(sc, &stats); + t4_tp_get_rdma_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_tcp_stats v4, v6; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); - t4_tp_get_tcp_stats(sc, &v4, &v6); + t4_tp_get_tcp_stats(sc, &v4, &v6, 0); mtx_unlock(&sc->reg_lock); sbuf_printf(sb, " IP IPv6\n"); sbuf_printf(sb, "OutRsts: %20u %20u\n", v4.tcp_out_rsts, v6.tcp_out_rsts); sbuf_printf(sb, "InSegs: %20ju %20ju\n", v4.tcp_in_segs, v6.tcp_in_segs); sbuf_printf(sb, "OutSegs: %20ju %20ju\n", v4.tcp_out_segs, v6.tcp_out_segs); sbuf_printf(sb, "RetransSegs: %20ju %20ju", v4.tcp_retrans_segs, v6.tcp_retrans_segs); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tids(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tid_info *t = &sc->tids; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (t->natids) { sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, t->atids_in_use); } if (t->ntids) { sbuf_printf(sb, "TID range: "); if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { uint32_t b, hb; if (chip_id(sc) <= CHELSIO_T5) { b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; } else { b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX); hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE); } if (b) sbuf_printf(sb, "0-%u, ", b - 1); sbuf_printf(sb, "%u-%u", hb, t->ntids - 1); } else sbuf_printf(sb, "0-%u", t->ntids - 1); sbuf_printf(sb, ", in use: %u\n", atomic_load_acq_int(&t->tids_in_use)); } if (t->nstids) { sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, t->stid_base + t->nstids - 1, t->stids_in_use); } if (t->nftids) { sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, t->ftid_base + t->nftids - 1); } if (t->netids) { sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, t->etid_base + t->netids - 1); } sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_err_stats stats; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); - t4_tp_get_err_stats(sc, &stats); + t4_tp_get_err_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3\n"); sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", stats.mac_in_errs[0], stats.mac_in_errs[1], stats.mac_in_errs[2], stats.mac_in_errs[3]); sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", stats.hdr_in_errs[0], stats.hdr_in_errs[1], stats.hdr_in_errs[2], stats.hdr_in_errs[3]); sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", stats.tcp_in_errs[0], stats.tcp_in_errs[1], stats.tcp_in_errs[2], stats.tcp_in_errs[3]); sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); } else { sbuf_printf(sb, " channel 0 channel 1\n"); sbuf_printf(sb, "macInErrs: %10u %10u\n", stats.mac_in_errs[0], stats.mac_in_errs[1]); sbuf_printf(sb, "hdrInErrs: %10u %10u\n", stats.hdr_in_errs[0], stats.hdr_in_errs[1]); sbuf_printf(sb, "tcpInErrs: %10u %10u\n", stats.tcp_in_errs[0], stats.tcp_in_errs[1]); sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); } sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", stats.ofld_no_neigh, stats.ofld_cong_defer); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct tp_params *tpp = &sc->params.tp; u_int mask; int rc; mask = tpp->la_mask >> 16; rc = sysctl_handle_int(oidp, &mask, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (mask > 0xffff) return (EINVAL); tpp->la_mask = mask << 16; t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask); return (0); } struct field_desc { const char *name; u_int start; u_int width; }; static void field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) { char buf[32]; int line_size = 0; while (f->name) { uint64_t mask = (1ULL << f->width) - 1; int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, ((uintmax_t)v >> f->start) & mask); if (line_size + len >= 79) { line_size = 8; sbuf_printf(sb, "\n "); } sbuf_printf(sb, "%s ", buf); line_size += len + 1; f++; } sbuf_printf(sb, "\n"); } static const struct field_desc tp_la0[] = { { "RcfOpCodeOut", 60, 4 }, { "State", 56, 4 }, { "WcfState", 52, 4 }, { "RcfOpcSrcOut", 50, 2 }, { "CRxError", 49, 1 }, { "ERxError", 48, 1 }, { "SanityFailed", 47, 1 }, { "SpuriousMsg", 46, 1 }, { "FlushInputMsg", 45, 1 }, { "FlushInputCpl", 44, 1 }, { "RssUpBit", 43, 1 }, { "RssFilterHit", 42, 1 }, { "Tid", 32, 10 }, { "InitTcb", 31, 1 }, { "LineNumber", 24, 7 }, { "Emsg", 23, 1 }, { "EdataOut", 22, 1 }, { "Cmsg", 21, 1 }, { "CdataOut", 20, 1 }, { "EreadPdu", 19, 1 }, { "CreadPdu", 18, 1 }, { "TunnelPkt", 17, 1 }, { "RcfPeerFin", 16, 1 }, { "RcfReasonOut", 12, 4 }, { "TxCchannel", 10, 2 }, { "RcfTxChannel", 8, 2 }, { "RxEchannel", 6, 2 }, { "RcfRxChannel", 5, 1 }, { "RcfDataOutSrdy", 4, 1 }, { "RxDvld", 3, 1 }, { "RxOoDvld", 2, 1 }, { "RxCongestion", 1, 1 }, { "TxCongestion", 0, 1 }, { NULL } }; static const struct field_desc tp_la1[] = { { "CplCmdIn", 56, 8 }, { "CplCmdOut", 48, 8 }, { "ESynOut", 47, 1 }, { "EAckOut", 46, 1 }, { "EFinOut", 45, 1 }, { "ERstOut", 44, 1 }, { "SynIn", 43, 1 }, { "AckIn", 42, 1 }, { "FinIn", 41, 1 }, { "RstIn", 40, 1 }, { "DataIn", 39, 1 }, { "DataInVld", 38, 1 }, { "PadIn", 37, 1 }, { "RxBufEmpty", 36, 1 }, { "RxDdp", 35, 1 }, { "RxFbCongestion", 34, 1 }, { "TxFbCongestion", 33, 1 }, { "TxPktSumSrdy", 32, 1 }, { "RcfUlpType", 28, 4 }, { "Eread", 27, 1 }, { "Ebypass", 26, 1 }, { "Esave", 25, 1 }, { "Static0", 24, 1 }, { "Cread", 23, 1 }, { "Cbypass", 22, 1 }, { "Csave", 21, 1 }, { "CPktOut", 20, 1 }, { "RxPagePoolFull", 18, 2 }, { "RxLpbkPkt", 17, 1 }, { "TxLpbkPkt", 16, 1 }, { "RxVfValid", 15, 1 }, { "SynLearned", 14, 1 }, { "SetDelEntry", 13, 1 }, { "SetInvEntry", 12, 1 }, { "CpcmdDvld", 11, 1 }, { "CpcmdSave", 10, 1 }, { "RxPstructsFull", 8, 2 }, { "EpcmdDvld", 7, 1 }, { "EpcmdFlush", 6, 1 }, { "EpcmdTrimPrefix", 5, 1 }, { "EpcmdTrimPostfix", 4, 1 }, { "ERssIp4Pkt", 3, 1 }, { "ERssIp6Pkt", 2, 1 }, { "ERssTcpUdpPkt", 1, 1 }, { "ERssFceFipPkt", 0, 1 }, { NULL } }; static const struct field_desc tp_la2[] = { { "CplCmdIn", 56, 8 }, { "MpsVfVld", 55, 1 }, { "MpsPf", 52, 3 }, { "MpsVf", 44, 8 }, { "SynIn", 43, 1 }, { "AckIn", 42, 1 }, { "FinIn", 41, 1 }, { "RstIn", 40, 1 }, { "DataIn", 39, 1 }, { "DataInVld", 38, 1 }, { "PadIn", 37, 1 }, { "RxBufEmpty", 36, 1 }, { "RxDdp", 35, 1 }, { "RxFbCongestion", 34, 1 }, { "TxFbCongestion", 33, 1 }, { "TxPktSumSrdy", 32, 1 }, { "RcfUlpType", 28, 4 }, { "Eread", 27, 1 }, { "Ebypass", 26, 1 }, { "Esave", 25, 1 }, { "Static0", 24, 1 }, { "Cread", 23, 1 }, { "Cbypass", 22, 1 }, { "Csave", 21, 1 }, { "CPktOut", 20, 1 }, { "RxPagePoolFull", 18, 2 }, { "RxLpbkPkt", 17, 1 }, { "TxLpbkPkt", 16, 1 }, { "RxVfValid", 15, 1 }, { "SynLearned", 14, 1 }, { "SetDelEntry", 13, 1 }, { "SetInvEntry", 12, 1 }, { "CpcmdDvld", 11, 1 }, { "CpcmdSave", 10, 1 }, { "RxPstructsFull", 8, 2 }, { "EpcmdDvld", 7, 1 }, { "EpcmdFlush", 6, 1 }, { "EpcmdTrimPrefix", 5, 1 }, { "EpcmdTrimPostfix", 4, 1 }, { "ERssIp4Pkt", 3, 1 }, { "ERssIp6Pkt", 2, 1 }, { "ERssTcpUdpPkt", 1, 1 }, { "ERssFceFipPkt", 0, 1 }, { NULL } }; static void tp_la_show(struct sbuf *sb, uint64_t *p, int idx) { field_desc_show(sb, *p, tp_la0); } static void tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) { if (idx) sbuf_printf(sb, "\n"); field_desc_show(sb, p[0], tp_la0); if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) field_desc_show(sb, p[1], tp_la0); } static void tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) { if (idx) sbuf_printf(sb, "\n"); field_desc_show(sb, p[0], tp_la0); if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); } static int sysctl_tp_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; uint64_t *buf, *p; int rc; u_int i, inc; void (*show_func)(struct sbuf *, uint64_t *, int); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); t4_tp_read_la(sc, buf, NULL); p = buf; switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { case 2: inc = 2; show_func = tp_la_show2; break; case 3: inc = 2; show_func = tp_la_show3; break; default: inc = 1; show_func = tp_la_show; } for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) (*show_func)(sb, p, i); rc = sbuf_finish(sb); sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); t4_get_chan_txrate(sc, nrate, orate); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3\n"); sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", nrate[0], nrate[1], nrate[2], nrate[3]); sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", orate[0], orate[1], orate[2], orate[3]); } else { sbuf_printf(sb, " channel 0 channel 1\n"); sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", nrate[0], nrate[1]); sbuf_printf(sb, "Offload B/s: %10ju %10ju", orate[0], orate[1]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; uint32_t *buf, *p; int rc, i; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); t4_ulprx_read_la(sc, buf); p = buf; sbuf_printf(sb, " Pcmd Type Message" " Data"); for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); } rc = sbuf_finish(sb); sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, v; MPASS(chip_id(sc) >= CHELSIO_T5); rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); v = t4_read_reg(sc, A_SGE_STAT_CFG); if (G_STATSOURCE_T5(v) == 7) { int mode; mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v); if (mode == 0) { sbuf_printf(sb, "total %d, incomplete %d", t4_read_reg(sc, A_SGE_STAT_TOTAL), t4_read_reg(sc, A_SGE_STAT_MATCH)); } else if (mode == 1) { sbuf_printf(sb, "total %d, data overflow %d", t4_read_reg(sc, A_SGE_STAT_TOTAL), t4_read_reg(sc, A_SGE_STAT_MATCH)); } else { sbuf_printf(sb, "unknown mode %d", mode); } } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tc_params(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct tx_cl_rl_params tc; struct sbuf *sb; int i, rc, port_id, mbps, gbps; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); port_id = arg2 >> 16; MPASS(port_id < sc->params.nports); MPASS(sc->port[port_id] != NULL); i = arg2 & 0xffff; MPASS(i < sc->chip_params->nsched_cls); mtx_lock(&sc->tc_lock); tc = sc->port[port_id]->sched_params->cl_rl[i]; mtx_unlock(&sc->tc_lock); if (tc.flags & TX_CLRL_ERROR) { sbuf_printf(sb, "error"); goto done; } if (tc.ratemode == SCHED_CLASS_RATEMODE_REL) { /* XXX: top speed or actual link speed? */ gbps = port_top_speed(sc->port[port_id]); sbuf_printf(sb, " %u%% of %uGbps", tc.maxrate, gbps); } else if (tc.ratemode == SCHED_CLASS_RATEMODE_ABS) { switch (tc.rateunit) { case SCHED_CLASS_RATEUNIT_BITS: mbps = tc.maxrate / 1000; gbps = tc.maxrate / 1000000; if (tc.maxrate == gbps * 1000000) sbuf_printf(sb, " %uGbps", gbps); else if (tc.maxrate == mbps * 1000) sbuf_printf(sb, " %uMbps", mbps); else sbuf_printf(sb, " %uKbps", tc.maxrate); break; case SCHED_CLASS_RATEUNIT_PKTS: sbuf_printf(sb, " %upps", tc.maxrate); break; default: rc = ENXIO; goto done; } } switch (tc.mode) { case SCHED_CLASS_MODE_CLASS: sbuf_printf(sb, " aggregate"); break; case SCHED_CLASS_MODE_FLOW: sbuf_printf(sb, " per-flow"); break; default: rc = ENXIO; goto done; } done: if (rc == 0) rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } #endif #ifdef TCP_OFFLOAD static void unit_conv(char *buf, size_t len, u_int val, u_int factor) { u_int rem = val % factor; if (rem == 0) snprintf(buf, len, "%u", val / factor); else { while (rem % 10 == 0) rem /= 10; snprintf(buf, len, "%u.%u", val / factor, rem); } } static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; char buf[16]; u_int res, re; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); switch (arg2) { case 0: /* timer_tick */ re = G_TIMERRESOLUTION(res); break; case 1: /* TCP timestamp tick */ re = G_TIMESTAMPRESOLUTION(res); break; case 2: /* DACK tick */ re = G_DELAYEDACKRESOLUTION(res); break; default: return (EDOOFUS); } unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000); return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); } static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int res, dack_re, v; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); dack_re = G_DELAYEDACKRESOLUTION(res); v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER); return (sysctl_handle_int(oidp, &v, 0, req)); } static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int reg = arg2; u_int tre; u_long tp_tick_us, v; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX || reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX || reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL || reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER); tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); tp_tick_us = (cclk_ps << tre) / 1000000; if (reg == A_TP_INIT_SRTT) v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg)); else v = tp_tick_us * t4_read_reg(sc, reg); return (sysctl_handle_long(oidp, &v, 0, req)); } /* * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is * passed to this function. */ static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int idx = arg2; u_int v; MPASS(idx >= 0 && idx <= 24); v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf; return (sysctl_handle_int(oidp, &v, 0, req)); } static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int idx = arg2; u_int shift, v, r; MPASS(idx >= 0 && idx < 16); r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3); shift = (idx & 3) << 3; v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0; return (sysctl_handle_int(oidp, &v, 0, req)); } #endif static uint32_t fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf) { uint32_t mode; mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; if (fconf & F_FRAGMENTATION) mode |= T4_FILTER_IP_FRAGMENT; if (fconf & F_MPSHITTYPE) mode |= T4_FILTER_MPS_HIT_TYPE; if (fconf & F_MACMATCH) mode |= T4_FILTER_MAC_IDX; if (fconf & F_ETHERTYPE) mode |= T4_FILTER_ETH_TYPE; if (fconf & F_PROTOCOL) mode |= T4_FILTER_IP_PROTO; if (fconf & F_TOS) mode |= T4_FILTER_IP_TOS; if (fconf & F_VLAN) mode |= T4_FILTER_VLAN; if (fconf & F_VNIC_ID) { mode |= T4_FILTER_VNIC; if (iconf & F_VNIC) mode |= T4_FILTER_IC_VNIC; } if (fconf & F_PORT) mode |= T4_FILTER_PORT; if (fconf & F_FCOE) mode |= T4_FILTER_FCoE; return (mode); } static uint32_t mode_to_fconf(uint32_t mode) { uint32_t fconf = 0; if (mode & T4_FILTER_IP_FRAGMENT) fconf |= F_FRAGMENTATION; if (mode & T4_FILTER_MPS_HIT_TYPE) fconf |= F_MPSHITTYPE; if (mode & T4_FILTER_MAC_IDX) fconf |= F_MACMATCH; if (mode & T4_FILTER_ETH_TYPE) fconf |= F_ETHERTYPE; if (mode & T4_FILTER_IP_PROTO) fconf |= F_PROTOCOL; if (mode & T4_FILTER_IP_TOS) fconf |= F_TOS; if (mode & T4_FILTER_VLAN) fconf |= F_VLAN; if (mode & T4_FILTER_VNIC) fconf |= F_VNIC_ID; if (mode & T4_FILTER_PORT) fconf |= F_PORT; if (mode & T4_FILTER_FCoE) fconf |= F_FCOE; return (fconf); } static uint32_t mode_to_iconf(uint32_t mode) { if (mode & T4_FILTER_IC_VNIC) return (F_VNIC); return (0); } static int check_fspec_against_fconf_iconf(struct adapter *sc, struct t4_filter_specification *fs) { struct tp_params *tpp = &sc->params.tp; uint32_t fconf = 0; if (fs->val.frag || fs->mask.frag) fconf |= F_FRAGMENTATION; if (fs->val.matchtype || fs->mask.matchtype) fconf |= F_MPSHITTYPE; if (fs->val.macidx || fs->mask.macidx) fconf |= F_MACMATCH; if (fs->val.ethtype || fs->mask.ethtype) fconf |= F_ETHERTYPE; if (fs->val.proto || fs->mask.proto) fconf |= F_PROTOCOL; if (fs->val.tos || fs->mask.tos) fconf |= F_TOS; if (fs->val.vlan_vld || fs->mask.vlan_vld) fconf |= F_VLAN; if (fs->val.ovlan_vld || fs->mask.ovlan_vld) { fconf |= F_VNIC_ID; if (tpp->ingress_config & F_VNIC) return (EINVAL); } if (fs->val.pfvf_vld || fs->mask.pfvf_vld) { fconf |= F_VNIC_ID; if ((tpp->ingress_config & F_VNIC) == 0) return (EINVAL); } if (fs->val.iport || fs->mask.iport) fconf |= F_PORT; if (fs->val.fcoe || fs->mask.fcoe) fconf |= F_FCOE; if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map) return (E2BIG); return (0); } static int get_filter_mode(struct adapter *sc, uint32_t *mode) { struct tp_params *tpp = &sc->params.tp; /* * We trust the cached values of the relevant TP registers. This means * things work reliably only if writes to those registers are always via * t4_set_filter_mode. */ *mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config); return (0); } static int set_filter_mode(struct adapter *sc, uint32_t mode) { struct tp_params *tpp = &sc->params.tp; uint32_t fconf, iconf; int rc; iconf = mode_to_iconf(mode); if ((iconf ^ tpp->ingress_config) & F_VNIC) { /* * For now we just complain if A_TP_INGRESS_CONFIG is not * already set to the correct value for the requested filter * mode. It's not clear if it's safe to write to this register * on the fly. (And we trust the cached value of the register). */ return (EBUSY); } fconf = mode_to_fconf(mode); rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4setfm"); if (rc) return (rc); if (sc->tids.ftids_in_use > 0) { rc = EBUSY; goto done; } #ifdef TCP_OFFLOAD if (uld_active(sc, ULD_TOM)) { rc = EBUSY; goto done; } #endif - rc = -t4_set_filter_mode(sc, fconf); + rc = -t4_set_filter_mode(sc, fconf, true); done: end_synchronized_op(sc, LOCK_HELD); return (rc); } static inline uint64_t get_filter_hits(struct adapter *sc, uint32_t fid) { uint32_t tcb_addr; tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + (fid + sc->tids.ftid_base) * TCB_SIZE; if (is_t4(sc)) { uint64_t hits; read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8); return (be64toh(hits)); } else { uint32_t hits; read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4); return (be32toh(hits)); } } static int get_filter(struct adapter *sc, struct t4_filter *t) { int i, rc, nfilters = sc->tids.nftids; struct filter_entry *f; rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4getf"); if (rc) return (rc); if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || t->idx >= nfilters) { t->idx = 0xffffffff; goto done; } f = &sc->tids.ftid_tab[t->idx]; for (i = t->idx; i < nfilters; i++, f++) { if (f->valid) { t->idx = i; t->l2tidx = f->l2t ? f->l2t->idx : 0; t->smtidx = f->smtidx; if (f->fs.hitcnts) t->hits = get_filter_hits(sc, t->idx); else t->hits = UINT64_MAX; t->fs = f->fs; goto done; } } t->idx = 0xffffffff; done: end_synchronized_op(sc, LOCK_HELD); return (0); } static int set_filter(struct adapter *sc, struct t4_filter *t) { unsigned int nfilters, nports; struct filter_entry *f; int i, rc; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); if (rc) return (rc); nfilters = sc->tids.nftids; nports = sc->params.nports; if (nfilters == 0) { rc = ENOTSUP; goto done; } if (t->idx >= nfilters) { rc = EINVAL; goto done; } /* Validate against the global filter mode and ingress config */ rc = check_fspec_against_fconf_iconf(sc, &t->fs); if (rc != 0) goto done; if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { rc = EINVAL; goto done; } if (t->fs.val.iport >= nports) { rc = EINVAL; goto done; } /* Can't specify an iq if not steering to it */ if (!t->fs.dirsteer && t->fs.iq) { rc = EINVAL; goto done; } /* IPv6 filter idx must be 4 aligned */ if (t->fs.type == 1 && ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { rc = EINVAL; goto done; } if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_full_init(sc)) != 0)) goto done; if (sc->tids.ftid_tab == NULL) { KASSERT(sc->tids.ftids_in_use == 0, ("%s: no memory allocated but filters_in_use > 0", __func__)); sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * nfilters, M_CXGBE, M_NOWAIT | M_ZERO); if (sc->tids.ftid_tab == NULL) { rc = ENOMEM; goto done; } mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); } for (i = 0; i < 4; i++) { f = &sc->tids.ftid_tab[t->idx + i]; if (f->pending || f->valid) { rc = EBUSY; goto done; } if (f->locked) { rc = EPERM; goto done; } if (t->fs.type == 0) break; } f = &sc->tids.ftid_tab[t->idx]; f->fs = t->fs; rc = set_filter_wr(sc, t->idx); done: end_synchronized_op(sc, 0); if (rc == 0) { mtx_lock(&sc->tids.ftid_lock); for (;;) { if (f->pending == 0) { rc = f->valid ? 0 : EIO; break; } if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, PCATCH, "t4setfw", 0)) { rc = EINPROGRESS; break; } } mtx_unlock(&sc->tids.ftid_lock); } return (rc); } static int del_filter(struct adapter *sc, struct t4_filter *t) { unsigned int nfilters; struct filter_entry *f; int rc; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); if (rc) return (rc); nfilters = sc->tids.nftids; if (nfilters == 0) { rc = ENOTSUP; goto done; } if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || t->idx >= nfilters) { rc = EINVAL; goto done; } if (!(sc->flags & FULL_INIT_DONE)) { rc = EAGAIN; goto done; } f = &sc->tids.ftid_tab[t->idx]; if (f->pending) { rc = EBUSY; goto done; } if (f->locked) { rc = EPERM; goto done; } if (f->valid) { t->fs = f->fs; /* extra info for the caller */ rc = del_filter_wr(sc, t->idx); } done: end_synchronized_op(sc, 0); if (rc == 0) { mtx_lock(&sc->tids.ftid_lock); for (;;) { if (f->pending == 0) { rc = f->valid ? EIO : 0; break; } if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, PCATCH, "t4delfw", 0)) { rc = EINPROGRESS; break; } } mtx_unlock(&sc->tids.ftid_lock); } return (rc); } static void clear_filter(struct filter_entry *f) { if (f->l2t) t4_l2t_release(f->l2t); bzero(f, sizeof (*f)); } static int set_filter_wr(struct adapter *sc, int fidx) { struct filter_entry *f = &sc->tids.ftid_tab[fidx]; struct fw_filter_wr *fwr; unsigned int ftid, vnic_vld, vnic_vld_mask; struct wrq_cookie cookie; ASSERT_SYNCHRONIZED_OP(sc); if (f->fs.newdmac || f->fs.newvlan) { /* This filter needs an L2T entry; allocate one. */ f->l2t = t4_l2t_alloc_switching(sc->l2t); if (f->l2t == NULL) return (EAGAIN); if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, f->fs.dmac)) { t4_l2t_release(f->l2t); f->l2t = NULL; return (ENOMEM); } } /* Already validated against fconf, iconf */ MPASS((f->fs.val.pfvf_vld & f->fs.val.ovlan_vld) == 0); MPASS((f->fs.mask.pfvf_vld & f->fs.mask.ovlan_vld) == 0); if (f->fs.val.pfvf_vld || f->fs.val.ovlan_vld) vnic_vld = 1; else vnic_vld = 0; if (f->fs.mask.pfvf_vld || f->fs.mask.ovlan_vld) vnic_vld_mask = 1; else vnic_vld_mask = 0; ftid = sc->tids.ftid_base + fidx; fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); if (fwr == NULL) return (ENOMEM); bzero(fwr, sizeof(*fwr)); fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); fwr->tid_to_iq = htobe32(V_FW_FILTER_WR_TID(ftid) | V_FW_FILTER_WR_RQTYPE(f->fs.type) | V_FW_FILTER_WR_NOREPLY(0) | V_FW_FILTER_WR_IQ(f->fs.iq)); fwr->del_filter_to_l2tix = htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | V_FW_FILTER_WR_DMAC(f->fs.newdmac) | V_FW_FILTER_WR_SMAC(f->fs.newsmac) | V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) | V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || f->fs.newvlan == VLAN_REWRITE) | V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | V_FW_FILTER_WR_TXCHAN(f->fs.eport) | V_FW_FILTER_WR_PRIO(f->fs.prio) | V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); fwr->ethtype = htobe16(f->fs.val.ethtype); fwr->ethtypem = htobe16(f->fs.mask.ethtype); fwr->frag_to_ovlan_vldm = (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) | V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask)); fwr->smac_sel = 0; fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); fwr->maci_to_matchtypem = htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | V_FW_FILTER_WR_PORT(f->fs.val.iport) | V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); fwr->ptcl = f->fs.val.proto; fwr->ptclm = f->fs.mask.proto; fwr->ttyp = f->fs.val.tos; fwr->ttypm = f->fs.mask.tos; fwr->ivlan = htobe16(f->fs.val.vlan); fwr->ivlanm = htobe16(f->fs.mask.vlan); fwr->ovlan = htobe16(f->fs.val.vnic); fwr->ovlanm = htobe16(f->fs.mask.vnic); bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); fwr->lp = htobe16(f->fs.val.dport); fwr->lpm = htobe16(f->fs.mask.dport); fwr->fp = htobe16(f->fs.val.sport); fwr->fpm = htobe16(f->fs.mask.sport); if (f->fs.newsmac) bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); f->pending = 1; sc->tids.ftids_in_use++; commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); return (0); } static int del_filter_wr(struct adapter *sc, int fidx) { struct filter_entry *f = &sc->tids.ftid_tab[fidx]; struct fw_filter_wr *fwr; unsigned int ftid; struct wrq_cookie cookie; ftid = sc->tids.ftid_base + fidx; fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); if (fwr == NULL) return (ENOMEM); bzero(fwr, sizeof (*fwr)); t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); f->pending = 1; commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); return (0); } int t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); unsigned int idx = GET_TID(rpl); unsigned int rc; struct filter_entry *f; KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); MPASS(iq == &sc->sge.fwq); MPASS(is_ftid(sc, idx)); idx -= sc->tids.ftid_base; f = &sc->tids.ftid_tab[idx]; rc = G_COOKIE(rpl->cookie); mtx_lock(&sc->tids.ftid_lock); if (rc == FW_FILTER_WR_FLT_ADDED) { KASSERT(f->pending, ("%s: filter[%u] isn't pending.", __func__, idx)); f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; f->pending = 0; /* asynchronous setup completed */ f->valid = 1; } else { if (rc != FW_FILTER_WR_FLT_DELETED) { /* Add or delete failed, display an error */ log(LOG_ERR, "filter %u setup failed with error %u\n", idx, rc); } clear_filter(f); sc->tids.ftids_in_use--; } wakeup(&sc->tids.ftid_tab); mtx_unlock(&sc->tids.ftid_lock); return (0); } static int set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { MPASS(iq->set_tcb_rpl != NULL); return (iq->set_tcb_rpl(iq, rss, m)); } static int l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { MPASS(iq->l2t_write_rpl != NULL); return (iq->l2t_write_rpl(iq, rss, m)); } static int get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) { int rc; if (cntxt->cid > M_CTXTQID) return (EINVAL); if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) return (EINVAL); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); if (rc) return (rc); if (sc->flags & FW_OK) { rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); if (rc == 0) goto done; } /* * Read via firmware failed or wasn't even attempted. Read directly via * the backdoor. */ rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); done: end_synchronized_op(sc, 0); return (rc); } static int load_fw(struct adapter *sc, struct t4_data *fw) { int rc; uint8_t *fw_data; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); if (rc) return (rc); if (sc->flags & FULL_INIT_DONE) { rc = EBUSY; goto done; } fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); if (fw_data == NULL) { rc = ENOMEM; goto done; } rc = copyin(fw->data, fw_data, fw->len); if (rc == 0) rc = -t4_load_fw(sc, fw_data, fw->len); free(fw_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_cfg(struct adapter *sc, struct t4_data *cfg) { int rc; uint8_t *cfg_data = NULL; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); if (rc) return (rc); if (cfg->len == 0) { /* clear */ rc = -t4_load_cfg(sc, NULL, 0); goto done; } cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK); if (cfg_data == NULL) { rc = ENOMEM; goto done; } rc = copyin(cfg->data, cfg_data, cfg->len); if (rc == 0) rc = -t4_load_cfg(sc, cfg_data, cfg->len); free(cfg_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_boot(struct adapter *sc, struct t4_bootrom *br) { int rc; uint8_t *br_data = NULL; u_int offset; if (br->len > 1024 * 1024) return (EFBIG); if (br->pf_offset == 0) { /* pfidx */ if (br->pfidx_addr > 7) return (EINVAL); offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr, A_PCIE_PF_EXPROM_OFST))); } else if (br->pf_offset == 1) { /* offset */ offset = G_OFFSET(br->pfidx_addr); } else { return (EINVAL); } rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr"); if (rc) return (rc); if (br->len == 0) { /* clear */ rc = -t4_load_boot(sc, NULL, offset, 0); goto done; } br_data = malloc(br->len, M_CXGBE, M_WAITOK); if (br_data == NULL) { rc = ENOMEM; goto done; } rc = copyin(br->data, br_data, br->len); if (rc == 0) rc = -t4_load_boot(sc, br_data, offset, br->len); free(br_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_bootcfg(struct adapter *sc, struct t4_data *bc) { int rc; uint8_t *bc_data = NULL; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); if (rc) return (rc); if (bc->len == 0) { /* clear */ rc = -t4_load_bootcfg(sc, NULL, 0); goto done; } bc_data = malloc(bc->len, M_CXGBE, M_WAITOK); if (bc_data == NULL) { rc = ENOMEM; goto done; } rc = copyin(bc->data, bc_data, bc->len); if (rc == 0) rc = -t4_load_bootcfg(sc, bc_data, bc->len); free(bc_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } #define MAX_READ_BUF_SIZE (128 * 1024) static int read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) { uint32_t addr, remaining, n; uint32_t *buf; int rc; uint8_t *dst; rc = validate_mem_range(sc, mr->addr, mr->len); if (rc != 0) return (rc); buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK); addr = mr->addr; remaining = mr->len; dst = (void *)mr->data; while (remaining) { n = min(remaining, MAX_READ_BUF_SIZE); read_via_memwin(sc, 2, addr, buf, n); rc = copyout(buf, dst, n); if (rc != 0) break; dst += n; remaining -= n; addr += n; } free(buf, M_CXGBE); return (rc); } #undef MAX_READ_BUF_SIZE static int read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) { int rc; if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) return (EINVAL); if (i2cd->len > sizeof(i2cd->data)) return (EFBIG); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); if (rc) return (rc); rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, i2cd->offset, i2cd->len, &i2cd->data[0]); end_synchronized_op(sc, 0); return (rc); } int t4_os_find_pci_capability(struct adapter *sc, int cap) { int i; return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); } int t4_os_pci_save_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_save(dev, dinfo, 0); return (0); } int t4_os_pci_restore_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_restore(dev, dinfo); return (0); } void t4_os_portmod_changed(struct port_info *pi, int old_ptype, int old_mtype, struct link_config *old_lc) { struct vi_info *vi; struct ifnet *ifp; int v; static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" }; PORT_LOCK(pi); for_each_vi(pi, v, vi) { build_medialist(pi, &vi->media); } PORT_UNLOCK(pi); if (begin_synchronized_op(pi->adapter, vi, HOLD_LOCK, "t4mod") == 0) { init_l1cfg(pi); end_synchronized_op(pi->adapter, LOCK_HELD); } ifp = pi->vi[0].ifp; if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) if_printf(ifp, "transceiver unplugged.\n"); else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) if_printf(ifp, "unknown transceiver inserted.\n"); else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) if_printf(ifp, "unsupported transceiver inserted.\n"); else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { if_printf(ifp, "%dGbps %s transceiver inserted.\n", port_top_speed(pi), mod_str[pi->mod_type]); } else { if_printf(ifp, "transceiver (type %d) inserted.\n", pi->mod_type); } } void t4_os_link_changed(struct port_info *pi, struct link_config *old_lc) { struct vi_info *vi; struct ifnet *ifp; struct link_config *lc; int v; for_each_vi(pi, v, vi) { ifp = vi->ifp; if (ifp == NULL) continue; lc = &pi->link_cfg; if (lc->link_ok) { ifp->if_baudrate = IF_Mbps(lc->speed); if_link_state_change(ifp, LINK_STATE_UP); } else { if_link_state_change(ifp, LINK_STATE_DOWN); } } } void t4_iterate(void (*func)(struct adapter *, void *), void *arg) { struct adapter *sc; sx_slock(&t4_list_lock); SLIST_FOREACH(sc, &t4_list, link) { /* * func should not make any assumptions about what state sc is * in - the only guarantee is that sc->sc_lock is a valid lock. */ func(sc, arg); } sx_sunlock(&t4_list_lock); } static int t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { int rc; struct adapter *sc = dev->si_drv1; rc = priv_check(td, PRIV_DRIVER); if (rc != 0) return (rc); switch (cmd) { case CHELSIO_T4_GETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); if (edata->size == 4) edata->val = t4_read_reg(sc, edata->addr); else if (edata->size == 8) edata->val = t4_read_reg64(sc, edata->addr); else return (EINVAL); break; } case CHELSIO_T4_SETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); if (edata->size == 4) { if (edata->val & 0xffffffff00000000) return (EINVAL); t4_write_reg(sc, edata->addr, (uint32_t) edata->val); } else if (edata->size == 8) t4_write_reg64(sc, edata->addr, edata->val); else return (EINVAL); break; } case CHELSIO_T4_REGDUMP: { struct t4_regdump *regs = (struct t4_regdump *)data; int reglen = t4_get_regs_len(sc); uint8_t *buf; if (regs->len < reglen) { regs->len = reglen; /* hint to the caller */ return (ENOBUFS); } regs->len = reglen; buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); get_regs(sc, regs, buf); rc = copyout(buf, regs->data, reglen); free(buf, M_CXGBE); break; } case CHELSIO_T4_GET_FILTER_MODE: rc = get_filter_mode(sc, (uint32_t *)data); break; case CHELSIO_T4_SET_FILTER_MODE: rc = set_filter_mode(sc, *(uint32_t *)data); break; case CHELSIO_T4_GET_FILTER: rc = get_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_SET_FILTER: rc = set_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_DEL_FILTER: rc = del_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_GET_SGE_CONTEXT: rc = get_sge_context(sc, (struct t4_sge_context *)data); break; case CHELSIO_T4_LOAD_FW: rc = load_fw(sc, (struct t4_data *)data); break; case CHELSIO_T4_GET_MEM: rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); break; case CHELSIO_T4_GET_I2C: rc = read_i2c(sc, (struct t4_i2c_data *)data); break; case CHELSIO_T4_CLEAR_STATS: { int i, v; u_int port_id = *(uint32_t *)data; struct port_info *pi; struct vi_info *vi; if (port_id >= sc->params.nports) return (EINVAL); pi = sc->port[port_id]; if (pi == NULL) return (EIO); /* MAC stats */ t4_clr_port_stats(sc, pi->tx_chan); pi->tx_parse_error = 0; mtx_lock(&sc->reg_lock); for_each_vi(pi, v, vi) { if (vi->flags & VI_INIT_DONE) t4_clr_vi_stats(sc, vi->viid); } mtx_unlock(&sc->reg_lock); /* * Since this command accepts a port, clear stats for * all VIs on this port. */ for_each_vi(pi, v, vi) { if (vi->flags & VI_INIT_DONE) { struct sge_rxq *rxq; struct sge_txq *txq; struct sge_wrq *wrq; for_each_rxq(vi, i, rxq) { #if defined(INET) || defined(INET6) rxq->lro.lro_queued = 0; rxq->lro.lro_flushed = 0; #endif rxq->rxcsum = 0; rxq->vlan_extraction = 0; } for_each_txq(vi, i, txq) { txq->txcsum = 0; txq->tso_wrs = 0; txq->vlan_insertion = 0; txq->imm_wrs = 0; txq->sgl_wrs = 0; txq->txpkt_wrs = 0; txq->txpkts0_wrs = 0; txq->txpkts1_wrs = 0; txq->txpkts0_pkts = 0; txq->txpkts1_pkts = 0; mp_ring_reset_stats(txq->r); } #ifdef TCP_OFFLOAD /* nothing to clear for each ofld_rxq */ for_each_ofld_txq(vi, i, wrq) { wrq->tx_wrs_direct = 0; wrq->tx_wrs_copied = 0; } #endif if (IS_MAIN_VI(vi)) { wrq = &sc->sge.ctrlq[pi->port_id]; wrq->tx_wrs_direct = 0; wrq->tx_wrs_copied = 0; } } } break; } case CHELSIO_T4_SCHED_CLASS: rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); break; case CHELSIO_T4_SCHED_QUEUE: rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); break; case CHELSIO_T4_GET_TRACER: rc = t4_get_tracer(sc, (struct t4_tracer *)data); break; case CHELSIO_T4_SET_TRACER: rc = t4_set_tracer(sc, (struct t4_tracer *)data); break; case CHELSIO_T4_LOAD_CFG: rc = load_cfg(sc, (struct t4_data *)data); break; case CHELSIO_T4_LOAD_BOOT: rc = load_boot(sc, (struct t4_bootrom *)data); break; case CHELSIO_T4_LOAD_BOOTCFG: rc = load_bootcfg(sc, (struct t4_data *)data); break; default: rc = ENOTTY; } return (rc); } void t4_db_full(struct adapter *sc) { CXGBE_UNIMPLEMENTED(__func__); } void t4_db_dropped(struct adapter *sc) { CXGBE_UNIMPLEMENTED(__func__); } #ifdef TCP_OFFLOAD static int toe_capability(struct vi_info *vi, int enable) { int rc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; ASSERT_SYNCHRONIZED_OP(sc); if (!is_offload(sc)) return (ENODEV); if (enable) { if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { /* TOE is already enabled. */ return (0); } /* * We need the port's queues around so that we're able to send * and receive CPLs to/from the TOE even if the ifnet for this * port has never been UP'd administratively. */ if (!(vi->flags & VI_INIT_DONE)) { rc = vi_full_init(vi); if (rc) return (rc); } if (!(pi->vi[0].flags & VI_INIT_DONE)) { rc = vi_full_init(&pi->vi[0]); if (rc) return (rc); } if (isset(&sc->offload_map, pi->port_id)) { /* TOE is enabled on another VI of this port. */ pi->uld_vis++; return (0); } if (!uld_active(sc, ULD_TOM)) { rc = t4_activate_uld(sc, ULD_TOM); if (rc == EAGAIN) { log(LOG_WARNING, "You must kldload t4_tom.ko before trying " "to enable TOE on a cxgbe interface.\n"); } if (rc != 0) return (rc); KASSERT(sc->tom_softc != NULL, ("%s: TOM activated but softc NULL", __func__)); KASSERT(uld_active(sc, ULD_TOM), ("%s: TOM activated but flag not set", __func__)); } /* Activate iWARP and iSCSI too, if the modules are loaded. */ if (!uld_active(sc, ULD_IWARP)) (void) t4_activate_uld(sc, ULD_IWARP); if (!uld_active(sc, ULD_ISCSI)) (void) t4_activate_uld(sc, ULD_ISCSI); pi->uld_vis++; setbit(&sc->offload_map, pi->port_id); } else { pi->uld_vis--; if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) return (0); KASSERT(uld_active(sc, ULD_TOM), ("%s: TOM never initialized?", __func__)); clrbit(&sc->offload_map, pi->port_id); } return (0); } /* * Add an upper layer driver to the global list. */ int t4_register_uld(struct uld_info *ui) { int rc = 0; struct uld_info *u; sx_xlock(&t4_uld_list_lock); SLIST_FOREACH(u, &t4_uld_list, link) { if (u->uld_id == ui->uld_id) { rc = EEXIST; goto done; } } SLIST_INSERT_HEAD(&t4_uld_list, ui, link); ui->refcount = 0; done: sx_xunlock(&t4_uld_list_lock); return (rc); } int t4_unregister_uld(struct uld_info *ui) { int rc = EINVAL; struct uld_info *u; sx_xlock(&t4_uld_list_lock); SLIST_FOREACH(u, &t4_uld_list, link) { if (u == ui) { if (ui->refcount > 0) { rc = EBUSY; goto done; } SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); rc = 0; goto done; } } done: sx_xunlock(&t4_uld_list_lock); return (rc); } int t4_activate_uld(struct adapter *sc, int id) { int rc; struct uld_info *ui; ASSERT_SYNCHRONIZED_OP(sc); if (id < 0 || id > ULD_MAX) return (EINVAL); rc = EAGAIN; /* kldoad the module with this ULD and try again. */ sx_slock(&t4_uld_list_lock); SLIST_FOREACH(ui, &t4_uld_list, link) { if (ui->uld_id == id) { if (!(sc->flags & FULL_INIT_DONE)) { rc = adapter_full_init(sc); if (rc != 0) break; } rc = ui->activate(sc); if (rc == 0) { setbit(&sc->active_ulds, id); ui->refcount++; } break; } } sx_sunlock(&t4_uld_list_lock); return (rc); } int t4_deactivate_uld(struct adapter *sc, int id) { int rc; struct uld_info *ui; ASSERT_SYNCHRONIZED_OP(sc); if (id < 0 || id > ULD_MAX) return (EINVAL); rc = ENXIO; sx_slock(&t4_uld_list_lock); SLIST_FOREACH(ui, &t4_uld_list, link) { if (ui->uld_id == id) { rc = ui->deactivate(sc); if (rc == 0) { clrbit(&sc->active_ulds, id); ui->refcount--; } break; } } sx_sunlock(&t4_uld_list_lock); return (rc); } int uld_active(struct adapter *sc, int uld_id) { MPASS(uld_id >= 0 && uld_id <= ULD_MAX); return (isset(&sc->active_ulds, uld_id)); } #endif /* * t = ptr to tunable. * nc = number of CPUs. * c = compiled in default for that tunable. */ static void calculate_nqueues(int *t, int nc, const int c) { int nq; if (*t > 0) return; nq = *t < 0 ? -*t : c; *t = min(nc, nq); } /* * Come up with reasonable defaults for some of the tunables, provided they're * not set by the user (in which case we'll use the values as is). */ static void tweak_tunables(void) { int nc = mp_ncpus; /* our snapshot of the number of CPUs */ if (t4_ntxq10g < 1) { #ifdef RSS t4_ntxq10g = rss_getnumbuckets(); #else calculate_nqueues(&t4_ntxq10g, nc, NTXQ_10G); #endif } if (t4_ntxq1g < 1) { #ifdef RSS /* XXX: way too many for 1GbE? */ t4_ntxq1g = rss_getnumbuckets(); #else calculate_nqueues(&t4_ntxq1g, nc, NTXQ_1G); #endif } calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI); if (t4_nrxq10g < 1) { #ifdef RSS t4_nrxq10g = rss_getnumbuckets(); #else calculate_nqueues(&t4_nrxq10g, nc, NRXQ_10G); #endif } if (t4_nrxq1g < 1) { #ifdef RSS /* XXX: way too many for 1GbE? */ t4_nrxq1g = rss_getnumbuckets(); #else calculate_nqueues(&t4_nrxq1g, nc, NRXQ_1G); #endif } calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI); #ifdef TCP_OFFLOAD calculate_nqueues(&t4_nofldtxq10g, nc, NOFLDTXQ_10G); calculate_nqueues(&t4_nofldtxq1g, nc, NOFLDTXQ_1G); calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); calculate_nqueues(&t4_nofldrxq10g, nc, NOFLDRXQ_10G); calculate_nqueues(&t4_nofldrxq1g, nc, NOFLDRXQ_1G); calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); if (t4_toecaps_allowed == -1) t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; if (t4_rdmacaps_allowed == -1) { t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | FW_CAPS_CONFIG_RDMA_RDMAC; } if (t4_iscsicaps_allowed == -1) { t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | FW_CAPS_CONFIG_ISCSI_TARGET_PDU | FW_CAPS_CONFIG_ISCSI_T10DIF; } #else if (t4_toecaps_allowed == -1) t4_toecaps_allowed = 0; if (t4_rdmacaps_allowed == -1) t4_rdmacaps_allowed = 0; if (t4_iscsicaps_allowed == -1) t4_iscsicaps_allowed = 0; #endif #ifdef DEV_NETMAP calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI); calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI); #endif if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) t4_tmr_idx_10g = TMR_IDX_10G; if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) t4_pktc_idx_10g = PKTC_IDX_10G; if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) t4_tmr_idx_1g = TMR_IDX_1G; if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) t4_pktc_idx_1g = PKTC_IDX_1G; if (t4_qsize_txq < 128) t4_qsize_txq = 128; if (t4_qsize_rxq < 128) t4_qsize_rxq = 128; while (t4_qsize_rxq & 7) t4_qsize_rxq++; t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; } #ifdef DDB static void t4_dump_tcb(struct adapter *sc, int tid) { uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos; reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2); save = t4_read_reg(sc, reg); base = sc->memwin[2].mw_base; /* Dump TCB for the tid */ tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE); tcb_addr += tid * TCB_SIZE; if (is_t4(sc)) { pf = 0; win_pos = tcb_addr & ~0xf; /* start must be 16B aligned */ } else { pf = V_PFNUM(sc->pf); win_pos = tcb_addr & ~0x7f; /* start must be 128B aligned */ } t4_write_reg(sc, reg, win_pos | pf); t4_read_reg(sc, reg); off = tcb_addr - win_pos; for (i = 0; i < 4; i++) { uint32_t buf[8]; for (j = 0; j < 8; j++, off += 4) buf[j] = htonl(t4_read_reg(sc, base + off)); db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7]); } t4_write_reg(sc, reg, save); t4_read_reg(sc, reg); } static void t4_dump_devlog(struct adapter *sc) { struct devlog_params *dparams = &sc->params.devlog; struct fw_devlog_e e; int i, first, j, m, nentries, rc; uint64_t ftstamp = UINT64_MAX; if (dparams->start == 0) { db_printf("devlog params not valid\n"); return; } nentries = dparams->size / sizeof(struct fw_devlog_e); m = fwmtype_to_hwmtype(dparams->memtype); /* Find the first entry. */ first = -1; for (i = 0; i < nentries && !db_pager_quit; i++) { rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), sizeof(e), (void *)&e); if (rc != 0) break; if (e.timestamp == 0) break; e.timestamp = be64toh(e.timestamp); if (e.timestamp < ftstamp) { ftstamp = e.timestamp; first = i; } } if (first == -1) return; i = first; do { rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), sizeof(e), (void *)&e); if (rc != 0) return; if (e.timestamp == 0) return; e.timestamp = be64toh(e.timestamp); e.seqno = be32toh(e.seqno); for (j = 0; j < 8; j++) e.params[j] = be32toh(e.params[j]); db_printf("%10d %15ju %8s %8s ", e.seqno, e.timestamp, (e.level < nitems(devlog_level_strings) ? devlog_level_strings[e.level] : "UNKNOWN"), (e.facility < nitems(devlog_facility_strings) ? devlog_facility_strings[e.facility] : "UNKNOWN")); db_printf(e.fmt, e.params[0], e.params[1], e.params[2], e.params[3], e.params[4], e.params[5], e.params[6], e.params[7]); if (++i == nentries) i = 0; } while (i != first && !db_pager_quit); } static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table); _DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table); DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL) { device_t dev; int t; bool valid; valid = false; t = db_read_token(); if (t == tIDENT) { dev = device_lookup_by_name(db_tok_string); valid = true; } db_skip_to_eol(); if (!valid) { db_printf("usage: show t4 devlog \n"); return; } if (dev == NULL) { db_printf("device not found\n"); return; } t4_dump_devlog(device_get_softc(dev)); } DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL) { device_t dev; int radix, tid, t; bool valid; valid = false; radix = db_radix; db_radix = 10; t = db_read_token(); if (t == tIDENT) { dev = device_lookup_by_name(db_tok_string); t = db_read_token(); if (t == tNUMBER) { tid = db_tok_number; valid = true; } } db_radix = radix; db_skip_to_eol(); if (!valid) { db_printf("usage: show t4 tcb \n"); return; } if (dev == NULL) { db_printf("device not found\n"); return; } if (tid < 0) { db_printf("invalid tid\n"); return; } t4_dump_tcb(device_get_softc(dev), tid); } #endif static struct sx mlu; /* mod load unload */ SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); static int mod_event(module_t mod, int cmd, void *arg) { int rc = 0; static int loaded = 0; switch (cmd) { case MOD_LOAD: sx_xlock(&mlu); if (loaded++ == 0) { t4_sge_modload(); t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl); t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl); t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt); t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt); sx_init(&t4_list_lock, "T4/T5 adapters"); SLIST_INIT(&t4_list); #ifdef TCP_OFFLOAD sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); SLIST_INIT(&t4_uld_list); #endif t4_tracer_modload(); tweak_tunables(); } sx_xunlock(&mlu); break; case MOD_UNLOAD: sx_xlock(&mlu); if (--loaded == 0) { int tries; sx_slock(&t4_list_lock); if (!SLIST_EMPTY(&t4_list)) { rc = EBUSY; sx_sunlock(&t4_list_lock); goto done_unload; } #ifdef TCP_OFFLOAD sx_slock(&t4_uld_list_lock); if (!SLIST_EMPTY(&t4_uld_list)) { rc = EBUSY; sx_sunlock(&t4_uld_list_lock); sx_sunlock(&t4_list_lock); goto done_unload; } #endif tries = 0; while (tries++ < 5 && t4_sge_extfree_refs() != 0) { uprintf("%ju clusters with custom free routine " "still is use.\n", t4_sge_extfree_refs()); pause("t4unload", 2 * hz); } #ifdef TCP_OFFLOAD sx_sunlock(&t4_uld_list_lock); #endif sx_sunlock(&t4_list_lock); if (t4_sge_extfree_refs() == 0) { t4_tracer_modunload(); #ifdef TCP_OFFLOAD sx_destroy(&t4_uld_list_lock); #endif sx_destroy(&t4_list_lock); t4_sge_modunload(); loaded = 0; } else { rc = EBUSY; loaded++; /* undo earlier decrement */ } } done_unload: sx_xunlock(&mlu); break; } return (rc); } static devclass_t t4_devclass, t5_devclass, t6_devclass; static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass; static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass; DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); MODULE_VERSION(t4nex, 1); MODULE_DEPEND(t4nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t4nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); MODULE_VERSION(t5nex, 1); MODULE_DEPEND(t5nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t5nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0); MODULE_VERSION(t6nex, 1); MODULE_DEPEND(t6nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t6nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); MODULE_VERSION(cxgbe, 1); DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); MODULE_VERSION(cxl, 1); DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0); MODULE_VERSION(cc, 1); DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); MODULE_VERSION(vcxgbe, 1); DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); MODULE_VERSION(vcxl, 1); DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0); MODULE_VERSION(vcc, 1); Index: projects/runtime-coverage/sys/dev/cxgbe/t4_sge.c =================================================================== --- projects/runtime-coverage/sys/dev/cxgbe/t4_sge.c (revision 321584) +++ projects/runtime-coverage/sys/dev/cxgbe/t4_sge.c (revision 321585) @@ -1,5344 +1,5344 @@ /*- * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_NETMAP #include #include #include #include #include #endif #include "common/common.h" #include "common/t4_regs.h" #include "common/t4_regs_values.h" #include "common/t4_msg.h" #include "t4_l2t.h" #include "t4_mp_ring.h" #ifdef T4_PKT_TIMESTAMP #define RX_COPY_THRESHOLD (MINCLSIZE - 8) #else #define RX_COPY_THRESHOLD MINCLSIZE #endif /* * Ethernet frames are DMA'd at this byte offset into the freelist buffer. * 0-7 are valid values. */ static int fl_pktshift = 2; TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift); /* * Pad ethernet payload up to this boundary. * -1: driver should figure out a good value. * 0: disable padding. * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. */ int fl_pad = -1; TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad); /* * Status page length. * -1: driver should figure out a good value. * 64 or 128 are the only other valid values. */ static int spg_len = -1; TUNABLE_INT("hw.cxgbe.spg_len", &spg_len); /* * Congestion drops. * -1: no congestion feedback (not recommended). * 0: backpressure the channel instead of dropping packets right away. * 1: no backpressure, drop packets for the congested queue immediately. */ static int cong_drop = 0; TUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop); /* * Deliver multiple frames in the same free list buffer if they fit. * -1: let the driver decide whether to enable buffer packing or not. * 0: disable buffer packing. * 1: enable buffer packing. */ static int buffer_packing = -1; TUNABLE_INT("hw.cxgbe.buffer_packing", &buffer_packing); /* * Start next frame in a packed buffer at this boundary. * -1: driver should figure out a good value. * T4: driver will ignore this and use the same value as fl_pad above. * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. */ static int fl_pack = -1; TUNABLE_INT("hw.cxgbe.fl_pack", &fl_pack); /* * Allow the driver to create mbuf(s) in a cluster allocated for rx. * 0: never; always allocate mbufs from the zone_mbuf UMA zone. * 1: ok to create mbuf(s) within a cluster if there is room. */ static int allow_mbufs_in_cluster = 1; TUNABLE_INT("hw.cxgbe.allow_mbufs_in_cluster", &allow_mbufs_in_cluster); /* * Largest rx cluster size that the driver is allowed to allocate. */ static int largest_rx_cluster = MJUM16BYTES; TUNABLE_INT("hw.cxgbe.largest_rx_cluster", &largest_rx_cluster); /* * Size of cluster allocation that's most likely to succeed. The driver will * fall back to this size if it fails to allocate clusters larger than this. */ static int safest_rx_cluster = PAGE_SIZE; TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster); /* * The interrupt holdoff timers are multiplied by this value on T6+. * 1 and 3-17 (both inclusive) are legal values. */ static int tscale = 1; TUNABLE_INT("hw.cxgbe.tscale", &tscale); /* * Number of LRO entries in the lro_ctrl structure per rx queue. */ static int lro_entries = TCP_LRO_ENTRIES; TUNABLE_INT("hw.cxgbe.lro_entries", &lro_entries); /* * This enables presorting of frames before they're fed into tcp_lro_rx. */ static int lro_mbufs = 0; TUNABLE_INT("hw.cxgbe.lro_mbufs", &lro_mbufs); struct txpkts { u_int wr_type; /* type 0 or type 1 */ u_int npkt; /* # of packets in this work request */ u_int plen; /* total payload (sum of all packets) */ u_int len16; /* # of 16B pieces used by this work request */ }; /* A packet's SGL. This + m_pkthdr has all info needed for tx */ struct sgl { struct sglist sg; struct sglist_seg seg[TX_SGL_SEGS]; }; static int service_iq(struct sge_iq *, int); static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *); static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, uint16_t, char *); static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, bus_addr_t *, void **); static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, void *); static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, int, int); static int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *); static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_fl *); static int alloc_fwq(struct adapter *); static int free_fwq(struct adapter *); static int alloc_mgmtq(struct adapter *); static int free_mgmtq(struct adapter *); static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, struct sysctl_oid *); static int free_rxq(struct vi_info *, struct sge_rxq *); #ifdef TCP_OFFLOAD static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, struct sysctl_oid *); static int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); #endif #ifdef DEV_NETMAP static int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int, struct sysctl_oid *); static int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); static int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int, struct sysctl_oid *); static int free_nm_txq(struct vi_info *, struct sge_nm_txq *); #endif static int ctrl_eq_alloc(struct adapter *, struct sge_eq *); static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); #ifdef TCP_OFFLOAD static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); #endif static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); static int free_eq(struct adapter *, struct sge_eq *); static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, struct sysctl_oid *); static int free_wrq(struct adapter *, struct sge_wrq *); static int alloc_txq(struct vi_info *, struct sge_txq *, int, struct sysctl_oid *); static int free_txq(struct vi_info *, struct sge_txq *); static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); static inline void ring_fl_db(struct adapter *, struct sge_fl *); static int refill_fl(struct adapter *, struct sge_fl *, int); static void refill_sfl(void *); static int alloc_fl_sdesc(struct sge_fl *); static void free_fl_sdesc(struct adapter *, struct sge_fl *); static void find_best_refill_source(struct adapter *, struct sge_fl *, int); static void find_safe_refill_source(struct adapter *, struct sge_fl *); static void add_fl_to_sfl(struct adapter *, struct sge_fl *); static inline void get_pkt_gl(struct mbuf *, struct sglist *); static inline u_int txpkt_len16(u_int, u_int); static inline u_int txpkt_vm_len16(u_int, u_int); static inline u_int txpkts0_len16(u_int); static inline u_int txpkts1_len16(void); static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *, struct mbuf *, u_int); static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, struct fw_eth_tx_pkt_vm_wr *, struct mbuf *, u_int); static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int); static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int); static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *, struct mbuf *, const struct txpkts *, u_int); static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); static inline uint16_t read_hw_cidx(struct sge_eq *); static inline u_int reclaimable_tx_desc(struct sge_eq *); static inline u_int total_available_tx_desc(struct sge_eq *); static u_int reclaim_tx_descs(struct sge_txq *, u_int); static void tx_reclaim(void *, int); static __be64 get_flit(struct sglist_seg *, int, int); static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, struct mbuf *); static int handle_fw_msg(struct sge_iq *, const struct rss_header *, struct mbuf *); static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); static void wrq_tx_drain(void *, int); static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); static int sysctl_uint16(SYSCTL_HANDLER_ARGS); static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); static int sysctl_tc(SYSCTL_HANDLER_ARGS); static counter_u64_t extfree_refs; static counter_u64_t extfree_rels; an_handler_t t4_an_handler; fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; static int an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) { #ifdef INVARIANTS panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); #else log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", __func__, iq, ctrl); #endif return (EDOOFUS); } int t4_register_an_handler(an_handler_t h) { uintptr_t *loc, new; new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; loc = (uintptr_t *) &t4_an_handler; atomic_store_rel_ptr(loc, new); return (0); } static int fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) { const struct cpl_fw6_msg *cpl = __containerof(rpl, struct cpl_fw6_msg, data[0]); #ifdef INVARIANTS panic("%s: fw_msg type %d", __func__, cpl->type); #else log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); #endif return (EDOOFUS); } int t4_register_fw_msg_handler(int type, fw_msg_handler_t h) { uintptr_t *loc, new; if (type >= nitems(t4_fw_msg_handler)) return (EINVAL); /* * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL * handler dispatch table. Reject any attempt to install a handler for * this subtype. */ if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) return (EINVAL); new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; loc = (uintptr_t *) &t4_fw_msg_handler[type]; atomic_store_rel_ptr(loc, new); return (0); } static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { #ifdef INVARIANTS panic("%s: opcode 0x%02x on iq %p with payload %p", __func__, rss->opcode, iq, m); #else log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", __func__, rss->opcode, iq, m); m_freem(m); #endif return (EDOOFUS); } int t4_register_cpl_handler(int opcode, cpl_handler_t h) { uintptr_t *loc, new; if (opcode >= nitems(t4_cpl_handler)) return (EINVAL); new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; loc = (uintptr_t *) &t4_cpl_handler[opcode]; atomic_store_rel_ptr(loc, new); return (0); } /* * Called on MOD_LOAD. Validates and calculates the SGE tunables. */ void t4_sge_modload(void) { int i; if (fl_pktshift < 0 || fl_pktshift > 7) { printf("Invalid hw.cxgbe.fl_pktshift value (%d)," " using 2 instead.\n", fl_pktshift); fl_pktshift = 2; } if (spg_len != 64 && spg_len != 128) { int len; #if defined(__i386__) || defined(__amd64__) len = cpu_clflush_line_size > 64 ? 128 : 64; #else len = 64; #endif if (spg_len != -1) { printf("Invalid hw.cxgbe.spg_len value (%d)," " using %d instead.\n", spg_len, len); } spg_len = len; } if (cong_drop < -1 || cong_drop > 1) { printf("Invalid hw.cxgbe.cong_drop value (%d)," " using 0 instead.\n", cong_drop); cong_drop = 0; } if (tscale != 1 && (tscale < 3 || tscale > 17)) { printf("Invalid hw.cxgbe.tscale value (%d)," " using 1 instead.\n", tscale); tscale = 1; } extfree_refs = counter_u64_alloc(M_WAITOK); extfree_rels = counter_u64_alloc(M_WAITOK); counter_u64_zero(extfree_refs); counter_u64_zero(extfree_rels); t4_an_handler = an_not_handled; for (i = 0; i < nitems(t4_fw_msg_handler); i++) t4_fw_msg_handler[i] = fw_msg_not_handled; for (i = 0; i < nitems(t4_cpl_handler); i++) t4_cpl_handler[i] = cpl_not_handled; t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); t4_register_cpl_handler(CPL_RX_PKT, t4_eth_rx); t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); } void t4_sge_modunload(void) { counter_u64_free(extfree_refs); counter_u64_free(extfree_rels); } uint64_t t4_sge_extfree_refs(void) { uint64_t refs, rels; rels = counter_u64_fetch(extfree_rels); refs = counter_u64_fetch(extfree_refs); return (refs - rels); } static inline void setup_pad_and_pack_boundaries(struct adapter *sc) { uint32_t v, m; int pad, pack, pad_shift; pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : X_INGPADBOUNDARY_SHIFT; pad = fl_pad; if (fl_pad < (1 << pad_shift) || fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || !powerof2(fl_pad)) { /* * If there is any chance that we might use buffer packing and * the chip is a T4, then pick 64 as the pad/pack boundary. Set * it to the minimum allowed in all other cases. */ pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; /* * For fl_pad = 0 we'll still write a reasonable value to the * register but all the freelists will opt out of padding. * We'll complain here only if the user tried to set it to a * value greater than 0 that was invalid. */ if (fl_pad > 0) { device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" " (%d), using %d instead.\n", fl_pad, pad); } } m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); t4_set_reg_field(sc, A_SGE_CONTROL, m, v); if (is_t4(sc)) { if (fl_pack != -1 && fl_pack != pad) { /* Complain but carry on. */ device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," " using %d instead.\n", fl_pack, pad); } return; } pack = fl_pack; if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || !powerof2(fl_pack)) { pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); MPASS(powerof2(pack)); if (pack < 16) pack = 16; if (pack == 32) pack = 64; if (pack > 4096) pack = 4096; if (fl_pack != -1) { device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" " (%d), using %d instead.\n", fl_pack, pack); } } m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); if (pack == 16) v = V_INGPACKBOUNDARY(0); else v = V_INGPACKBOUNDARY(ilog2(pack) - 5); MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); } /* * adap->params.vpd.cclk must be set up before this is called. */ void t4_tweak_chip_settings(struct adapter *sc) { int i; uint32_t v, m; int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); static int sge_flbuf_sizes[] = { MCLBYTES, #if MJUMPAGESIZE != MCLBYTES MJUMPAGESIZE, MJUMPAGESIZE - CL_METADATA_SIZE, MJUMPAGESIZE - 2 * MSIZE - CL_METADATA_SIZE, #endif MJUM9BYTES, MJUM16BYTES, MCLBYTES - MSIZE - CL_METADATA_SIZE, MJUM9BYTES - CL_METADATA_SIZE, MJUM16BYTES - CL_METADATA_SIZE, }; KASSERT(sc->flags & MASTER_PF, ("%s: trying to change chip settings when not master.", __func__)); m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | V_EGRSTATUSPAGESIZE(spg_len == 128); t4_set_reg_field(sc, A_SGE_CONTROL, m, v); setup_pad_and_pack_boundaries(sc); v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); KASSERT(nitems(sge_flbuf_sizes) <= SGE_FLBUF_SIZES, ("%s: hw buffer size table too big", __func__)); for (i = 0; i < min(nitems(sge_flbuf_sizes), SGE_FLBUF_SIZES); i++) { t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i), sge_flbuf_sizes[i]); } v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); KASSERT(intr_timer[0] <= timer_max, ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], timer_max)); for (i = 1; i < nitems(intr_timer); i++) { KASSERT(intr_timer[i] >= intr_timer[i - 1], ("%s: timers not listed in increasing order (%d)", __func__, i)); while (intr_timer[i] > timer_max) { if (i == nitems(intr_timer) - 1) { intr_timer[i] = timer_max; break; } intr_timer[i] += intr_timer[i - 1]; intr_timer[i] /= 2; } } v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); if (chip_id(sc) >= CHELSIO_T6) { m = V_TSCALE(M_TSCALE); if (tscale == 1) v = 0; else v = V_TSCALE(tscale - 2); t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); } /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */ v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); /* * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we * may have to deal with is MAXPHYS + 1 page. */ v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4); t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v); /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); } /* * SGE wants the buffer to be at least 64B and then a multiple of 16. If * padding is in use, the buffer's start and end need to be aligned to the pad * boundary as well. We'll just make sure that the size is a multiple of the * boundary here, it is up to the buffer allocation code to make sure the start * of the buffer is aligned as well. */ static inline int hwsz_ok(struct adapter *sc, int hwsz) { int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; return (hwsz >= 64 && (hwsz & mask) == 0); } /* * XXX: driver really should be able to deal with unexpected settings. */ int t4_read_chip_settings(struct adapter *sc) { struct sge *s = &sc->sge; struct sge_params *sp = &sc->params.sge; int i, j, n, rc = 0; uint32_t m, v, r; uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); static int sw_buf_sizes[] = { /* Sorted by size */ MCLBYTES, #if MJUMPAGESIZE != MCLBYTES MJUMPAGESIZE, #endif MJUM9BYTES, MJUM16BYTES }; struct sw_zone_info *swz, *safe_swz; struct hw_buf_info *hwb; m = F_RXPKTCPLMODE; v = F_RXPKTCPLMODE; r = sc->params.sge.sge_control; if ((r & m) != v) { device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); rc = EINVAL; } /* * If this changes then every single use of PAGE_SHIFT in the driver * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. */ if (sp->page_shift != PAGE_SHIFT) { device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); rc = EINVAL; } /* Filter out unusable hw buffer sizes entirely (mark with -2). */ hwb = &s->hw_buf_info[0]; for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) { r = sc->params.sge.sge_fl_buffer_size[i]; hwb->size = r; hwb->zidx = hwsz_ok(sc, r) ? -1 : -2; hwb->next = -1; } /* * Create a sorted list in decreasing order of hw buffer sizes (and so * increasing order of spare area) for each software zone. * * If padding is enabled then the start and end of the buffer must align * to the pad boundary; if packing is enabled then they must align with * the pack boundary as well. Allocations from the cluster zones are * aligned to min(size, 4K), so the buffer starts at that alignment and * ends at hwb->size alignment. If mbuf inlining is allowed the * starting alignment will be reduced to MSIZE and the driver will * exercise appropriate caution when deciding on the best buffer layout * to use. */ n = 0; /* no usable buffer size to begin with */ swz = &s->sw_zone_info[0]; safe_swz = NULL; for (i = 0; i < SW_ZONE_SIZES; i++, swz++) { int8_t head = -1, tail = -1; swz->size = sw_buf_sizes[i]; swz->zone = m_getzone(swz->size); swz->type = m_gettype(swz->size); if (swz->size < PAGE_SIZE) { MPASS(powerof2(swz->size)); if (fl_pad && (swz->size % sp->pad_boundary != 0)) continue; } if (swz->size == safest_rx_cluster) safe_swz = swz; hwb = &s->hw_buf_info[0]; for (j = 0; j < SGE_FLBUF_SIZES; j++, hwb++) { if (hwb->zidx != -1 || hwb->size > swz->size) continue; #ifdef INVARIANTS if (fl_pad) MPASS(hwb->size % sp->pad_boundary == 0); #endif hwb->zidx = i; if (head == -1) head = tail = j; else if (hwb->size < s->hw_buf_info[tail].size) { s->hw_buf_info[tail].next = j; tail = j; } else { int8_t *cur; struct hw_buf_info *t; for (cur = &head; *cur != -1; cur = &t->next) { t = &s->hw_buf_info[*cur]; if (hwb->size == t->size) { hwb->zidx = -2; break; } if (hwb->size > t->size) { hwb->next = *cur; *cur = j; break; } } } } swz->head_hwidx = head; swz->tail_hwidx = tail; if (tail != -1) { n++; if (swz->size - s->hw_buf_info[tail].size >= CL_METADATA_SIZE) sc->flags |= BUF_PACKING_OK; } } if (n == 0) { device_printf(sc->dev, "no usable SGE FL buffer size.\n"); rc = EINVAL; } s->safe_hwidx1 = -1; s->safe_hwidx2 = -1; if (safe_swz != NULL) { s->safe_hwidx1 = safe_swz->head_hwidx; for (i = safe_swz->head_hwidx; i != -1; i = hwb->next) { int spare; hwb = &s->hw_buf_info[i]; #ifdef INVARIANTS if (fl_pad) MPASS(hwb->size % sp->pad_boundary == 0); #endif spare = safe_swz->size - hwb->size; if (spare >= CL_METADATA_SIZE) { s->safe_hwidx2 = i; break; } } } if (sc->flags & IS_VF) return (0); v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); if (r != v) { device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); rc = EINVAL; } m = v = F_TDDPTAGTCB; r = t4_read_reg(sc, A_ULP_RX_CTL); if ((r & m) != v) { device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); rc = EINVAL; } m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; r = t4_read_reg(sc, A_TP_PARA_REG5); if ((r & m) != v) { device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); rc = EINVAL; } - t4_init_tp_params(sc); + t4_init_tp_params(sc, 1); t4_read_mtu_tbl(sc, sc->params.mtus, NULL); t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); return (rc); } int t4_create_dma_tag(struct adapter *sc) { int rc; rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->dmat); if (rc != 0) { device_printf(sc->dev, "failed to create main DMA tag: %d\n", rc); } return (rc); } void t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, struct sysctl_oid_list *children) { struct sge_params *sp = &sc->params.sge; SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", CTLTYPE_STRING | CTLFLAG_RD, &sc->sge, 0, sysctl_bufsizes, "A", "freelist buffer sizes"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, NULL, sp->pad_boundary, "payload pad boundary (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, NULL, sp->spg_len, "status page size (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, NULL, cong_drop, "congestion drop setting"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, NULL, sp->pack_boundary, "payload pack boundary (bytes)"); } int t4_destroy_dma_tag(struct adapter *sc) { if (sc->dmat) bus_dma_tag_destroy(sc->dmat); return (0); } /* * Allocate and initialize the firmware event queue and the management queue. * * Returns errno on failure. Resources allocated up to that point may still be * allocated. Caller is responsible for cleanup in case this function fails. */ int t4_setup_adapter_queues(struct adapter *sc) { int rc; ADAPTER_LOCK_ASSERT_NOTOWNED(sc); sysctl_ctx_init(&sc->ctx); sc->flags |= ADAP_SYSCTL_CTX; /* * Firmware event queue */ rc = alloc_fwq(sc); if (rc != 0) return (rc); /* * Management queue. This is just a control queue that uses the fwq as * its associated iq. */ if (!(sc->flags & IS_VF)) rc = alloc_mgmtq(sc); return (rc); } /* * Idempotent */ int t4_teardown_adapter_queues(struct adapter *sc) { ADAPTER_LOCK_ASSERT_NOTOWNED(sc); /* Do this before freeing the queue */ if (sc->flags & ADAP_SYSCTL_CTX) { sysctl_ctx_free(&sc->ctx); sc->flags &= ~ADAP_SYSCTL_CTX; } free_mgmtq(sc); free_fwq(sc); return (0); } static inline int first_vector(struct vi_info *vi) { struct adapter *sc = vi->pi->adapter; if (sc->intr_count == 1) return (0); return (vi->first_intr); } /* * Given an arbitrary "index," come up with an iq that can be used by other * queues (of this VI) for interrupt forwarding, SGE egress updates, etc. * The iq returned is guaranteed to be something that takes direct interrupts. */ static struct sge_iq * vi_intr_iq(struct vi_info *vi, int idx) { struct adapter *sc = vi->pi->adapter; struct sge *s = &sc->sge; struct sge_iq *iq = NULL; int nintr, i; if (sc->intr_count == 1) return (&sc->sge.fwq); nintr = vi->nintr; #ifdef DEV_NETMAP /* Do not consider any netmap-only interrupts */ if (vi->flags & INTR_RXQ && vi->nnmrxq > vi->nrxq) nintr -= vi->nnmrxq - vi->nrxq; #endif KASSERT(nintr != 0, ("%s: vi %p has no exclusive interrupts, total interrupts = %d", __func__, vi, sc->intr_count)); i = idx % nintr; if (vi->flags & INTR_RXQ) { if (i < vi->nrxq) { iq = &s->rxq[vi->first_rxq + i].iq; goto done; } i -= vi->nrxq; } #ifdef TCP_OFFLOAD if (vi->flags & INTR_OFLD_RXQ) { if (i < vi->nofldrxq) { iq = &s->ofld_rxq[vi->first_ofld_rxq + i].iq; goto done; } i -= vi->nofldrxq; } #endif panic("%s: vi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__, vi, vi->flags & INTR_ALL, idx, nintr); done: MPASS(iq != NULL); KASSERT(iq->flags & IQ_INTR, ("%s: iq %p (vi %p, intr_flags 0x%lx, idx %d)", __func__, iq, vi, vi->flags & INTR_ALL, idx)); return (iq); } /* Maximum payload that can be delivered with a single iq descriptor */ static inline int mtu_to_max_payload(struct adapter *sc, int mtu, const int toe) { int payload; #ifdef TCP_OFFLOAD if (toe) { payload = sc->tt.rx_coalesce ? G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)) : mtu; } else { #endif /* large enough even when hw VLAN extraction is disabled */ payload = sc->params.sge.fl_pktshift + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + mtu; #ifdef TCP_OFFLOAD } #endif return (payload); } int t4_setup_vi_queues(struct vi_info *vi) { int rc = 0, i, j, intr_idx, iqid; struct sge_rxq *rxq; struct sge_txq *txq; struct sge_wrq *ctrlq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ofld_txq; #endif #ifdef DEV_NETMAP int saved_idx; struct sge_nm_rxq *nm_rxq; struct sge_nm_txq *nm_txq; #endif char name[16]; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct ifnet *ifp = vi->ifp; struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); int maxp, mtu = ifp->if_mtu; /* Interrupt vector to start from (when using multiple vectors) */ intr_idx = first_vector(vi); #ifdef DEV_NETMAP saved_idx = intr_idx; if (ifp->if_capabilities & IFCAP_NETMAP) { /* netmap is supported with direct interrupts only. */ MPASS(vi->flags & INTR_RXQ); /* * We don't have buffers to back the netmap rx queues * right now so we create the queues in a way that * doesn't set off any congestion signal in the chip. */ oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq", CTLFLAG_RD, NULL, "rx queues"); for_each_nm_rxq(vi, i, nm_rxq) { rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq", CTLFLAG_RD, NULL, "tx queues"); for_each_nm_txq(vi, i, nm_txq) { iqid = vi->first_nm_rxq + (i % vi->nnmrxq); rc = alloc_nm_txq(vi, nm_txq, iqid, i, oid); if (rc != 0) goto done; } } /* Normal rx queues and netmap rx queues share the same interrupts. */ intr_idx = saved_idx; #endif /* * First pass over all NIC and TOE rx queues: * a) initialize iq and fl * b) allocate queue iff it will take direct interrupts. */ maxp = mtu_to_max_payload(sc, mtu, 0); if (vi->flags & INTR_RXQ) { oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD, NULL, "rx queues"); } for_each_rxq(vi, i, rxq) { init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); snprintf(name, sizeof(name), "%s rxq%d-fl", device_get_nameunit(vi->dev), i); init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); if (vi->flags & INTR_RXQ) { rxq->iq.flags |= IQ_INTR; rc = alloc_rxq(vi, rxq, intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } } #ifdef DEV_NETMAP if (ifp->if_capabilities & IFCAP_NETMAP) intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); #endif #ifdef TCP_OFFLOAD maxp = mtu_to_max_payload(sc, mtu, 1); if (vi->flags & INTR_OFLD_RXQ) { oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", CTLFLAG_RD, NULL, "rx queues for offloaded TCP connections"); } for_each_ofld_rxq(vi, i, ofld_rxq) { init_iq(&ofld_rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", device_get_nameunit(vi->dev), i); init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); if (vi->flags & INTR_OFLD_RXQ) { ofld_rxq->iq.flags |= IQ_INTR; rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } } #endif /* * Second pass over all NIC and TOE rx queues. The queues forwarding * their interrupts are allocated now. */ j = 0; if (!(vi->flags & INTR_RXQ)) { oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD, NULL, "rx queues"); for_each_rxq(vi, i, rxq) { MPASS(!(rxq->iq.flags & IQ_INTR)); intr_idx = vi_intr_iq(vi, j)->abs_id; rc = alloc_rxq(vi, rxq, intr_idx, i, oid); if (rc != 0) goto done; j++; } } #ifdef TCP_OFFLOAD if (vi->nofldrxq != 0 && !(vi->flags & INTR_OFLD_RXQ)) { oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", CTLFLAG_RD, NULL, "rx queues for offloaded TCP connections"); for_each_ofld_rxq(vi, i, ofld_rxq) { MPASS(!(ofld_rxq->iq.flags & IQ_INTR)); intr_idx = vi_intr_iq(vi, j)->abs_id; rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid); if (rc != 0) goto done; j++; } } #endif /* * Now the tx queues. Only one pass needed. */ oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD, NULL, "tx queues"); j = 0; for_each_txq(vi, i, txq) { iqid = vi_intr_iq(vi, j)->cntxt_id; snprintf(name, sizeof(name), "%s txq%d", device_get_nameunit(vi->dev), i); init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, iqid, name); rc = alloc_txq(vi, txq, i, oid); if (rc != 0) goto done; j++; } #ifdef TCP_OFFLOAD oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections"); for_each_ofld_txq(vi, i, ofld_txq) { struct sysctl_oid *oid2; iqid = vi_intr_iq(vi, j)->cntxt_id; snprintf(name, sizeof(name), "%s ofld_txq%d", device_get_nameunit(vi->dev), i); init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan, iqid, name); snprintf(name, sizeof(name), "%d", i); oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL, "offload tx queue"); rc = alloc_wrq(sc, vi, ofld_txq, oid2); if (rc != 0) goto done; j++; } #endif /* * Finally, the control queue. */ if (!IS_MAIN_VI(vi) || sc->flags & IS_VF) goto done; oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD, NULL, "ctrl queue"); ctrlq = &sc->sge.ctrlq[pi->port_id]; iqid = vi_intr_iq(vi, 0)->cntxt_id; snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(vi->dev)); init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid, name); rc = alloc_wrq(sc, vi, ctrlq, oid); done: if (rc) t4_teardown_vi_queues(vi); return (rc); } /* * Idempotent */ int t4_teardown_vi_queues(struct vi_info *vi) { int i; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_rxq *rxq; struct sge_txq *txq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ofld_txq; #endif #ifdef DEV_NETMAP struct sge_nm_rxq *nm_rxq; struct sge_nm_txq *nm_txq; #endif /* Do this before freeing the queues */ if (vi->flags & VI_SYSCTL_CTX) { sysctl_ctx_free(&vi->ctx); vi->flags &= ~VI_SYSCTL_CTX; } #ifdef DEV_NETMAP if (vi->ifp->if_capabilities & IFCAP_NETMAP) { for_each_nm_txq(vi, i, nm_txq) { free_nm_txq(vi, nm_txq); } for_each_nm_rxq(vi, i, nm_rxq) { free_nm_rxq(vi, nm_rxq); } } #endif /* * Take down all the tx queues first, as they reference the rx queues * (for egress updates, etc.). */ if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF)) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); for_each_txq(vi, i, txq) { free_txq(vi, txq); } #ifdef TCP_OFFLOAD for_each_ofld_txq(vi, i, ofld_txq) { free_wrq(sc, ofld_txq); } #endif /* * Then take down the rx queues that forward their interrupts, as they * reference other rx queues. */ for_each_rxq(vi, i, rxq) { if ((rxq->iq.flags & IQ_INTR) == 0) free_rxq(vi, rxq); } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { if ((ofld_rxq->iq.flags & IQ_INTR) == 0) free_ofld_rxq(vi, ofld_rxq); } #endif /* * Then take down the rx queues that take direct interrupts. */ for_each_rxq(vi, i, rxq) { if (rxq->iq.flags & IQ_INTR) free_rxq(vi, rxq); } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { if (ofld_rxq->iq.flags & IQ_INTR) free_ofld_rxq(vi, ofld_rxq); } #endif return (0); } /* * Deals with errors and the firmware event queue. All data rx queues forward * their interrupt to the firmware event queue. */ void t4_intr_all(void *arg) { struct adapter *sc = arg; struct sge_iq *fwq = &sc->sge.fwq; t4_intr_err(arg); if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) { service_iq(fwq, 0); atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE); } } /* Deals with error interrupts */ void t4_intr_err(void *arg) { struct adapter *sc = arg; t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); t4_slow_intr_handler(sc); } void t4_intr_evt(void *arg) { struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { service_iq(iq, 0); atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } void t4_intr(void *arg) { struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { service_iq(iq, 0); atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } void t4_vi_intr(void *arg) { struct irq *irq = arg; #ifdef DEV_NETMAP if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) { t4_nm_intr(irq->nm_rxq); atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON); } #endif if (irq->rxq != NULL) t4_intr(irq->rxq); } static inline int sort_before_lro(struct lro_ctrl *lro) { return (lro->lro_mbuf_max != 0); } /* * Deals with anything and everything on the given ingress queue. */ static int service_iq(struct sge_iq *iq, int budget) { struct sge_iq *q; struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ struct sge_fl *fl; /* Use iff IQ_HAS_FL */ struct adapter *sc = iq->adapter; struct iq_desc *d = &iq->desc[iq->cidx]; int ndescs = 0, limit; int rsp_type, refill; uint32_t lq; uint16_t fl_hw_cidx; struct mbuf *m0; STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); #if defined(INET) || defined(INET6) const struct timeval lro_timeout = {0, sc->lro_timeout}; struct lro_ctrl *lro = &rxq->lro; #endif KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); limit = budget ? budget : iq->qsize / 16; if (iq->flags & IQ_HAS_FL) { fl = &rxq->fl; fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ } else { fl = NULL; fl_hw_cidx = 0; /* to silence gcc warning */ } #if defined(INET) || defined(INET6) if (iq->flags & IQ_ADJ_CREDIT) { MPASS(sort_before_lro(lro)); iq->flags &= ~IQ_ADJ_CREDIT; if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { tcp_lro_flush_all(lro); t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); return (0); } ndescs = 1; } #else MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); #endif /* * We always come back and check the descriptor ring for new indirect * interrupts and other responses after running a single handler. */ for (;;) { while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { rmb(); refill = 0; m0 = NULL; rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); lq = be32toh(d->rsp.pldbuflen_qid); switch (rsp_type) { case X_RSPD_TYPE_FLBUF: KASSERT(iq->flags & IQ_HAS_FL, ("%s: data for an iq (%p) with no freelist", __func__, iq)); m0 = get_fl_payload(sc, fl, lq); if (__predict_false(m0 == NULL)) goto process_iql; refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2; #ifdef T4_PKT_TIMESTAMP /* * 60 bit timestamp for the payload is * *(uint64_t *)m0->m_pktdat. Note that it is * in the leading free-space in the mbuf. The * kernel can clobber it during a pullup, * m_copymdata, etc. You need to make sure that * the mbuf reaches you unmolested if you care * about the timestamp. */ *(uint64_t *)m0->m_pktdat = be64toh(ctrl->u.last_flit) & 0xfffffffffffffff; #endif /* fall through */ case X_RSPD_TYPE_CPL: KASSERT(d->rss.opcode < NUM_CPL_CMDS, ("%s: bad opcode %02x.", __func__, d->rss.opcode)); t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); break; case X_RSPD_TYPE_INTR: /* * Interrupts should be forwarded only to queues * that are not forwarding their interrupts. * This means service_iq can recurse but only 1 * level deep. */ KASSERT(budget == 0, ("%s: budget %u, rsp_type %u", __func__, budget, rsp_type)); /* * There are 1K interrupt-capable queues (qids 0 * through 1023). A response type indicating a * forwarded interrupt with a qid >= 1K is an * iWARP async notification. */ if (lq >= 1024) { t4_an_handler(iq, &d->rsp); break; } q = sc->sge.iqmap[lq - sc->sge.iq_start - sc->sge.iq_base]; if (atomic_cmpset_int(&q->state, IQS_IDLE, IQS_BUSY)) { if (service_iq(q, q->qsize / 16) == 0) { atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); } else { STAILQ_INSERT_TAIL(&iql, q, link); } } break; default: KASSERT(0, ("%s: illegal response type %d on iq %p", __func__, rsp_type, iq)); log(LOG_ERR, "%s: illegal response type %d on iq %p", device_get_nameunit(sc->dev), rsp_type, iq); break; } d++; if (__predict_false(++iq->cidx == iq->sidx)) { iq->cidx = 0; iq->gen ^= F_RSPD_GEN; d = &iq->desc[0]; } if (__predict_false(++ndescs == limit)) { t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); ndescs = 0; #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED && !sort_before_lro(lro) && sc->lro_timeout != 0) { tcp_lro_flush_inactive(lro, &lro_timeout); } #endif if (budget) { if (iq->flags & IQ_HAS_FL) { FL_LOCK(fl); refill_fl(sc, fl, 32); FL_UNLOCK(fl); } return (EINPROGRESS); } } if (refill) { FL_LOCK(fl); refill_fl(sc, fl, 32); FL_UNLOCK(fl); fl_hw_cidx = fl->hw_cidx; } } process_iql: if (STAILQ_EMPTY(&iql)) break; /* * Process the head only, and send it to the back of the list if * it's still not done. */ q = STAILQ_FIRST(&iql); STAILQ_REMOVE_HEAD(&iql, link); if (service_iq(q, q->qsize / 8) == 0) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); else STAILQ_INSERT_TAIL(&iql, q, link); } #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED) { if (ndescs > 0 && lro->lro_mbuf_count > 8) { MPASS(sort_before_lro(lro)); /* hold back one credit and don't flush LRO state */ iq->flags |= IQ_ADJ_CREDIT; ndescs--; } else { tcp_lro_flush_all(lro); } } #endif t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); if (iq->flags & IQ_HAS_FL) { int starved; FL_LOCK(fl); starved = refill_fl(sc, fl, 64); FL_UNLOCK(fl); if (__predict_false(starved != 0)) add_fl_to_sfl(sc, fl); } return (0); } static inline int cl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll) { int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0; if (rc) MPASS(cll->region3 >= CL_METADATA_SIZE); return (rc); } static inline struct cluster_metadata * cl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll, caddr_t cl) { if (cl_has_metadata(fl, cll)) { struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; return ((struct cluster_metadata *)(cl + swz->size) - 1); } return (NULL); } static void rxb_free(struct mbuf *m, void *arg1, void *arg2) { uma_zone_t zone = arg1; caddr_t cl = arg2; uma_zfree(zone, cl); counter_u64_add(extfree_rels, 1); } /* * The mbuf returned by this function could be allocated from zone_mbuf or * constructed in spare room in the cluster. * * The mbuf carries the payload in one of these ways * a) frame inside the mbuf (mbuf from zone_mbuf) * b) m_cljset (for clusters without metadata) zone_mbuf * c) m_extaddref (cluster with metadata) inline mbuf * d) m_extaddref (cluster with metadata) zone_mbuf */ static struct mbuf * get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, int remaining) { struct mbuf *m; struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; struct cluster_layout *cll = &sd->cll; struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx]; struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl); int len, blen; caddr_t payload; blen = hwb->size - fl->rx_offset; /* max possible in this buf */ len = min(remaining, blen); payload = sd->cl + cll->region1 + fl->rx_offset; if (fl->flags & FL_BUF_PACKING) { const u_int l = fr_offset + len; const u_int pad = roundup2(l, fl->buf_boundary) - l; if (fl->rx_offset + len + pad < hwb->size) blen = len + pad; MPASS(fl->rx_offset + blen <= hwb->size); } else { MPASS(fl->rx_offset == 0); /* not packing */ } if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { /* * Copy payload into a freshly allocated mbuf. */ m = fr_offset == 0 ? m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); if (m == NULL) return (NULL); fl->mbuf_allocated++; #ifdef T4_PKT_TIMESTAMP /* Leave room for a timestamp */ m->m_data += 8; #endif /* copy data to mbuf */ bcopy(payload, mtod(m, caddr_t), len); } else if (sd->nmbuf * MSIZE < cll->region1) { /* * There's spare room in the cluster for an mbuf. Create one * and associate it with the payload that's in the cluster. */ MPASS(clm != NULL); m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE); /* No bzero required */ if (m_init(m, M_NOWAIT, MT_DATA, fr_offset == 0 ? M_PKTHDR | M_NOFREE : M_NOFREE)) return (NULL); fl->mbuf_inlined++; m_extaddref(m, payload, blen, &clm->refcount, rxb_free, swz->zone, sd->cl); if (sd->nmbuf++ == 0) counter_u64_add(extfree_refs, 1); } else { /* * Grab an mbuf from zone_mbuf and associate it with the * payload in the cluster. */ m = fr_offset == 0 ? m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); if (m == NULL) return (NULL); fl->mbuf_allocated++; if (clm != NULL) { m_extaddref(m, payload, blen, &clm->refcount, rxb_free, swz->zone, sd->cl); if (sd->nmbuf++ == 0) counter_u64_add(extfree_refs, 1); } else { m_cljset(m, sd->cl, swz->type); sd->cl = NULL; /* consumed, not a recycle candidate */ } } if (fr_offset == 0) m->m_pkthdr.len = remaining; m->m_len = len; if (fl->flags & FL_BUF_PACKING) { fl->rx_offset += blen; MPASS(fl->rx_offset <= hwb->size); if (fl->rx_offset < hwb->size) return (m); /* without advancing the cidx */ } if (__predict_false(++fl->cidx % 8 == 0)) { uint16_t cidx = fl->cidx / 8; if (__predict_false(cidx == fl->sidx)) fl->cidx = cidx = 0; fl->hw_cidx = cidx; } fl->rx_offset = 0; return (m); } static struct mbuf * get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf) { struct mbuf *m0, *m, **pnext; u_int remaining; const u_int total = G_RSPD_LEN(len_newbuf); if (__predict_false(fl->flags & FL_BUF_RESUME)) { M_ASSERTPKTHDR(fl->m0); MPASS(fl->m0->m_pkthdr.len == total); MPASS(fl->remaining < total); m0 = fl->m0; pnext = fl->pnext; remaining = fl->remaining; fl->flags &= ~FL_BUF_RESUME; goto get_segment; } if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) { fl->rx_offset = 0; if (__predict_false(++fl->cidx % 8 == 0)) { uint16_t cidx = fl->cidx / 8; if (__predict_false(cidx == fl->sidx)) fl->cidx = cidx = 0; fl->hw_cidx = cidx; } } /* * Payload starts at rx_offset in the current hw buffer. Its length is * 'len' and it may span multiple hw buffers. */ m0 = get_scatter_segment(sc, fl, 0, total); if (m0 == NULL) return (NULL); remaining = total - m0->m_len; pnext = &m0->m_next; while (remaining > 0) { get_segment: MPASS(fl->rx_offset == 0); m = get_scatter_segment(sc, fl, total - remaining, remaining); if (__predict_false(m == NULL)) { fl->m0 = m0; fl->pnext = pnext; fl->remaining = remaining; fl->flags |= FL_BUF_RESUME; return (NULL); } *pnext = m; pnext = &m->m_next; remaining -= m->m_len; } *pnext = NULL; M_ASSERTPKTHDR(m0); return (m0); } static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) { struct sge_rxq *rxq = iq_to_rxq(iq); struct ifnet *ifp = rxq->ifp; struct adapter *sc = iq->adapter; const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); #if defined(INET) || defined(INET6) struct lro_ctrl *lro = &rxq->lro; #endif static const int sw_hashtype[4][2] = { {M_HASHTYPE_NONE, M_HASHTYPE_NONE}, {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6}, {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6}, {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6}, }; KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__, rss->opcode)); m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; m0->m_len -= sc->params.sge.fl_pktshift; m0->m_data += sc->params.sge.fl_pktshift; m0->m_pkthdr.rcvif = ifp; M_HASHTYPE_SET(m0, sw_hashtype[rss->hash_type][rss->ipv6]); m0->m_pkthdr.flowid = be32toh(rss->hash_val); if (cpl->csum_calc && !(cpl->err_vec & sc->params.tp.err_vec_mask)) { if (ifp->if_capenable & IFCAP_RXCSUM && cpl->l2info & htobe32(F_RXF_IP)) { m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); rxq->rxcsum++; } else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 && cpl->l2info & htobe32(F_RXF_IP6)) { m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | CSUM_PSEUDO_HDR); rxq->rxcsum++; } if (__predict_false(cpl->ip_frag)) m0->m_pkthdr.csum_data = be16toh(cpl->csum); else m0->m_pkthdr.csum_data = 0xffff; } if (cpl->vlan_ex) { m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); m0->m_flags |= M_VLANTAG; rxq->vlan_extraction++; } #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED) { if (sort_before_lro(lro)) { tcp_lro_queue_mbuf(lro, m0); return (0); /* queued for sort, then LRO */ } if (tcp_lro_rx(lro, m0, 0) == 0) return (0); /* queued for LRO */ } #endif ifp->if_input(ifp, m0); return (0); } /* * Must drain the wrq or make sure that someone else will. */ static void wrq_tx_drain(void *arg, int n) { struct sge_wrq *wrq = arg; struct sge_eq *eq = &wrq->eq; EQ_LOCK(eq); if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(wrq->adapter, wrq); EQ_UNLOCK(eq); } static void drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) { struct sge_eq *eq = &wrq->eq; u_int available, dbdiff; /* # of hardware descriptors */ u_int n; struct wrqe *wr; struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ EQ_LOCK_ASSERT_OWNED(eq); MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); wr = STAILQ_FIRST(&wrq->wr_list); MPASS(wr != NULL); /* Must be called with something useful to do */ MPASS(eq->pidx == eq->dbidx); dbdiff = 0; do { eq->cidx = read_hw_cidx(eq); if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; MPASS(wr->wrq == wrq); n = howmany(wr->wr_len, EQ_ESIZE); if (available < n) break; dst = (void *)&eq->desc[eq->pidx]; if (__predict_true(eq->sidx - eq->pidx > n)) { /* Won't wrap, won't end exactly at the status page. */ bcopy(&wr->wr[0], dst, wr->wr_len); eq->pidx += n; } else { int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; bcopy(&wr->wr[0], dst, first_portion); if (wr->wr_len > first_portion) { bcopy(&wr->wr[first_portion], &eq->desc[0], wr->wr_len - first_portion); } eq->pidx = n - (eq->sidx - eq->pidx); } wrq->tx_wrs_copied++; if (available < eq->sidx / 4 && atomic_cmpset_int(&eq->equiq, 0, 1)) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } dbdiff += n; if (dbdiff >= 16) { ring_eq_db(sc, eq, dbdiff); dbdiff = 0; } STAILQ_REMOVE_HEAD(&wrq->wr_list, link); free_wrqe(wr); MPASS(wrq->nwr_pending > 0); wrq->nwr_pending--; MPASS(wrq->ndesc_needed >= n); wrq->ndesc_needed -= n; } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); if (dbdiff) ring_eq_db(sc, eq, dbdiff); } /* * Doesn't fail. Holds on to work requests it can't send right away. */ void t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) { #ifdef INVARIANTS struct sge_eq *eq = &wrq->eq; #endif EQ_LOCK_ASSERT_OWNED(eq); MPASS(wr != NULL); MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); MPASS((wr->wr_len & 0x7) == 0); STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); wrq->nwr_pending++; wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) return; /* commit_wrq_wr will drain wr_list as well. */ drain_wrq_wr_list(sc, wrq); /* Doorbell must have caught up to the pidx. */ MPASS(eq->pidx == eq->dbidx); } void t4_update_fl_bufsize(struct ifnet *ifp) { struct vi_info *vi = ifp->if_softc; struct adapter *sc = vi->pi->adapter; struct sge_rxq *rxq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif struct sge_fl *fl; int i, maxp, mtu = ifp->if_mtu; maxp = mtu_to_max_payload(sc, mtu, 0); for_each_rxq(vi, i, rxq) { fl = &rxq->fl; FL_LOCK(fl); find_best_refill_source(sc, fl, maxp); FL_UNLOCK(fl); } #ifdef TCP_OFFLOAD maxp = mtu_to_max_payload(sc, mtu, 1); for_each_ofld_rxq(vi, i, ofld_rxq) { fl = &ofld_rxq->fl; FL_LOCK(fl); find_best_refill_source(sc, fl, maxp); FL_UNLOCK(fl); } #endif } static inline int mbuf_nsegs(struct mbuf *m) { M_ASSERTPKTHDR(m); KASSERT(m->m_pkthdr.l5hlen > 0, ("%s: mbuf %p missing information on # of segments.", __func__, m)); return (m->m_pkthdr.l5hlen); } static inline void set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) { M_ASSERTPKTHDR(m); m->m_pkthdr.l5hlen = nsegs; } static inline int mbuf_len16(struct mbuf *m) { int n; M_ASSERTPKTHDR(m); n = m->m_pkthdr.PH_loc.eight[0]; MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); return (n); } static inline void set_mbuf_len16(struct mbuf *m, uint8_t len16) { M_ASSERTPKTHDR(m); m->m_pkthdr.PH_loc.eight[0] = len16; } static inline int needs_tso(struct mbuf *m) { M_ASSERTPKTHDR(m); if (m->m_pkthdr.csum_flags & CSUM_TSO) { KASSERT(m->m_pkthdr.tso_segsz > 0, ("%s: TSO requested in mbuf %p but MSS not provided", __func__, m)); return (1); } return (0); } static inline int needs_l3_csum(struct mbuf *m) { M_ASSERTPKTHDR(m); if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) return (1); return (0); } static inline int needs_l4_csum(struct mbuf *m) { M_ASSERTPKTHDR(m); if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) return (1); return (0); } static inline int needs_vlan_insertion(struct mbuf *m) { M_ASSERTPKTHDR(m); if (m->m_flags & M_VLANTAG) { KASSERT(m->m_pkthdr.ether_vtag != 0, ("%s: HWVLAN requested in mbuf %p but tag not provided", __func__, m)); return (1); } return (0); } static void * m_advance(struct mbuf **pm, int *poffset, int len) { struct mbuf *m = *pm; int offset = *poffset; uintptr_t p = 0; MPASS(len > 0); for (;;) { if (offset + len < m->m_len) { offset += len; p = mtod(m, uintptr_t) + offset; break; } len -= m->m_len - offset; m = m->m_next; offset = 0; MPASS(m != NULL); } *poffset = offset; *pm = m; return ((void *)p); } /* * Can deal with empty mbufs in the chain that have m_len = 0, but the chain * must have at least one mbuf that's not empty. */ static inline int count_mbuf_nsegs(struct mbuf *m) { vm_paddr_t lastb, next; vm_offset_t va; int len, nsegs; MPASS(m != NULL); nsegs = 0; lastb = 0; for (; m; m = m->m_next) { len = m->m_len; if (__predict_false(len == 0)) continue; va = mtod(m, vm_offset_t); next = pmap_kextract(va); nsegs += sglist_count(m->m_data, len); if (lastb + 1 == next) nsegs--; lastb = pmap_kextract(va + len - 1); } MPASS(nsegs > 0); return (nsegs); } /* * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: * a) caller can assume it's been freed if this function returns with an error. * b) it may get defragged up if the gather list is too long for the hardware. */ int parse_pkt(struct adapter *sc, struct mbuf **mp) { struct mbuf *m0 = *mp, *m; int rc, nsegs, defragged = 0, offset; struct ether_header *eh; void *l3hdr; #if defined(INET) || defined(INET6) struct tcphdr *tcp; #endif uint16_t eh_type; M_ASSERTPKTHDR(m0); if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { rc = EINVAL; fail: m_freem(m0); *mp = NULL; return (rc); } restart: /* * First count the number of gather list segments in the payload. * Defrag the mbuf if nsegs exceeds the hardware limit. */ M_ASSERTPKTHDR(m0); MPASS(m0->m_pkthdr.len > 0); nsegs = count_mbuf_nsegs(m0); if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { rc = EFBIG; goto fail; } *mp = m0 = m; /* update caller's copy after defrag */ goto restart; } if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) { m0 = m_pullup(m0, m0->m_pkthdr.len); if (m0 == NULL) { /* Should have left well enough alone. */ rc = EFBIG; goto fail; } *mp = m0; /* update caller's copy after pullup */ goto restart; } set_mbuf_nsegs(m0, nsegs); if (sc->flags & IS_VF) set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); else set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); if (!needs_tso(m0) && !(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0)))) return (0); m = m0; eh = mtod(m, struct ether_header *); eh_type = ntohs(eh->ether_type); if (eh_type == ETHERTYPE_VLAN) { struct ether_vlan_header *evh = (void *)eh; eh_type = ntohs(evh->evl_proto); m0->m_pkthdr.l2hlen = sizeof(*evh); } else m0->m_pkthdr.l2hlen = sizeof(*eh); offset = 0; l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); switch (eh_type) { #ifdef INET6 case ETHERTYPE_IPV6: { struct ip6_hdr *ip6 = l3hdr; MPASS(!needs_tso(m0) || ip6->ip6_nxt == IPPROTO_TCP); m0->m_pkthdr.l3hlen = sizeof(*ip6); break; } #endif #ifdef INET case ETHERTYPE_IP: { struct ip *ip = l3hdr; m0->m_pkthdr.l3hlen = ip->ip_hl * 4; break; } #endif default: panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" " with the same INET/INET6 options as the kernel.", __func__, eh_type); } #if defined(INET) || defined(INET6) if (needs_tso(m0)) { tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); m0->m_pkthdr.l4hlen = tcp->th_off * 4; } #endif MPASS(m0 == *mp); return (0); } void * start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) { struct sge_eq *eq = &wrq->eq; struct adapter *sc = wrq->adapter; int ndesc, available; struct wrqe *wr; void *w; MPASS(len16 > 0); ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); EQ_LOCK(eq); if (!STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(sc, wrq); if (!STAILQ_EMPTY(&wrq->wr_list)) { slowpath: EQ_UNLOCK(eq); wr = alloc_wrqe(len16 * 16, wrq); if (__predict_false(wr == NULL)) return (NULL); cookie->pidx = -1; cookie->ndesc = ndesc; return (&wr->wr); } eq->cidx = read_hw_cidx(eq); if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; if (available < ndesc) goto slowpath; cookie->pidx = eq->pidx; cookie->ndesc = ndesc; TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); w = &eq->desc[eq->pidx]; IDXINCR(eq->pidx, ndesc, eq->sidx); if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { w = &wrq->ss[0]; wrq->ss_pidx = cookie->pidx; wrq->ss_len = len16 * 16; } EQ_UNLOCK(eq); return (w); } void commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) { struct sge_eq *eq = &wrq->eq; struct adapter *sc = wrq->adapter; int ndesc, pidx; struct wrq_cookie *prev, *next; if (cookie->pidx == -1) { struct wrqe *wr = __containerof(w, struct wrqe, wr); t4_wrq_tx(sc, wr); return; } ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ pidx = cookie->pidx; MPASS(pidx >= 0 && pidx < eq->sidx); if (__predict_false(w == &wrq->ss[0])) { int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; MPASS(wrq->ss_len > n); /* WR had better wrap around. */ bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); wrq->tx_wrs_ss++; } else wrq->tx_wrs_direct++; EQ_LOCK(eq); prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); next = TAILQ_NEXT(cookie, link); if (prev == NULL) { MPASS(pidx == eq->dbidx); if (next == NULL || ndesc >= 16) ring_eq_db(wrq->adapter, eq, ndesc); else { MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); next->pidx = pidx; next->ndesc += ndesc; } } else { MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); prev->ndesc += ndesc; } TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(sc, wrq); #ifdef INVARIANTS if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { /* Doorbell must have caught up to the pidx. */ MPASS(wrq->eq.pidx == wrq->eq.dbidx); } #endif EQ_UNLOCK(eq); } static u_int can_resume_eth_tx(struct mp_ring *r) { struct sge_eq *eq = r->cookie; return (total_available_tx_desc(eq) > eq->sidx / 8); } static inline int cannot_use_txpkts(struct mbuf *m) { /* maybe put a GL limit too, to avoid silliness? */ return (needs_tso(m)); } static inline int discard_tx(struct sge_eq *eq) { return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); } /* * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to * be consumed. Return the actual number consumed. 0 indicates a stall. */ static u_int eth_tx(struct mp_ring *r, u_int cidx, u_int pidx) { struct sge_txq *txq = r->cookie; struct sge_eq *eq = &txq->eq; struct ifnet *ifp = txq->ifp; struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; u_int total, remaining; /* # of packets */ u_int available, dbdiff; /* # of hardware descriptors */ u_int n, next_cidx; struct mbuf *m0, *tail; struct txpkts txp; struct fw_eth_tx_pkts_wr *wr; /* any fw WR struct will do */ remaining = IDXDIFF(pidx, cidx, r->size); MPASS(remaining > 0); /* Must not be called without work to do. */ total = 0; TXQ_LOCK(txq); if (__predict_false(discard_tx(eq))) { while (cidx != pidx) { m0 = r->items[cidx]; m_freem(m0); if (++cidx == r->size) cidx = 0; } reclaim_tx_descs(txq, 2048); total = remaining; goto done; } /* How many hardware descriptors do we have readily available. */ if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); while (remaining > 0) { m0 = r->items[cidx]; M_ASSERTPKTHDR(m0); MPASS(m0->m_nextpkt == NULL); if (available < SGE_MAX_WR_NDESC) { available += reclaim_tx_descs(txq, 64); if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) break; /* out of descriptors */ } next_cidx = cidx + 1; if (__predict_false(next_cidx == r->size)) next_cidx = 0; wr = (void *)&eq->desc[eq->pidx]; if (sc->flags & IS_VF) { total++; remaining--; ETHER_BPF_MTAP(ifp, m0); n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0, available); } else if (remaining > 1 && try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { /* pkts at cidx, next_cidx should both be in txp. */ MPASS(txp.npkt == 2); tail = r->items[next_cidx]; MPASS(tail->m_nextpkt == NULL); ETHER_BPF_MTAP(ifp, m0); ETHER_BPF_MTAP(ifp, tail); m0->m_nextpkt = tail; if (__predict_false(++next_cidx == r->size)) next_cidx = 0; while (next_cidx != pidx) { if (add_to_txpkts(r->items[next_cidx], &txp, available) != 0) break; tail->m_nextpkt = r->items[next_cidx]; tail = tail->m_nextpkt; ETHER_BPF_MTAP(ifp, tail); if (__predict_false(++next_cidx == r->size)) next_cidx = 0; } n = write_txpkts_wr(txq, wr, m0, &txp, available); total += txp.npkt; remaining -= txp.npkt; } else { total++; remaining--; ETHER_BPF_MTAP(ifp, m0); n = write_txpkt_wr(txq, (void *)wr, m0, available); } MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC); available -= n; dbdiff += n; IDXINCR(eq->pidx, n, eq->sidx); if (total_available_tx_desc(eq) < eq->sidx / 4 && atomic_cmpset_int(&eq->equiq, 0, 1)) { wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } if (dbdiff >= 16 && remaining >= 4) { ring_eq_db(sc, eq, dbdiff); available += reclaim_tx_descs(txq, 4 * dbdiff); dbdiff = 0; } cidx = next_cidx; } if (dbdiff != 0) { ring_eq_db(sc, eq, dbdiff); reclaim_tx_descs(txq, 32); } done: TXQ_UNLOCK(txq); return (total); } static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, int qsize) { KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, ("%s: bad tmr_idx %d", __func__, tmr_idx)); KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ ("%s: bad pktc_idx %d", __func__, pktc_idx)); iq->flags = 0; iq->adapter = sc; iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); iq->intr_pktc_idx = SGE_NCOUNTERS - 1; if (pktc_idx >= 0) { iq->intr_params |= F_QINTR_CNT_EN; iq->intr_pktc_idx = pktc_idx; } iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; } static inline void init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) { fl->qsize = qsize; fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; strlcpy(fl->lockname, name, sizeof(fl->lockname)); if (sc->flags & BUF_PACKING_OK && ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ fl->flags |= FL_BUF_PACKING; find_best_refill_source(sc, fl, maxp); find_safe_refill_source(sc, fl); } static inline void init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, uint8_t tx_chan, uint16_t iqid, char *name) { KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); eq->flags = eqtype & EQ_TYPEMASK; eq->tx_chan = tx_chan; eq->iqid = iqid; eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; strlcpy(eq->lockname, name, sizeof(eq->lockname)); } static int alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, bus_dmamap_t *map, bus_addr_t *pa, void **va) { int rc; rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); if (rc != 0) { device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc); goto done; } rc = bus_dmamem_alloc(*tag, va, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); if (rc != 0) { device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc); goto done; } rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); if (rc != 0) { device_printf(sc->dev, "cannot load DMA map: %d\n", rc); goto done; } done: if (rc) free_ring(sc, *tag, *map, *pa, *va); return (rc); } static int free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, bus_addr_t pa, void *va) { if (pa) bus_dmamap_unload(tag, map); if (va) bus_dmamem_free(tag, va, map); if (tag) bus_dma_tag_destroy(tag); return (0); } /* * Allocates the ring for an ingress queue and an optional freelist. If the * freelist is specified it will be allocated and then associated with the * ingress queue. * * Returns errno on failure. Resources allocated up to that point may still be * allocated. Caller is responsible for cleanup in case this function fails. * * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then * the intr_idx specifies the vector, starting from 0. Otherwise it specifies * the abs_id of the ingress queue to which its interrupts should be forwarded. */ static int alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, int intr_idx, int cong) { int rc, i, cntxt_id; size_t len; struct fw_iq_cmd c; struct port_info *pi = vi->pi; struct adapter *sc = iq->adapter; struct sge_params *sp = &sc->params.sge; __be32 v = 0; len = iq->qsize * IQ_ESIZE; rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, (void **)&iq->desc); if (rc != 0) return (rc); bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | V_FW_IQ_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | FW_LEN16(c)); /* Special handling for firmware event queue */ if (iq == &sc->sge.fwq) v |= F_FW_IQ_CMD_IQASYNCH; if (iq->flags & IQ_INTR) { KASSERT(intr_idx < sc->intr_count, ("%s: invalid direct intr_idx %d", __func__, intr_idx)); } else v |= F_FW_IQ_CMD_IQANDST; v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); c.type_to_iqandstindex = htobe32(v | V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | V_FW_IQ_CMD_VIID(vi->viid) | V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); c.iqsize = htobe16(iq->qsize); c.iqaddr = htobe64(iq->ba); if (cong >= 0) c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); if (fl) { mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); len = fl->qsize * EQ_ESIZE; rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, &fl->ba, (void **)&fl->desc); if (rc) return (rc); /* Allocate space for one software descriptor per buffer. */ rc = alloc_fl_sdesc(fl); if (rc != 0) { device_printf(sc->dev, "failed to setup fl software descriptors: %d\n", rc); return (rc); } if (fl->flags & FL_BUF_PACKING) { fl->lowat = roundup2(sp->fl_starve_threshold2, 8); fl->buf_boundary = sp->pack_boundary; } else { fl->lowat = roundup2(sp->fl_starve_threshold, 8); fl->buf_boundary = 16; } if (fl_pad && fl->buf_boundary < sp->pad_boundary) fl->buf_boundary = sp->pad_boundary; c.iqns_to_fl0congen |= htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 0)); if (cong >= 0) { c.iqns_to_fl0congen |= htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | F_FW_IQ_CMD_FL0CONGCIF | F_FW_IQ_CMD_FL0CONGEN); } c.fl0dcaen_to_fl0cidxfthresh = htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) | V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); c.fl0size = htobe16(fl->qsize); c.fl0addr = htobe64(fl->ba); } rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(sc->dev, "failed to create ingress queue: %d\n", rc); return (rc); } iq->cidx = 0; iq->gen = F_RSPD_GEN; iq->intr_next = iq->intr_params; iq->cntxt_id = be16toh(c.iqid); iq->abs_id = be16toh(c.physiqid); iq->flags |= IQ_ALLOCATED; cntxt_id = iq->cntxt_id - sc->sge.iq_start; if (cntxt_id >= sc->sge.niq) { panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.niq - 1); } sc->sge.iqmap[cntxt_id] = iq; if (fl) { u_int qid; iq->flags |= IQ_HAS_FL; fl->cntxt_id = be16toh(c.fl0id); fl->pidx = fl->cidx = 0; cntxt_id = fl->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) { panic("%s: fl->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); } sc->sge.eqmap[cntxt_id] = (void *)fl; qid = fl->cntxt_id; if (isset(&sc->doorbells, DOORBELL_UDB)) { uint32_t s_qpp = sc->params.sge.eq_s_qpp; uint32_t mask = (1 << s_qpp) - 1; volatile uint8_t *udb; udb = sc->udbs_base + UDBS_DB_OFFSET; udb += (qid >> s_qpp) << PAGE_SHIFT; qid &= mask; if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { udb += qid << UDBS_SEG_SHIFT; qid = 0; } fl->udb = (volatile void *)udb; } fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; FL_LOCK(fl); /* Enough to make sure the SGE doesn't think it's starved */ refill_fl(sc, fl, fl->lowat); FL_UNLOCK(fl); } if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) { uint32_t param, val; param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); if (cong == 0) val = 1 << 19; else { val = 2 << 19; for (i = 0; i < 4; i++) { if (cong & (1 << i)) val |= 1 << (i << 2); } } rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc != 0) { /* report error but carry on */ device_printf(sc->dev, "failed to set congestion manager context for " "ingress queue %d: %d\n", iq->cntxt_id, rc); } } /* Enable IQ interrupts */ atomic_store_rel_int(&iq->state, IQS_IDLE); t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); return (0); } static int free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) { int rc; struct adapter *sc = iq->adapter; device_t dev; if (sc == NULL) return (0); /* nothing to do */ dev = vi ? vi->dev : sc->dev; if (iq->flags & IQ_ALLOCATED) { rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); if (rc != 0) { device_printf(dev, "failed to free queue %p: %d\n", iq, rc); return (rc); } iq->flags &= ~IQ_ALLOCATED; } free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); bzero(iq, sizeof(*iq)); if (fl) { free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc); if (fl->sdesc) free_fl_sdesc(sc, fl); if (mtx_initialized(&fl->fl_lock)) mtx_destroy(&fl->fl_lock); bzero(fl, sizeof(*fl)); } return (0); } static void add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, struct sge_fl *fl) { struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, "freelist"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &fl->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the freelist"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, fl_pad ? 1 : 0, "padding enabled"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 0, "consumer index"); if (fl->flags & FL_BUF_PACKING) { SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); } SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 0, "producer index"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_allocated", CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_inlined", CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); } static int alloc_fwq(struct adapter *sc) { int rc, intr_idx; struct sge_iq *fwq = &sc->sge.fwq; struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); fwq->flags |= IQ_INTR; /* always */ if (sc->flags & IS_VF) intr_idx = 0; else { intr_idx = sc->intr_count > 1 ? 1 : 0; fwq->set_tcb_rpl = t4_filter_rpl; fwq->l2t_write_rpl = do_l2t_write_rpl; } rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); if (rc != 0) { device_printf(sc->dev, "failed to create firmware event queue: %d\n", rc); return (rc); } oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD, NULL, "firmware event queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(&sc->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &fwq->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(&sc->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, fwq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I", "consumer index"); return (0); } static int free_fwq(struct adapter *sc) { return free_iq_fl(NULL, &sc->sge.fwq, NULL); } static int alloc_mgmtq(struct adapter *sc) { int rc; struct sge_wrq *mgmtq = &sc->sge.mgmtq; char name[16]; struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD, NULL, "management queue"); snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev)); init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, sc->sge.fwq.cntxt_id, name); rc = alloc_wrq(sc, NULL, mgmtq, oid); if (rc != 0) { device_printf(sc->dev, "failed to create management queue: %d\n", rc); return (rc); } return (0); } static int free_mgmtq(struct adapter *sc) { return free_wrq(sc, &sc->sge.mgmtq); } int tnl_cong(struct port_info *pi, int drop) { if (drop == -1) return (-1); else if (drop == 1) return (0); else return (pi->rx_chan_map); } static int alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, struct sysctl_oid *oid) { int rc; struct adapter *sc = vi->pi->adapter; struct sysctl_oid_list *children; char name[16]; rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(vi->pi, cong_drop)); if (rc != 0) return (rc); if (idx == 0) sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; else KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, ("iq_base mismatch")); KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, ("PF with non-zero iq_base")); /* * The freelist is just barely above the starvation threshold right now, * fill it up a bit more. */ FL_LOCK(&rxq->fl); refill_fl(sc, &rxq->fl, 128); FL_UNLOCK(&rxq->fl); #if defined(INET) || defined(INET6) rc = tcp_lro_init_args(&rxq->lro, vi->ifp, lro_entries, lro_mbufs); if (rc != 0) return (rc); MPASS(rxq->lro.ifp == vi->ifp); /* also indicates LRO init'ed */ if (vi->ifp->if_capenable & IFCAP_LRO) rxq->iq.flags |= IQ_LRO_ENABLED; #endif rxq->ifp = vi->ifp; children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &rxq->iq.ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, rxq->iq.qsize * IQ_ESIZE, "descriptor ring size in bytes"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cidx, 0, sysctl_uint16, "I", "consumer index"); #if defined(INET) || defined(INET6) SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, &rxq->lro.lro_queued, 0, NULL); SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, &rxq->lro.lro_flushed, 0, NULL); #endif SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, &rxq->rxcsum, "# of times hardware assisted with checksum"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD, &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag"); add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl); return (rc); } static int free_rxq(struct vi_info *vi, struct sge_rxq *rxq) { int rc; #if defined(INET) || defined(INET6) if (rxq->lro.ifp) { tcp_lro_free(&rxq->lro); rxq->lro.ifp = NULL; } #endif rc = free_iq_fl(vi, &rxq->iq, &rxq->fl); if (rc == 0) bzero(rxq, sizeof(*rxq)); return (rc); } #ifdef TCP_OFFLOAD static int alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int intr_idx, int idx, struct sysctl_oid *oid) { struct port_info *pi = vi->pi; int rc; struct sysctl_oid_list *children; char name[16]; rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, pi->rx_chan_map); if (rc != 0) return (rc); children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &ofld_rxq->iq.ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, ofld_rxq->iq.qsize * IQ_ESIZE, "descriptor ring size in bytes"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I", "consumer index"); add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl); return (rc); } static int free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) { int rc; rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl); if (rc == 0) bzero(ofld_rxq, sizeof(*ofld_rxq)); return (rc); } #endif #ifdef DEV_NETMAP static int alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx, int idx, struct sysctl_oid *oid) { int rc; struct sysctl_oid_list *children; struct sysctl_ctx_list *ctx; char name[16]; size_t len; struct adapter *sc = vi->pi->adapter; struct netmap_adapter *na = NA(vi->ifp); MPASS(na != NULL); len = vi->qsize_rxq * IQ_ESIZE; rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map, &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc); if (rc != 0) return (rc); len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len; rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map, &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc); if (rc != 0) return (rc); nm_rxq->vi = vi; nm_rxq->nid = idx; nm_rxq->iq_cidx = 0; nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE; nm_rxq->iq_gen = F_RSPD_GEN; nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0; nm_rxq->fl_sidx = na->num_rx_desc; nm_rxq->intr_idx = intr_idx; nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID; ctx = &vi->ctx; children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I", "consumer index"); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, "freelist"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the freelist"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &nm_rxq->fl_cidx, 0, "consumer index"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &nm_rxq->fl_pidx, 0, "producer index"); return (rc); } static int free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) { struct adapter *sc = vi->pi->adapter; MPASS(nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID); free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba, nm_rxq->iq_desc); free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba, nm_rxq->fl_desc); return (0); } static int alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx, struct sysctl_oid *oid) { int rc; size_t len; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct netmap_adapter *na = NA(vi->ifp); char name[16]; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len; rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map, &nm_txq->ba, (void **)&nm_txq->desc); if (rc) return (rc); nm_txq->pidx = nm_txq->cidx = 0; nm_txq->sidx = na->num_tx_desc; nm_txq->nid = idx; nm_txq->iqidx = iqidx; nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); nm_txq->cntxt_id = INVALID_NM_TXQ_CNTXT_ID; snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "netmap tx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &nm_txq->cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I", "producer index"); return (rc); } static int free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq) { struct adapter *sc = vi->pi->adapter; MPASS(nm_txq->cntxt_id == INVALID_NM_TXQ_CNTXT_ID); free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba, nm_txq->desc); return (0); } #endif static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_ctrl_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | V_FW_EQ_CTRL_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); c.physeqid_pkd = htobe32(0); c.fetchszm_to_iqid = htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(sc->dev, "failed to create control queue %d: %d\n", eq->tx_chan, rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } static int eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_eth_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | V_FW_EQ_ETH_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); c.fetchszm_to_iqid = htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | V_FW_EQ_ETH_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_ETH_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(vi->dev, "failed to create Ethernet egress queue: %d\n", rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } #ifdef TCP_OFFLOAD static int ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_ofld_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | V_FW_EQ_OFLD_CMD_VFN(0)); c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); c.fetchszm_to_iqid = htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(vi->dev, "failed to create egress queue for TCP offload: %d\n", rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } #endif static int alloc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, qsize; size_t len; mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; len = qsize * EQ_ESIZE; rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba, (void **)&eq->desc); if (rc) return (rc); eq->pidx = eq->cidx = 0; eq->equeqidx = eq->dbidx = 0; eq->doorbells = sc->doorbells; switch (eq->flags & EQ_TYPEMASK) { case EQ_CTRL: rc = ctrl_eq_alloc(sc, eq); break; case EQ_ETH: rc = eth_eq_alloc(sc, vi, eq); break; #ifdef TCP_OFFLOAD case EQ_OFLD: rc = ofld_eq_alloc(sc, vi, eq); break; #endif default: panic("%s: invalid eq type %d.", __func__, eq->flags & EQ_TYPEMASK); } if (rc != 0) { device_printf(sc->dev, "failed to allocate egress queue(%d): %d\n", eq->flags & EQ_TYPEMASK, rc); } if (isset(&eq->doorbells, DOORBELL_UDB) || isset(&eq->doorbells, DOORBELL_UDBWC) || isset(&eq->doorbells, DOORBELL_WCWR)) { uint32_t s_qpp = sc->params.sge.eq_s_qpp; uint32_t mask = (1 << s_qpp) - 1; volatile uint8_t *udb; udb = sc->udbs_base + UDBS_DB_OFFSET; udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ eq->udb_qid = eq->cntxt_id & mask; /* id in page */ if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) clrbit(&eq->doorbells, DOORBELL_WCWR); else { udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ eq->udb_qid = 0; } eq->udb = (volatile void *)udb; } return (rc); } static int free_eq(struct adapter *sc, struct sge_eq *eq) { int rc; if (eq->flags & EQ_ALLOCATED) { switch (eq->flags & EQ_TYPEMASK) { case EQ_CTRL: rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; case EQ_ETH: rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; #ifdef TCP_OFFLOAD case EQ_OFLD: rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; #endif default: panic("%s: invalid eq type %d.", __func__, eq->flags & EQ_TYPEMASK); } if (rc != 0) { device_printf(sc->dev, "failed to free egress queue (%d): %d\n", eq->flags & EQ_TYPEMASK, rc); return (rc); } eq->flags &= ~EQ_ALLOCATED; } free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); if (mtx_initialized(&eq->eq_lock)) mtx_destroy(&eq->eq_lock); bzero(eq, sizeof(*eq)); return (0); } static int alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, struct sysctl_oid *oid) { int rc; struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); rc = alloc_eq(sc, vi, &wrq->eq); if (rc) return (rc); wrq->adapter = sc; TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); TAILQ_INIT(&wrq->incomplete_wrs); STAILQ_INIT(&wrq->wr_list); wrq->nwr_pending = 0; wrq->ndesc_needed = 0; SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &wrq->eq.ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, wrq->eq.sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I", "producer index"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, wrq->eq.sidx, "status page index"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, &wrq->tx_wrs_direct, "# of work requests (direct)"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, &wrq->tx_wrs_copied, "# of work requests (copied)"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); return (rc); } static int free_wrq(struct adapter *sc, struct sge_wrq *wrq) { int rc; rc = free_eq(sc, &wrq->eq); if (rc) return (rc); bzero(wrq, sizeof(*wrq)); return (0); } static int alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, struct sysctl_oid *oid) { int rc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_eq *eq = &txq->eq; char name[16]; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, M_CXGBE, M_WAITOK); if (rc != 0) { device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); return (rc); } rc = alloc_eq(sc, vi, eq); if (rc != 0) { mp_ring_free(txq->r); txq->r = NULL; return (rc); } /* Can't fail after this point. */ if (idx == 0) sc->sge.eq_base = eq->abs_id - eq->cntxt_id; else KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, ("eq_base mismatch")); KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, ("PF with non-zero eq_base")); TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); txq->ifp = vi->ifp; txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); if (sc->flags & IS_VF) txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | V_TXPKT_INTF(pi->tx_chan)); else txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); txq->tc_idx = -1; txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, M_ZERO | M_WAITOK); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "tx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, &eq->abs_id, 0, "absolute id of the queue"); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &eq->cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I", "producer index"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, eq->sidx, "status page index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc", CTLTYPE_INT | CTLFLAG_RW, vi, idx, sysctl_tc, "I", "traffic class (-1 means none)"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, &txq->txcsum, "# of times hardware assisted with checksum"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD, &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, &txq->tso_wrs, "# of TSO work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, &txq->imm_wrs, "# of work requests with immediate data"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, &txq->sgl_wrs, "# of work requests with direct SGL"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD, &txq->txpkts0_wrs, "# of txpkts (type 0) work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD, &txq->txpkts1_wrs, "# of txpkts (type 1) work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD, &txq->txpkts0_pkts, "# of frames tx'd using type0 txpkts work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD, &txq->txpkts1_pkts, "# of frames tx'd using type1 txpkts work requests"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues", CTLFLAG_RD, &txq->r->enqueues, "# of enqueues to the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops", CTLFLAG_RD, &txq->r->drops, "# of drops in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts", CTLFLAG_RD, &txq->r->starts, "# of normal consumer starts in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_stalls", CTLFLAG_RD, &txq->r->stalls, "# of consumer stalls in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_restarts", CTLFLAG_RD, &txq->r->restarts, "# of consumer restarts in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_abdications", CTLFLAG_RD, &txq->r->abdications, "# of consumer abdications in the mp_ring for this queue"); return (0); } static int free_txq(struct vi_info *vi, struct sge_txq *txq) { int rc; struct adapter *sc = vi->pi->adapter; struct sge_eq *eq = &txq->eq; rc = free_eq(sc, eq); if (rc) return (rc); sglist_free(txq->gl); free(txq->sdesc, M_CXGBE); mp_ring_free(txq->r); bzero(txq, sizeof(*txq)); return (0); } static void oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *ba = arg; KASSERT(nseg == 1, ("%s meant for single segment mappings only.", __func__)); *ba = error ? 0 : segs->ds_addr; } static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl) { uint32_t n, v; n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx); MPASS(n > 0); wmb(); v = fl->dbval | V_PIDX(n); if (fl->udb) *fl->udb = htole32(v); else t4_write_reg(sc, sc->sge_kdoorbell_reg, v); IDXINCR(fl->dbidx, n, fl->sidx); } /* * Fills up the freelist by allocating up to 'n' buffers. Buffers that are * recycled do not count towards this allocation budget. * * Returns non-zero to indicate that this freelist should be added to the list * of starving freelists. */ static int refill_fl(struct adapter *sc, struct sge_fl *fl, int n) { __be64 *d; struct fl_sdesc *sd; uintptr_t pa; caddr_t cl; struct cluster_layout *cll; struct sw_zone_info *swz; struct cluster_metadata *clm; uint16_t max_pidx; uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ FL_LOCK_ASSERT_OWNED(fl); /* * We always stop at the beginning of the hardware descriptor that's just * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, * which would mean an empty freelist to the chip. */ max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; if (fl->pidx == max_pidx * 8) return (0); d = &fl->desc[fl->pidx]; sd = &fl->sdesc[fl->pidx]; cll = &fl->cll_def; /* default layout */ swz = &sc->sge.sw_zone_info[cll->zidx]; while (n > 0) { if (sd->cl != NULL) { if (sd->nmbuf == 0) { /* * Fast recycle without involving any atomics on * the cluster's metadata (if the cluster has * metadata). This happens when all frames * received in the cluster were small enough to * fit within a single mbuf each. */ fl->cl_fast_recycled++; #ifdef INVARIANTS clm = cl_metadata(sc, fl, &sd->cll, sd->cl); if (clm != NULL) MPASS(clm->refcount == 1); #endif goto recycled_fast; } /* * Cluster is guaranteed to have metadata. Clusters * without metadata always take the fast recycle path * when they're recycled. */ clm = cl_metadata(sc, fl, &sd->cll, sd->cl); MPASS(clm != NULL); if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { fl->cl_recycled++; counter_u64_add(extfree_rels, 1); goto recycled; } sd->cl = NULL; /* gave up my reference */ } MPASS(sd->cl == NULL); alloc: cl = uma_zalloc(swz->zone, M_NOWAIT); if (__predict_false(cl == NULL)) { if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 || fl->cll_def.zidx == fl->cll_alt.zidx) break; /* fall back to the safe zone */ cll = &fl->cll_alt; swz = &sc->sge.sw_zone_info[cll->zidx]; goto alloc; } fl->cl_allocated++; n--; pa = pmap_kextract((vm_offset_t)cl); pa += cll->region1; sd->cl = cl; sd->cll = *cll; *d = htobe64(pa | cll->hwidx); clm = cl_metadata(sc, fl, cll, cl); if (clm != NULL) { recycled: #ifdef INVARIANTS clm->sd = sd; #endif clm->refcount = 1; } sd->nmbuf = 0; recycled_fast: d++; sd++; if (__predict_false(++fl->pidx % 8 == 0)) { uint16_t pidx = fl->pidx / 8; if (__predict_false(pidx == fl->sidx)) { fl->pidx = 0; pidx = 0; sd = fl->sdesc; d = fl->desc; } if (pidx == max_pidx) break; if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) ring_fl_db(sc, fl); } } if (fl->pidx / 8 != fl->dbidx) ring_fl_db(sc, fl); return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); } /* * Attempt to refill all starving freelists. */ static void refill_sfl(void *arg) { struct adapter *sc = arg; struct sge_fl *fl, *fl_temp; mtx_assert(&sc->sfl_lock, MA_OWNED); TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { FL_LOCK(fl); refill_fl(sc, fl, 64); if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { TAILQ_REMOVE(&sc->sfl, fl, link); fl->flags &= ~FL_STARVING; } FL_UNLOCK(fl); } if (!TAILQ_EMPTY(&sc->sfl)) callout_schedule(&sc->sfl_callout, hz / 5); } static int alloc_fl_sdesc(struct sge_fl *fl) { fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE, M_ZERO | M_WAITOK); return (0); } static void free_fl_sdesc(struct adapter *sc, struct sge_fl *fl) { struct fl_sdesc *sd; struct cluster_metadata *clm; struct cluster_layout *cll; int i; sd = fl->sdesc; for (i = 0; i < fl->sidx * 8; i++, sd++) { if (sd->cl == NULL) continue; cll = &sd->cll; clm = cl_metadata(sc, fl, cll, sd->cl); if (sd->nmbuf == 0) uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) { uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); counter_u64_add(extfree_rels, 1); } sd->cl = NULL; } free(fl->sdesc, M_CXGBE); fl->sdesc = NULL; } static inline void get_pkt_gl(struct mbuf *m, struct sglist *gl) { int rc; M_ASSERTPKTHDR(m); sglist_reset(gl); rc = sglist_append_mbuf(gl, m); if (__predict_false(rc != 0)) { panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " "with %d.", __func__, m, mbuf_nsegs(m), rc); } KASSERT(gl->sg_nseg == mbuf_nsegs(m), ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, mbuf_nsegs(m), gl->sg_nseg)); KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS), ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)); } /* * len16 for a txpkt WR with a GL. Includes the firmware work request header. */ static inline u_int txpkt_len16(u_int nsegs, u_int tso) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); if (tso) n += sizeof(struct cpl_tx_pkt_lso_core); return (howmany(n, 16)); } /* * len16 for a txpkt_vm WR with a GL. Includes the firmware work * request header. */ static inline u_int txpkt_vm_len16(u_int nsegs, u_int tso) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct fw_eth_tx_pkt_vm_wr) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); if (tso) n += sizeof(struct cpl_tx_pkt_lso_core); return (howmany(n, 16)); } /* * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work * request header. */ static inline u_int txpkts0_len16(u_int nsegs) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); return (howmany(n, 16)); } /* * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work * request header. */ static inline u_int txpkts1_len16(void) { u_int n; n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); return (howmany(n, 16)); } static inline u_int imm_payload(u_int ndesc) { u_int n; n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - sizeof(struct cpl_tx_pkt_core); return (n); } /* * Write a VM txpkt WR for this packet to the hardware descriptors, update the * software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct fw_eth_tx_pkt_vm_wr *wr, struct mbuf *m0, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; /* used in many unrelated places */ uint64_t ctrl1; int csum_type, len16, ndesc, pktlen, nsegs; caddr_t dst; TXQ_LOCK_ASSERT_OWNED(txq); M_ASSERTPKTHDR(m0); MPASS(available > 0 && available < eq->sidx); len16 = mbuf_len16(m0); nsegs = mbuf_nsegs(m0); pktlen = m0->m_pkthdr.len; ctrl = sizeof(struct cpl_tx_pkt_core); if (needs_tso(m0)) ctrl += sizeof(struct cpl_tx_pkt_lso_core); ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc <= available); /* Firmware work request header */ MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); ctrl = V_FW_WR_LEN16(len16); wr->equiq_to_len16 = htobe32(ctrl); wr->r3[0] = 0; wr->r3[1] = 0; /* * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. * vlantci is ignored unless the ethtype is 0x8100, so it's * simpler to always copy it rather than making it * conditional. Also, it seems that we do not have to set * vlantci or fake the ethtype when doing VLAN tag insertion. */ m_copydata(m0, 0, sizeof(struct ether_header) + 2, wr->ethmacdst); csum_type = -1; if (needs_tso(m0)) { struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && m0->m_pkthdr.l4hlen > 0, ("%s: mbuf %p needs TSO but missing header lengths", __func__, m0)); ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) ctrl |= V_LSO_ETHHDR_LEN(1); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) ctrl |= F_LSO_IPV6; lso->lso_ctrl = htobe32(ctrl); lso->ipid_ofst = htobe16(0); lso->mss = htobe16(m0->m_pkthdr.tso_segsz); lso->seqno_offset = htobe32(0); lso->len = htobe32(pktlen); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) csum_type = TX_CSUM_TCPIP6; else csum_type = TX_CSUM_TCPIP; cpl = (void *)(lso + 1); txq->tso_wrs++; } else { if (m0->m_pkthdr.csum_flags & CSUM_IP_TCP) csum_type = TX_CSUM_TCPIP; else if (m0->m_pkthdr.csum_flags & CSUM_IP_UDP) csum_type = TX_CSUM_UDPIP; else if (m0->m_pkthdr.csum_flags & CSUM_IP6_TCP) csum_type = TX_CSUM_TCPIP6; else if (m0->m_pkthdr.csum_flags & CSUM_IP6_UDP) csum_type = TX_CSUM_UDPIP6; #if defined(INET) else if (m0->m_pkthdr.csum_flags & CSUM_IP) { /* * XXX: The firmware appears to stomp on the * fragment/flags field of the IP header when * using TX_CSUM_IP. Fall back to doing * software checksums. */ u_short *sump; struct mbuf *m; int offset; m = m0; offset = 0; sump = m_advance(&m, &offset, m0->m_pkthdr.l2hlen + offsetof(struct ip, ip_sum)); *sump = in_cksum_skip(m0, m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen, m0->m_pkthdr.l2hlen); m0->m_pkthdr.csum_flags &= ~CSUM_IP; } #endif cpl = (void *)(wr + 1); } /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m0) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (csum_type >= 0) { KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0, ("%s: mbuf %p needs checksum offload but missing header lengths", __func__, m0)); if (chip_id(sc) <= CHELSIO_T5) { ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN); } else { ctrl1 |= V_T6_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN); } ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen); ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type); } else ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m0)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(pktlen); cpl->ctrl1 = htobe64(ctrl1); /* SGL */ dst = (void *)(cpl + 1); /* * A packet using TSO will use up an entire descriptor for the * firmware work request header, LSO CPL, and TX_PKT_XT CPL. * If this descriptor is the last descriptor in the ring, wrap * around to the front of the ring explicitly for the start of * the sgl. */ if (dst == (void *)&eq->desc[eq->sidx]) { dst = (void *)&eq->desc[0]; write_gl_to_txd(txq, m0, &dst, 0); } else write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); txq->sgl_wrs++; txq->txpkt_wrs++; txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } /* * Write a txpkt WR for this packet to the hardware descriptors, update the * software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr, struct mbuf *m0, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; /* used in many unrelated places */ uint64_t ctrl1; int len16, ndesc, pktlen, nsegs; caddr_t dst; TXQ_LOCK_ASSERT_OWNED(txq); M_ASSERTPKTHDR(m0); MPASS(available > 0 && available < eq->sidx); len16 = mbuf_len16(m0); nsegs = mbuf_nsegs(m0); pktlen = m0->m_pkthdr.len; ctrl = sizeof(struct cpl_tx_pkt_core); if (needs_tso(m0)) ctrl += sizeof(struct cpl_tx_pkt_lso_core); else if (pktlen <= imm_payload(2) && available >= 2) { /* Immediate data. Recalculate len16 and set nsegs to 0. */ ctrl += pktlen; len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + pktlen, 16); nsegs = 0; } ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc <= available); /* Firmware work request header */ MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); ctrl = V_FW_WR_LEN16(len16); wr->equiq_to_len16 = htobe32(ctrl); wr->r3 = 0; if (needs_tso(m0)) { struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && m0->m_pkthdr.l4hlen > 0, ("%s: mbuf %p needs TSO but missing header lengths", __func__, m0)); ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) ctrl |= V_LSO_ETHHDR_LEN(1); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) ctrl |= F_LSO_IPV6; lso->lso_ctrl = htobe32(ctrl); lso->ipid_ofst = htobe16(0); lso->mss = htobe16(m0->m_pkthdr.tso_segsz); lso->seqno_offset = htobe32(0); lso->len = htobe32(pktlen); cpl = (void *)(lso + 1); txq->tso_wrs++; } else cpl = (void *)(wr + 1); /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m0) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (needs_l4_csum(m0) == 0) ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m0)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(pktlen); cpl->ctrl1 = htobe64(ctrl1); /* SGL */ dst = (void *)(cpl + 1); if (nsegs > 0) { write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); txq->sgl_wrs++; } else { struct mbuf *m; for (m = m0; m != NULL; m = m->m_next) { copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); #ifdef INVARIANTS pktlen -= m->m_len; #endif } #ifdef INVARIANTS KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); #endif txq->imm_wrs++; } txq->txpkt_wrs++; txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } static int try_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available) { u_int needed, nsegs1, nsegs2, l1, l2; if (cannot_use_txpkts(m) || cannot_use_txpkts(n)) return (1); nsegs1 = mbuf_nsegs(m); nsegs2 = mbuf_nsegs(n); if (nsegs1 + nsegs2 == 2) { txp->wr_type = 1; l1 = l2 = txpkts1_len16(); } else { txp->wr_type = 0; l1 = txpkts0_len16(nsegs1); l2 = txpkts0_len16(nsegs2); } txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2; needed = howmany(txp->len16, EQ_ESIZE / 16); if (needed > SGE_MAX_WR_NDESC || needed > available) return (1); txp->plen = m->m_pkthdr.len + n->m_pkthdr.len; if (txp->plen > 65535) return (1); txp->npkt = 2; set_mbuf_len16(m, l1); set_mbuf_len16(n, l2); return (0); } static int add_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available) { u_int plen, len16, needed, nsegs; MPASS(txp->wr_type == 0 || txp->wr_type == 1); nsegs = mbuf_nsegs(m); if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1)) return (1); plen = txp->plen + m->m_pkthdr.len; if (plen > 65535) return (1); if (txp->wr_type == 0) len16 = txpkts0_len16(nsegs); else len16 = txpkts1_len16(); needed = howmany(txp->len16 + len16, EQ_ESIZE / 16); if (needed > SGE_MAX_WR_NDESC || needed > available) return (1); txp->npkt++; txp->plen = plen; txp->len16 += len16; set_mbuf_len16(m, len16); return (0); } /* * Write a txpkts WR for the packets in txp to the hardware descriptors, update * the software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr, struct mbuf *m0, const struct txpkts *txp, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; uint64_t ctrl1; int ndesc, checkwrap; struct mbuf *m; void *flitp; TXQ_LOCK_ASSERT_OWNED(txq); MPASS(txp->npkt > 0); MPASS(txp->plen < 65536); MPASS(m0 != NULL); MPASS(m0->m_nextpkt != NULL); MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); MPASS(available > 0 && available < eq->sidx); ndesc = howmany(txp->len16, EQ_ESIZE / 16); MPASS(ndesc <= available); MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); ctrl = V_FW_WR_LEN16(txp->len16); wr->equiq_to_len16 = htobe32(ctrl); wr->plen = htobe16(txp->plen); wr->npkt = txp->npkt; wr->r3 = 0; wr->type = txp->wr_type; flitp = wr + 1; /* * At this point we are 16B into a hardware descriptor. If checkwrap is * set then we know the WR is going to wrap around somewhere. We'll * check for that at appropriate points. */ checkwrap = eq->sidx - ndesc < eq->pidx; for (m = m0; m != NULL; m = m->m_nextpkt) { if (txp->wr_type == 0) { struct ulp_txpkt *ulpmc; struct ulptx_idata *ulpsc; /* ULP master command */ ulpmc = flitp; ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); ulpmc->len = htobe32(mbuf_len16(m)); /* ULP subcommand */ ulpsc = (void *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | F_ULP_TX_SC_MORE); ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); cpl = (void *)(ulpsc + 1); if (checkwrap && (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) cpl = (void *)&eq->desc[0]; } else { cpl = flitp; } /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (needs_l4_csum(m) == 0) ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(m->m_pkthdr.len); cpl->ctrl1 = htobe64(ctrl1); flitp = cpl + 1; if (checkwrap && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) flitp = (void *)&eq->desc[0]; write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); } if (txp->wr_type == 0) { txq->txpkts0_pkts += txp->npkt; txq->txpkts0_wrs++; } else { txq->txpkts1_pkts += txp->npkt; txq->txpkts1_wrs++; } txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } /* * If the SGL ends on an address that is not 16 byte aligned, this function will * add a 0 filled flit at the end. */ static void write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) { struct sge_eq *eq = &txq->eq; struct sglist *gl = txq->gl; struct sglist_seg *seg; __be64 *flitp, *wrap; struct ulptx_sgl *usgl; int i, nflits, nsegs; KASSERT(((uintptr_t)(*to) & 0xf) == 0, ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); get_pkt_gl(m, gl); nsegs = gl->sg_nseg; MPASS(nsegs > 0); nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; flitp = (__be64 *)(*to); wrap = (__be64 *)(&eq->desc[eq->sidx]); seg = &gl->sg_segs[0]; usgl = (void *)flitp; /* * We start at a 16 byte boundary somewhere inside the tx descriptor * ring, so we're at least 16 bytes away from the status page. There is * no chance of a wrap around in the middle of usgl (which is 16 bytes). */ usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); usgl->len0 = htobe32(seg->ss_len); usgl->addr0 = htobe64(seg->ss_paddr); seg++; if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { /* Won't wrap around at all */ for (i = 0; i < nsegs - 1; i++, seg++) { usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); } if (i & 1) usgl->sge[i / 2].len[1] = htobe32(0); flitp += nflits; } else { /* Will wrap somewhere in the rest of the SGL */ /* 2 flits already written, write the rest flit by flit */ flitp = (void *)(usgl + 1); for (i = 0; i < nflits - 2; i++) { if (flitp == wrap) flitp = (void *)eq->desc; *flitp++ = get_flit(seg, nsegs - 1, i); } } if (nflits & 1) { MPASS(((uintptr_t)flitp) & 0xf); *flitp++ = 0; } MPASS((((uintptr_t)flitp) & 0xf) == 0); if (__predict_false(flitp == wrap)) *to = (void *)eq->desc; else *to = (void *)flitp; } static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) { MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)&eq->desc[eq->sidx])) { bcopy(from, *to, len); (*to) += len; } else { int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); bcopy(from, *to, portion); from += portion; portion = len - portion; /* remaining */ bcopy(from, (void *)eq->desc, portion); (*to) = (caddr_t)eq->desc + portion; } } static inline void ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) { u_int db; MPASS(n > 0); db = eq->doorbells; if (n > 1) clrbit(&db, DOORBELL_WCWR); wmb(); switch (ffs(db) - 1) { case DOORBELL_UDB: *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); break; case DOORBELL_WCWR: { volatile uint64_t *dst, *src; int i; /* * Queues whose 128B doorbell segment fits in the page do not * use relative qid (udb_qid is always 0). Only queues with * doorbell segments can do WCWR. */ KASSERT(eq->udb_qid == 0 && n == 1, ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", __func__, eq->doorbells, n, eq->dbidx, eq)); dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - UDBS_DB_OFFSET); i = eq->dbidx; src = (void *)&eq->desc[i]; while (src != (void *)&eq->desc[i + 1]) *dst++ = *src++; wmb(); break; } case DOORBELL_UDBWC: *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); wmb(); break; case DOORBELL_KDB: t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(eq->cntxt_id) | V_PIDX(n)); break; } IDXINCR(eq->dbidx, n, eq->sidx); } static inline u_int reclaimable_tx_desc(struct sge_eq *eq) { uint16_t hw_cidx; hw_cidx = read_hw_cidx(eq); return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); } static inline u_int total_available_tx_desc(struct sge_eq *eq) { uint16_t hw_cidx, pidx; hw_cidx = read_hw_cidx(eq); pidx = eq->pidx; if (pidx == hw_cidx) return (eq->sidx - 1); else return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); } static inline uint16_t read_hw_cidx(struct sge_eq *eq) { struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; uint16_t cidx = spg->cidx; /* stable snapshot */ return (be16toh(cidx)); } /* * Reclaim 'n' descriptors approximately. */ static u_int reclaim_tx_descs(struct sge_txq *txq, u_int n) { struct tx_sdesc *txsd; struct sge_eq *eq = &txq->eq; u_int can_reclaim, reclaimed; TXQ_LOCK_ASSERT_OWNED(txq); MPASS(n > 0); reclaimed = 0; can_reclaim = reclaimable_tx_desc(eq); while (can_reclaim && reclaimed < n) { int ndesc; struct mbuf *m, *nextpkt; txsd = &txq->sdesc[eq->cidx]; ndesc = txsd->desc_used; /* Firmware doesn't return "partial" credits. */ KASSERT(can_reclaim >= ndesc, ("%s: unexpected number of credits: %d, %d", __func__, can_reclaim, ndesc)); for (m = txsd->m; m != NULL; m = nextpkt) { nextpkt = m->m_nextpkt; m->m_nextpkt = NULL; m_freem(m); } reclaimed += ndesc; can_reclaim -= ndesc; IDXINCR(eq->cidx, ndesc, eq->sidx); } return (reclaimed); } static void tx_reclaim(void *arg, int n) { struct sge_txq *txq = arg; struct sge_eq *eq = &txq->eq; do { if (TXQ_TRYLOCK(txq) == 0) break; n = reclaim_tx_descs(txq, 32); if (eq->cidx == eq->pidx) eq->equeqidx = eq->pidx; TXQ_UNLOCK(txq); } while (n > 0); } static __be64 get_flit(struct sglist_seg *segs, int nsegs, int idx) { int i = (idx / 3) * 2; switch (idx % 3) { case 0: { __be64 rc; rc = htobe32(segs[i].ss_len); if (i + 1 < nsegs) rc |= (uint64_t)htobe32(segs[i + 1].ss_len) << 32; return (rc); } case 1: return (htobe64(segs[i].ss_paddr)); case 2: return (htobe64(segs[i + 1].ss_paddr)); } return (0); } static void find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp) { int8_t zidx, hwidx, idx; uint16_t region1, region3; int spare, spare_needed, n; struct sw_zone_info *swz; struct hw_buf_info *hwb, *hwb_list = &sc->sge.hw_buf_info[0]; /* * Buffer Packing: Look for PAGE_SIZE or larger zone which has a bufsize * large enough for the max payload and cluster metadata. Otherwise * settle for the largest bufsize that leaves enough room in the cluster * for metadata. * * Without buffer packing: Look for the smallest zone which has a * bufsize large enough for the max payload. Settle for the largest * bufsize available if there's nothing big enough for max payload. */ spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0; swz = &sc->sge.sw_zone_info[0]; hwidx = -1; for (zidx = 0; zidx < SW_ZONE_SIZES; zidx++, swz++) { if (swz->size > largest_rx_cluster) { if (__predict_true(hwidx != -1)) break; /* * This is a misconfiguration. largest_rx_cluster is * preventing us from finding a refill source. See * dev.t5nex..buffer_sizes to figure out why. */ device_printf(sc->dev, "largest_rx_cluster=%u leaves no" " refill source for fl %p (dma %u). Ignored.\n", largest_rx_cluster, fl, maxp); } for (idx = swz->head_hwidx; idx != -1; idx = hwb->next) { hwb = &hwb_list[idx]; spare = swz->size - hwb->size; if (spare < spare_needed) continue; hwidx = idx; /* best option so far */ if (hwb->size >= maxp) { if ((fl->flags & FL_BUF_PACKING) == 0) goto done; /* stop looking (not packing) */ if (swz->size >= safest_rx_cluster) goto done; /* stop looking (packing) */ } break; /* keep looking, next zone */ } } done: /* A usable hwidx has been located. */ MPASS(hwidx != -1); hwb = &hwb_list[hwidx]; zidx = hwb->zidx; swz = &sc->sge.sw_zone_info[zidx]; region1 = 0; region3 = swz->size - hwb->size; /* * Stay within this zone and see if there is a better match when mbuf * inlining is allowed. Remember that the hwidx's are sorted in * decreasing order of size (so in increasing order of spare area). */ for (idx = hwidx; idx != -1; idx = hwb->next) { hwb = &hwb_list[idx]; spare = swz->size - hwb->size; if (allow_mbufs_in_cluster == 0 || hwb->size < maxp) break; /* * Do not inline mbufs if doing so would violate the pad/pack * boundary alignment requirement. */ if (fl_pad && (MSIZE % sc->params.sge.pad_boundary) != 0) continue; if (fl->flags & FL_BUF_PACKING && (MSIZE % sc->params.sge.pack_boundary) != 0) continue; if (spare < CL_METADATA_SIZE + MSIZE) continue; n = (spare - CL_METADATA_SIZE) / MSIZE; if (n > howmany(hwb->size, maxp)) break; hwidx = idx; if (fl->flags & FL_BUF_PACKING) { region1 = n * MSIZE; region3 = spare - region1; } else { region1 = MSIZE; region3 = spare - region1; break; } } KASSERT(zidx >= 0 && zidx < SW_ZONE_SIZES, ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp)); KASSERT(hwidx >= 0 && hwidx <= SGE_FLBUF_SIZES, ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp)); KASSERT(region1 + sc->sge.hw_buf_info[hwidx].size + region3 == sc->sge.sw_zone_info[zidx].size, ("%s: bad buffer layout for fl %p, maxp %d. " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); if (fl->flags & FL_BUF_PACKING || region1 > 0) { KASSERT(region3 >= CL_METADATA_SIZE, ("%s: no room for metadata. fl %p, maxp %d; " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); KASSERT(region1 % MSIZE == 0, ("%s: bad mbuf region for fl %p, maxp %d. " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); } fl->cll_def.zidx = zidx; fl->cll_def.hwidx = hwidx; fl->cll_def.region1 = region1; fl->cll_def.region3 = region3; } static void find_safe_refill_source(struct adapter *sc, struct sge_fl *fl) { struct sge *s = &sc->sge; struct hw_buf_info *hwb; struct sw_zone_info *swz; int spare; int8_t hwidx; if (fl->flags & FL_BUF_PACKING) hwidx = s->safe_hwidx2; /* with room for metadata */ else if (allow_mbufs_in_cluster && s->safe_hwidx2 != -1) { hwidx = s->safe_hwidx2; hwb = &s->hw_buf_info[hwidx]; swz = &s->sw_zone_info[hwb->zidx]; spare = swz->size - hwb->size; /* no good if there isn't room for an mbuf as well */ if (spare < CL_METADATA_SIZE + MSIZE) hwidx = s->safe_hwidx1; } else hwidx = s->safe_hwidx1; if (hwidx == -1) { /* No fallback source */ fl->cll_alt.hwidx = -1; fl->cll_alt.zidx = -1; return; } hwb = &s->hw_buf_info[hwidx]; swz = &s->sw_zone_info[hwb->zidx]; spare = swz->size - hwb->size; fl->cll_alt.hwidx = hwidx; fl->cll_alt.zidx = hwb->zidx; if (allow_mbufs_in_cluster && (fl_pad == 0 || (MSIZE % sc->params.sge.pad_boundary) == 0)) fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE; else fl->cll_alt.region1 = 0; fl->cll_alt.region3 = spare - fl->cll_alt.region1; } static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) { mtx_lock(&sc->sfl_lock); FL_LOCK(fl); if ((fl->flags & FL_DOOMED) == 0) { fl->flags |= FL_STARVING; TAILQ_INSERT_TAIL(&sc->sfl, fl, link); callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); } FL_UNLOCK(fl); mtx_unlock(&sc->sfl_lock); } static void handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) { struct sge_wrq *wrq = (void *)eq; atomic_readandclear_int(&eq->equiq); taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); } static void handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) { struct sge_txq *txq = (void *)eq; MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); atomic_readandclear_int(&eq->equiq); mp_ring_check_drainage(txq->r, 0); taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); } static int handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); struct adapter *sc = iq->adapter; struct sge *s = &sc->sge; struct sge_eq *eq; static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, &handle_wrq_egr_update, &handle_eth_egr_update, &handle_wrq_egr_update}; KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); eq = s->eqmap[qid - s->eq_start - s->eq_base]; (*h[eq->flags & EQ_TYPEMASK])(sc, eq); return (0); } /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ offsetof(struct cpl_fw6_msg, data)); static int handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { const struct rss_header *rss2; rss2 = (const struct rss_header *)&cpl->data[0]; return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); } return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); } /** * t4_handle_wrerr_rpl - process a FW work request error message * @adap: the adapter * @rpl: start of the FW message */ static int t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) { u8 opcode = *(const u8 *)rpl; const struct fw_error_cmd *e = (const void *)rpl; unsigned int i; if (opcode != FW_ERROR_CMD) { log(LOG_ERR, "%s: Received WRERR_RPL message with opcode %#x\n", device_get_nameunit(adap->dev), opcode); return (EINVAL); } log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : "non-fatal"); switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { case FW_ERROR_TYPE_EXCEPTION: log(LOG_ERR, "exception info:\n"); for (i = 0; i < nitems(e->u.exception.info); i++) log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", be32toh(e->u.exception.info[i])); log(LOG_ERR, "\n"); break; case FW_ERROR_TYPE_HWMODULE: log(LOG_ERR, "HW module regaddr %08x regval %08x\n", be32toh(e->u.hwmodule.regaddr), be32toh(e->u.hwmodule.regval)); break; case FW_ERROR_TYPE_WR: log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", be16toh(e->u.wr.cidx), G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), be32toh(e->u.wr.eqid)); for (i = 0; i < nitems(e->u.wr.wrhdr); i++) log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", e->u.wr.wrhdr[i]); log(LOG_ERR, "\n"); break; case FW_ERROR_TYPE_ACL: log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", be16toh(e->u.acl.cidx), G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), be32toh(e->u.acl.eqid), G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : "MAC"); for (i = 0; i < nitems(e->u.acl.val); i++) log(LOG_ERR, " %02x", e->u.acl.val[i]); log(LOG_ERR, "\n"); break; default: log(LOG_ERR, "type %#x\n", G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); return (EINVAL); } return (0); } static int sysctl_uint16(SYSCTL_HANDLER_ARGS) { uint16_t *id = arg1; int i = *id; return sysctl_handle_int(oidp, &i, 0, req); } static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS) { struct sge *s = arg1; struct hw_buf_info *hwb = &s->hw_buf_info[0]; struct sw_zone_info *swz = &s->sw_zone_info[0]; int i, rc; struct sbuf sb; char c; sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) { if (hwb->zidx >= 0 && swz[hwb->zidx].size <= largest_rx_cluster) c = '*'; else c = '\0'; sbuf_printf(&sb, "%u%c ", hwb->size, c); } sbuf_trim(&sb); sbuf_finish(&sb); rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); sbuf_delete(&sb); return (rc); } static int sysctl_tc(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct port_info *pi; struct adapter *sc; struct sge_txq *txq; struct tx_cl_rl_params *tc; int qidx = arg2, rc, tc_idx; uint32_t fw_queue, fw_class; MPASS(qidx >= 0 && qidx < vi->ntxq); pi = vi->pi; sc = pi->adapter; txq = &sc->sge.txq[vi->first_txq + qidx]; tc_idx = txq->tc_idx; rc = sysctl_handle_int(oidp, &tc_idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (sc->flags & IS_VF) return (EPERM); /* Note that -1 is legitimate input (it means unbind). */ if (tc_idx < -1 || tc_idx >= sc->chip_params->nsched_cls) return (EINVAL); mtx_lock(&sc->tc_lock); if (tc_idx == txq->tc_idx) { rc = 0; /* No change, nothing to do. */ goto done; } fw_queue = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id); if (tc_idx == -1) fw_class = 0xffffffff; /* Unbind. */ else { /* * Bind to a different class. */ tc = &pi->sched_params->cl_rl[tc_idx]; if (tc->flags & TX_CLRL_ERROR) { /* Previous attempt to set the cl-rl params failed. */ rc = EIO; goto done; } else { /* * Ok to proceed. Place a reference on the new class * while still holding on to the reference on the * previous class, if any. */ fw_class = tc_idx; tc->refcount++; } } mtx_unlock(&sc->tc_lock); rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4stc"); if (rc) return (rc); rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, &fw_class); end_synchronized_op(sc, 0); mtx_lock(&sc->tc_lock); if (rc == 0) { if (txq->tc_idx != -1) { tc = &pi->sched_params->cl_rl[txq->tc_idx]; MPASS(tc->refcount > 0); tc->refcount--; } txq->tc_idx = tc_idx; } else if (tc_idx != -1) { tc = &pi->sched_params->cl_rl[tc_idx]; MPASS(tc->refcount > 0); tc->refcount--; } done: mtx_unlock(&sc->tc_lock); return (rc); } Index: projects/runtime-coverage/sys/dev/iicbus/iicbus.h =================================================================== --- projects/runtime-coverage/sys/dev/iicbus/iicbus.h (revision 321584) +++ projects/runtime-coverage/sys/dev/iicbus/iicbus.h (revision 321585) @@ -1,78 +1,79 @@ /*- * Copyright (c) 1998 Nicolas Souchu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __IICBUS_H #define __IICBUS_H #include #include #define IICBUS_IVAR(d) (struct iicbus_ivar *) device_get_ivars(d) #define IICBUS_SOFTC(d) (struct iicbus_softc *) device_get_softc(d) struct iicbus_softc { device_t dev; /* Myself */ device_t owner; /* iicbus owner device structure */ + u_int owncount; /* iicbus ownership nesting count */ u_char started; /* address of the 'started' slave * 0 if no start condition succeeded */ u_char strict; /* deny operations that violate the * I2C protocol */ struct mtx lock; u_int bus_freq; /* Configured bus Hz. */ }; struct iicbus_ivar { uint32_t addr; struct resource_list rl; bool nostop; }; enum { IICBUS_IVAR_ADDR, /* Address or base address */ IICBUS_IVAR_NOSTOP, /* nostop defaults */ }; #define IICBUS_ACCESSOR(A, B, T) \ __BUS_ACCESSOR(iicbus, A, IICBUS, B, T) IICBUS_ACCESSOR(addr, ADDR, uint32_t) IICBUS_ACCESSOR(nostop, NOSTOP, bool) #define IICBUS_LOCK(sc) mtx_lock(&(sc)->lock) #define IICBUS_UNLOCK(sc) mtx_unlock(&(sc)->lock) #define IICBUS_ASSERT_LOCKED(sc) mtx_assert(&(sc)->lock, MA_OWNED) int iicbus_generic_intr(device_t dev, int event, char *buf); void iicbus_init_frequency(device_t dev, u_int bus_freq); extern driver_t iicbus_driver; extern devclass_t iicbus_devclass; #endif Index: projects/runtime-coverage/sys/dev/iicbus/iiconf.c =================================================================== --- projects/runtime-coverage/sys/dev/iicbus/iiconf.c (revision 321584) +++ projects/runtime-coverage/sys/dev/iicbus/iiconf.c (revision 321585) @@ -1,472 +1,517 @@ /*- * Copyright (c) 1998 Nicolas Souchu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" /* * Translate IIC_Exxxxx status values to vaguely-equivelent errno values. */ int iic2errno(int iic_status) { switch (iic_status) { case IIC_NOERR: return (0); case IIC_EBUSERR: return (EALREADY); case IIC_ENOACK: return (EIO); case IIC_ETIMEOUT: return (ETIMEDOUT); case IIC_EBUSBSY: return (EWOULDBLOCK); case IIC_ESTATUS: return (EPROTO); case IIC_EUNDERFLOW: return (EIO); case IIC_EOVERFLOW: return (EOVERFLOW); case IIC_ENOTSUPP: return (EOPNOTSUPP); case IIC_ENOADDR: return (EADDRNOTAVAIL); case IIC_ERESOURCE: return (ENOMEM); default: return (EIO); } } /* * iicbus_intr() */ void iicbus_intr(device_t bus, int event, char *buf) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); /* call owner's intr routine */ if (sc->owner) IICBUS_INTR(sc->owner, event, buf); return; } static int iicbus_poll(struct iicbus_softc *sc, int how) { int error; IICBUS_ASSERT_LOCKED(sc); switch (how) { case IIC_WAIT | IIC_INTR: error = mtx_sleep(sc, &sc->lock, IICPRI|PCATCH, "iicreq", 0); break; case IIC_WAIT | IIC_NOINTR: error = mtx_sleep(sc, &sc->lock, IICPRI, "iicreq", 0); break; default: return (IIC_EBUSBSY); } return (error); } /* * iicbus_request_bus() * * Allocate the device to perform transfers. * * how : IIC_WAIT or IIC_DONTWAIT */ int iicbus_request_bus(device_t bus, device_t dev, int how) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); int error = 0; IICBUS_LOCK(sc); - while ((error == 0) && (sc->owner != NULL)) + while (error == 0 && sc->owner != NULL && sc->owner != dev) error = iicbus_poll(sc, how); if (error == 0) { - sc->owner = dev; - /* - * Drop the lock around the call to the bus driver. - * This call should be allowed to sleep in the IIC_WAIT case. - * Drivers might also need to grab locks that would cause LOR - * if our lock is held. - */ - IICBUS_UNLOCK(sc); - /* Ask the underlying layers if the request is ok */ - error = IICBUS_CALLBACK(device_get_parent(bus), - IIC_REQUEST_BUS, (caddr_t)&how); - IICBUS_LOCK(sc); - - if (error != 0) { - sc->owner = NULL; - wakeup_one(sc); + ++sc->owncount; + if (sc->owner == NULL) { + sc->owner = dev; + /* + * Drop the lock around the call to the bus driver, it + * should be allowed to sleep in the IIC_WAIT case. + * Drivers might also need to grab locks that would + * cause a LOR if our lock is held. + */ + IICBUS_UNLOCK(sc); + /* Ask the underlying layers if the request is ok */ + error = IICBUS_CALLBACK(device_get_parent(bus), + IIC_REQUEST_BUS, (caddr_t)&how); + IICBUS_LOCK(sc); + + if (error != 0) { + sc->owner = NULL; + sc->owncount = 0; + wakeup_one(sc); + } } } IICBUS_UNLOCK(sc); return (error); } /* * iicbus_release_bus() * * Release the device allocated with iicbus_request_dev() */ int iicbus_release_bus(device_t bus, device_t dev) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); - int error; IICBUS_LOCK(sc); if (sc->owner != dev) { IICBUS_UNLOCK(sc); return (IIC_EBUSBSY); } - /* - * Drop the lock around the call to the bus driver. - * This call should be allowed to sleep in the IIC_WAIT case. - * Drivers might also need to grab locks that would cause LOR - * if our lock is held. - */ - IICBUS_UNLOCK(sc); - /* Ask the underlying layers if the release is ok */ - error = IICBUS_CALLBACK(device_get_parent(bus), IIC_RELEASE_BUS, NULL); - - if (error == 0) { + if (--sc->owncount == 0) { + /* Drop the lock while informing the low-level driver. */ + IICBUS_UNLOCK(sc); + IICBUS_CALLBACK(device_get_parent(bus), IIC_RELEASE_BUS, NULL); IICBUS_LOCK(sc); sc->owner = NULL; - - /* wakeup a waiting thread */ wakeup_one(sc); - IICBUS_UNLOCK(sc); } - - return (error); + IICBUS_UNLOCK(sc); + return (0); } /* * iicbus_started() * * Test if the iicbus is started by the controller */ int iicbus_started(device_t bus) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); return (sc->started); } /* * iicbus_start() * * Send start condition to the slave addressed by 'slave' */ int iicbus_start(device_t bus, u_char slave, int timeout) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); int error = 0; if (sc->started) return (IIC_ESTATUS); /* protocol error, bus already started */ if (!(error = IICBUS_START(device_get_parent(bus), slave, timeout))) sc->started = slave; else sc->started = 0; return (error); } /* * iicbus_repeated_start() * * Send start condition to the slave addressed by 'slave' */ int iicbus_repeated_start(device_t bus, u_char slave, int timeout) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); int error = 0; if (!sc->started) return (IIC_ESTATUS); /* protocol error, bus not started */ if (!(error = IICBUS_REPEATED_START(device_get_parent(bus), slave, timeout))) sc->started = slave; else sc->started = 0; return (error); } /* * iicbus_stop() * * Send stop condition to the bus */ int iicbus_stop(device_t bus) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); int error = 0; if (!sc->started) return (IIC_ESTATUS); /* protocol error, bus not started */ error = IICBUS_STOP(device_get_parent(bus)); /* refuse any further access */ sc->started = 0; return (error); } /* * iicbus_write() * * Write a block of data to the slave previously started by * iicbus_start() call */ int iicbus_write(device_t bus, const char *buf, int len, int *sent, int timeout) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); /* a slave must have been started for writing */ if (sc->started == 0 || (sc->strict != 0 && (sc->started & LSB) != 0)) return (IIC_ESTATUS); return (IICBUS_WRITE(device_get_parent(bus), buf, len, sent, timeout)); } /* * iicbus_read() * * Read a block of data from the slave previously started by * iicbus_read() call */ int iicbus_read(device_t bus, char *buf, int len, int *read, int last, int delay) { struct iicbus_softc *sc = (struct iicbus_softc *)device_get_softc(bus); /* a slave must have been started for reading */ if (sc->started == 0 || (sc->strict != 0 && (sc->started & LSB) == 0)) return (IIC_ESTATUS); return (IICBUS_READ(device_get_parent(bus), buf, len, read, last, delay)); } /* * iicbus_write_byte() * * Write a byte to the slave previously started by iicbus_start() call */ int iicbus_write_byte(device_t bus, char byte, int timeout) { struct iicbus_softc *sc = device_get_softc(bus); char data = byte; int sent; /* a slave must have been started for writing */ if (sc->started == 0 || (sc->strict != 0 && (sc->started & LSB) != 0)) return (IIC_ESTATUS); return (iicbus_write(bus, &data, 1, &sent, timeout)); } /* * iicbus_read_byte() * * Read a byte from the slave previously started by iicbus_start() call */ int iicbus_read_byte(device_t bus, char *byte, int timeout) { struct iicbus_softc *sc = device_get_softc(bus); int read; /* a slave must have been started for reading */ if (sc->started == 0 || (sc->strict != 0 && (sc->started & LSB) == 0)) return (IIC_ESTATUS); return (iicbus_read(bus, byte, 1, &read, IIC_LAST_READ, timeout)); } /* * iicbus_block_write() * * Write a block of data to slave ; start/stop protocol managed */ int iicbus_block_write(device_t bus, u_char slave, char *buf, int len, int *sent) { u_char addr = slave & ~LSB; int error; if ((error = iicbus_start(bus, addr, 0))) return (error); error = iicbus_write(bus, buf, len, sent, 0); iicbus_stop(bus); return (error); } /* * iicbus_block_read() * * Read a block of data from slave ; start/stop protocol managed */ int iicbus_block_read(device_t bus, u_char slave, char *buf, int len, int *read) { u_char addr = slave | LSB; int error; if ((error = iicbus_start(bus, addr, 0))) return (error); error = iicbus_read(bus, buf, len, read, IIC_LAST_READ, 0); iicbus_stop(bus); return (error); } /* * iicbus_transfer() * * Do an aribtrary number of transfers on the iicbus. We pass these * raw requests to the bridge driver. If the bridge driver supports * them directly, then it manages all the details. If not, it can use * the helper function iicbus_transfer_gen() which will do the * transfers at a low level. * * Pointers passed in as part of iic_msg must be kernel pointers. * Callers that have user addresses to manage must do so on their own. */ int iicbus_transfer(device_t bus, struct iic_msg *msgs, uint32_t nmsgs) { return (IICBUS_TRANSFER(device_get_parent(bus), msgs, nmsgs)); } int iicbus_transfer_excl(device_t dev, struct iic_msg *msgs, uint32_t nmsgs, int how) { device_t bus; int error; bus = device_get_parent(dev); error = iicbus_request_bus(bus, dev, how); if (error == 0) error = IICBUS_TRANSFER(bus, msgs, nmsgs); iicbus_release_bus(bus, dev); return (error); } /* * Generic version of iicbus_transfer that calls the appropriate * routines to accomplish this. See note above about acceptable * buffer addresses. */ int iicbus_transfer_gen(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) { int i, error, lenread, lenwrote, nkid, rpstart, addr; device_t *children, bus; bool nostop, started; if ((error = device_get_children(dev, &children, &nkid)) != 0) return (IIC_ERESOURCE); if (nkid != 1) { free(children, M_TEMP); return (IIC_ENOTSUPP); } bus = children[0]; rpstart = 0; free(children, M_TEMP); nostop = iicbus_get_nostop(dev); started = false; for (i = 0, error = 0; i < nmsgs && error == 0; i++) { addr = msgs[i].slave; if (msgs[i].flags & IIC_M_RD) addr |= LSB; else addr &= ~LSB; if (!(msgs[i].flags & IIC_M_NOSTART)) { if (rpstart) error = iicbus_repeated_start(bus, addr, 0); else error = iicbus_start(bus, addr, 0); if (error != 0) break; started = true; } if (msgs[i].flags & IIC_M_RD) error = iicbus_read(bus, msgs[i].buf, msgs[i].len, &lenread, IIC_LAST_READ, 0); else error = iicbus_write(bus, msgs[i].buf, msgs[i].len, &lenwrote, 0); if (error != 0) break; if ((msgs[i].flags & IIC_M_NOSTOP) != 0 || (nostop && i + 1 < nmsgs)) { rpstart = 1; /* Next message gets repeated start */ } else { rpstart = 0; iicbus_stop(bus); } } if (error != 0 && started) iicbus_stop(bus); return (error); +} + +int +iicdev_readfrom(device_t slavedev, uint8_t regaddr, void *buffer, + uint16_t buflen, int waithow) +{ + struct iic_msg msgs[2]; + uint8_t slaveaddr; + + /* + * Two transfers back to back with a repeat-start between them; first we + * write the address-within-device, then we read from the device. + */ + slaveaddr = iicbus_get_addr(slavedev); + + msgs[0].slave = slaveaddr; + msgs[0].flags = IIC_M_WR | IIC_M_NOSTOP; + msgs[0].len = 1; + msgs[0].buf = ®addr; + + msgs[1].slave = slaveaddr; + msgs[1].flags = IIC_M_RD; + msgs[1].len = buflen; + msgs[1].buf = buffer; + + return (iicbus_transfer_excl(slavedev, msgs, nitems(msgs), waithow)); +} + +int iicdev_writeto(device_t slavedev, uint8_t regaddr, void *buffer, + uint16_t buflen, int waithow) +{ + struct iic_msg msgs[2]; + uint8_t slaveaddr; + + /* + * Two transfers back to back with no stop or start between them; first + * we write the address then we write the data to that address, all in a + * single transfer from two scattered buffers. + */ + slaveaddr = iicbus_get_addr(slavedev); + + msgs[0].slave = slaveaddr; + msgs[0].flags = IIC_M_WR | IIC_M_NOSTOP; + msgs[0].len = 1; + msgs[0].buf = ®addr; + + msgs[1].slave = slaveaddr; + msgs[1].flags = IIC_M_WR | IIC_M_NOSTART; + msgs[1].len = buflen; + msgs[1].buf = buffer; + + return (iicbus_transfer_excl(slavedev, msgs, nitems(msgs), waithow)); } Index: projects/runtime-coverage/sys/dev/iicbus/iiconf.h =================================================================== --- projects/runtime-coverage/sys/dev/iicbus/iiconf.h (revision 321584) +++ projects/runtime-coverage/sys/dev/iicbus/iiconf.h (revision 321585) @@ -1,149 +1,159 @@ /*- * Copyright (c) 1998, 2001 Nicolas Souchu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __IICONF_H #define __IICONF_H #include #include #define IICPRI (PZERO+8) /* XXX sleep/wakeup queue priority */ #define LSB 0x1 /* * How tsleep() is called in iic_request_bus(). */ #define IIC_DONTWAIT 0 #define IIC_NOINTR 0 #define IIC_WAIT 0x1 #define IIC_INTR 0x2 #define IIC_INTRWAIT (IIC_INTR | IIC_WAIT) /* * i2c modes */ #define IIC_MASTER 0x1 #define IIC_SLAVE 0x2 #define IIC_POLLED 0x4 /* * i2c speed */ #define IIC_UNKNOWN 0x0 #define IIC_SLOW 0x1 #define IIC_FAST 0x2 #define IIC_FASTEST 0x3 #define IIC_LAST_READ 0x1 /* * callback index */ #define IIC_REQUEST_BUS 0x1 #define IIC_RELEASE_BUS 0x2 /* * interrupt events */ #define INTR_GENERAL 0x1 /* general call received */ #define INTR_START 0x2 /* the I2C interface is addressed */ #define INTR_STOP 0x3 /* stop condition received */ #define INTR_RECEIVE 0x4 /* character received */ #define INTR_TRANSMIT 0x5 /* character to transmit */ #define INTR_ERROR 0x6 /* error */ #define INTR_NOACK 0x7 /* no ack from master receiver */ /* * adapter layer errors */ #define IIC_NOERR 0x0 /* no error occurred */ #define IIC_EBUSERR 0x1 /* bus error (hardware not in expected state) */ #define IIC_ENOACK 0x2 /* ack not received until timeout */ #define IIC_ETIMEOUT 0x3 /* timeout */ #define IIC_EBUSBSY 0x4 /* bus busy (reserved by another client) */ #define IIC_ESTATUS 0x5 /* status error */ #define IIC_EUNDERFLOW 0x6 /* slave ready for more data */ #define IIC_EOVERFLOW 0x7 /* too much data */ #define IIC_ENOTSUPP 0x8 /* request not supported */ #define IIC_ENOADDR 0x9 /* no address assigned to the interface */ #define IIC_ERESOURCE 0xa /* resources (memory, whatever) unavailable */ /* * Note that all iicbus functions return IIC_Exxxxx status values, * except iic2errno() (obviously) and iicbus_started() (returns bool). */ extern int iic2errno(int); extern int iicbus_request_bus(device_t, device_t, int); extern int iicbus_release_bus(device_t, device_t); extern device_t iicbus_alloc_bus(device_t); extern void iicbus_intr(device_t, int, char *); extern int iicbus_null_repeated_start(device_t, u_char); extern int iicbus_null_callback(device_t, int, caddr_t); #define iicbus_reset(bus,speed,addr,oldaddr) \ (IICBUS_RESET(device_get_parent(bus), speed, addr, oldaddr)) /* basic I2C operations */ extern int iicbus_started(device_t); extern int iicbus_start(device_t, u_char, int); extern int iicbus_stop(device_t); extern int iicbus_repeated_start(device_t, u_char, int); extern int iicbus_write(device_t, const char *, int, int *, int); extern int iicbus_read(device_t, char *, int, int *, int, int); /* single byte read/write functions, start/stop not managed */ extern int iicbus_write_byte(device_t, char, int); extern int iicbus_read_byte(device_t, char *, int); /* Read/write operations with start/stop conditions managed */ extern int iicbus_block_write(device_t, u_char, char *, int, int *); extern int iicbus_block_read(device_t, u_char, char *, int, int *); /* vectors of iic operations to pass to bridge */ int iicbus_transfer(device_t bus, struct iic_msg *msgs, uint32_t nmsgs); int iicbus_transfer_excl(device_t bus, struct iic_msg *msgs, uint32_t nmsgs, int how); int iicbus_transfer_gen(device_t bus, struct iic_msg *msgs, uint32_t nmsgs); +/* + * Simple register read/write routines, but the "register" can be any size. + * The transfers are done with iicbus_transfer_excl(). Reads use a repeat-start + * between sending the address and reading; writes use a single start/stop. + */ +int iicdev_readfrom(device_t _slavedev, uint8_t _regaddr, void *_buffer, + uint16_t _buflen, int _waithow); +int iicdev_writeto(device_t _slavedev, uint8_t _regaddr, void *_buffer, + uint16_t _buflen, int _waithow); + #define IICBUS_MODVER 1 #define IICBUS_MINVER 1 #define IICBUS_MAXVER 1 #define IICBUS_PREFVER IICBUS_MODVER extern driver_t iicbb_driver; extern devclass_t iicbb_devclass; #define IICBB_MODVER 1 #define IICBB_MINVER 1 #define IICBB_MAXVER 1 #define IICBB_PREFVER IICBB_MODVER #endif Index: projects/runtime-coverage/sys/dev/iwm/if_iwm.c =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwm.c (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwm.c (revision 321585) @@ -1,6444 +1,6458 @@ /* $OpenBSD: if_iwm.c,v 1.42 2015/05/30 02:49:23 deraadt Exp $ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* From DragonflyBSD */ #define mtodoff(m, t, off) ((t)((m)->m_data + (off))) const uint8_t iwm_nvm_channels[] = { /* 2.4 GHz */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 5 GHz */ 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165 }; _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS, "IWM_NUM_CHANNELS is too small"); const uint8_t iwm_nvm_channels_8000[] = { /* 2.4 GHz */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 5 GHz */ 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, 181 }; _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000, "IWM_NUM_CHANNELS_8000 is too small"); #define IWM_NUM_2GHZ_CHANNELS 14 #define IWM_N_HW_ADDR_MASK 0xF /* * XXX For now, there's simply a fixed set of rate table entries * that are populated. */ const struct iwm_rate { uint8_t rate; uint8_t plcp; } iwm_rates[] = { { 2, IWM_RATE_1M_PLCP }, { 4, IWM_RATE_2M_PLCP }, { 11, IWM_RATE_5M_PLCP }, { 22, IWM_RATE_11M_PLCP }, { 12, IWM_RATE_6M_PLCP }, { 18, IWM_RATE_9M_PLCP }, { 24, IWM_RATE_12M_PLCP }, { 36, IWM_RATE_18M_PLCP }, { 48, IWM_RATE_24M_PLCP }, { 72, IWM_RATE_36M_PLCP }, { 96, IWM_RATE_48M_PLCP }, { 108, IWM_RATE_54M_PLCP }, }; #define IWM_RIDX_CCK 0 #define IWM_RIDX_OFDM 4 #define IWM_RIDX_MAX (nitems(iwm_rates)-1) #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM) #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM) struct iwm_nvm_section { uint16_t length; uint8_t *data; }; #define IWM_MVM_UCODE_ALIVE_TIMEOUT hz #define IWM_MVM_UCODE_CALIB_TIMEOUT (2*hz) struct iwm_mvm_alive_data { int valid; uint32_t scd_base_addr; }; static int iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t); static int iwm_firmware_store_section(struct iwm_softc *, enum iwm_ucode_type, const uint8_t *, size_t); static int iwm_set_default_calib(struct iwm_softc *, const void *); static void iwm_fw_info_free(struct iwm_fw_info *); static int iwm_read_firmware(struct iwm_softc *, enum iwm_ucode_type); static int iwm_alloc_fwmem(struct iwm_softc *); static int iwm_alloc_sched(struct iwm_softc *); static int iwm_alloc_kw(struct iwm_softc *); static int iwm_alloc_ict(struct iwm_softc *); static int iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static void iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static void iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static int iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *, int); static void iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *); static void iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *); static void iwm_enable_interrupts(struct iwm_softc *); static void iwm_restore_interrupts(struct iwm_softc *); static void iwm_disable_interrupts(struct iwm_softc *); static void iwm_ict_reset(struct iwm_softc *); static int iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *); static void iwm_stop_device(struct iwm_softc *); static void iwm_mvm_nic_config(struct iwm_softc *); static int iwm_nic_rx_init(struct iwm_softc *); static int iwm_nic_tx_init(struct iwm_softc *); static int iwm_nic_init(struct iwm_softc *); static int iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t); static int iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t, uint16_t, uint8_t *, uint16_t *); static int iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *, uint16_t *, uint32_t); static uint32_t iwm_eeprom_channel_flags(uint16_t); static void iwm_add_channel_band(struct iwm_softc *, struct ieee80211_channel[], int, int *, int, size_t, const uint8_t[]); static void iwm_init_channel_map(struct ieee80211com *, int, int *, struct ieee80211_channel[]); static struct iwm_nvm_data * iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *); static void iwm_free_nvm_data(struct iwm_nvm_data *); static void iwm_set_hw_address_family_8000(struct iwm_softc *, struct iwm_nvm_data *, const uint16_t *, const uint16_t *); static int iwm_get_sku(const struct iwm_softc *, const uint16_t *, const uint16_t *); static int iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *); static int iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *, const uint16_t *); static int iwm_get_n_hw_addrs(const struct iwm_softc *, const uint16_t *); static void iwm_set_radio_cfg(const struct iwm_softc *, struct iwm_nvm_data *, uint32_t); static struct iwm_nvm_data * iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *); static int iwm_nvm_init(struct iwm_softc *); static int iwm_pcie_load_section(struct iwm_softc *, uint8_t, const struct iwm_fw_desc *); static int iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t, bus_addr_t, uint32_t); static int iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc, const struct iwm_fw_sects *, int, int *); static int iwm_pcie_load_cpu_sections(struct iwm_softc *, const struct iwm_fw_sects *, int, int *); static int iwm_pcie_load_given_ucode_8000(struct iwm_softc *, const struct iwm_fw_sects *); static int iwm_pcie_load_given_ucode(struct iwm_softc *, const struct iwm_fw_sects *); static int iwm_start_fw(struct iwm_softc *, const struct iwm_fw_sects *); static int iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t); static int iwm_send_phy_cfg_cmd(struct iwm_softc *); static int iwm_mvm_load_ucode_wait_alive(struct iwm_softc *, enum iwm_ucode_type); static int iwm_run_init_mvm_ucode(struct iwm_softc *, int); static int iwm_rx_addbuf(struct iwm_softc *, int, int); static int iwm_mvm_get_signal_strength(struct iwm_softc *, struct iwm_rx_phy_info *); static void iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *, struct iwm_rx_packet *); -static int iwm_get_noise(struct iwm_softc *sc, +static int iwm_get_noise(struct iwm_softc *, const struct iwm_mvm_statistics_rx_non_phy *); +static void iwm_mvm_handle_rx_statistics(struct iwm_softc *, + struct iwm_rx_packet *); static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *, uint32_t, boolean_t); static int iwm_mvm_rx_tx_cmd_single(struct iwm_softc *, struct iwm_rx_packet *, struct iwm_node *); static void iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *); static void iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *); #if 0 static void iwm_update_sched(struct iwm_softc *, int, int, uint8_t, uint16_t); #endif static const struct iwm_rate * iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *, struct mbuf *, struct iwm_tx_cmd *); static int iwm_tx(struct iwm_softc *, struct mbuf *, struct ieee80211_node *, int); static int iwm_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static int iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_vap *); static int iwm_auth(struct ieee80211vap *, struct iwm_softc *); static int iwm_release(struct iwm_softc *, struct iwm_node *); static struct ieee80211_node * iwm_node_alloc(struct ieee80211vap *, const uint8_t[IEEE80211_ADDR_LEN]); static void iwm_setrates(struct iwm_softc *, struct iwm_node *); static int iwm_media_change(struct ifnet *); static int iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int); static void iwm_endscan_cb(void *, int); static int iwm_send_bt_init_conf(struct iwm_softc *); static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *); static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *); static int iwm_send_update_mcc_cmd(struct iwm_softc *, const char *); static void iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t); static int iwm_init_hw(struct iwm_softc *); static void iwm_init(struct iwm_softc *); static void iwm_start(struct iwm_softc *); static void iwm_stop(struct iwm_softc *); static void iwm_watchdog(void *); static void iwm_parent(struct ieee80211com *); #ifdef IWM_DEBUG static const char * iwm_desc_lookup(uint32_t); static void iwm_nic_error(struct iwm_softc *); static void iwm_nic_umac_error(struct iwm_softc *); #endif static void iwm_handle_rxb(struct iwm_softc *, struct mbuf *); static void iwm_notif_intr(struct iwm_softc *); static void iwm_intr(void *); static int iwm_attach(device_t); static int iwm_is_valid_ether_addr(uint8_t *); static void iwm_preinit(void *); static int iwm_detach_local(struct iwm_softc *sc, int); static void iwm_init_task(void *); static void iwm_radiotap_attach(struct iwm_softc *); static struct ieee80211vap * iwm_vap_create(struct ieee80211com *, const char [IFNAMSIZ], int, enum ieee80211_opmode, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); static void iwm_vap_delete(struct ieee80211vap *); +static void iwm_xmit_queue_drain(struct iwm_softc *); static void iwm_scan_start(struct ieee80211com *); static void iwm_scan_end(struct ieee80211com *); static void iwm_update_mcast(struct ieee80211com *); static void iwm_set_channel(struct ieee80211com *); static void iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long); static void iwm_scan_mindwell(struct ieee80211_scan_state *); static int iwm_detach(device_t); static int iwm_lar_disable = 0; TUNABLE_INT("hw.iwm.lar.disable", &iwm_lar_disable); /* * Firmware parser. */ static int iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen) { const struct iwm_fw_cscheme_list *l = (const void *)data; if (dlen < sizeof(*l) || dlen < sizeof(l->size) + l->size * sizeof(*l->cs)) return EINVAL; /* we don't actually store anything for now, always use s/w crypto */ return 0; } static int iwm_firmware_store_section(struct iwm_softc *sc, enum iwm_ucode_type type, const uint8_t *data, size_t dlen) { struct iwm_fw_sects *fws; struct iwm_fw_desc *fwone; if (type >= IWM_UCODE_TYPE_MAX) return EINVAL; if (dlen < sizeof(uint32_t)) return EINVAL; fws = &sc->sc_fw.fw_sects[type]; if (fws->fw_count >= IWM_UCODE_SECTION_MAX) return EINVAL; fwone = &fws->fw_sect[fws->fw_count]; /* first 32bit are device load offset */ memcpy(&fwone->offset, data, sizeof(uint32_t)); /* rest is data */ fwone->data = data + sizeof(uint32_t); fwone->len = dlen - sizeof(uint32_t); fws->fw_count++; return 0; } #define IWM_DEFAULT_SCAN_CHANNELS 40 /* iwlwifi: iwl-drv.c */ struct iwm_tlv_calib_data { uint32_t ucode_type; struct iwm_tlv_calib_ctrl calib; } __packed; static int iwm_set_default_calib(struct iwm_softc *sc, const void *data) { const struct iwm_tlv_calib_data *def_calib = data; uint32_t ucode_type = le32toh(def_calib->ucode_type); if (ucode_type >= IWM_UCODE_TYPE_MAX) { device_printf(sc->sc_dev, "Wrong ucode_type %u for default " "calibration.\n", ucode_type); return EINVAL; } sc->sc_default_calib[ucode_type].flow_trigger = def_calib->calib.flow_trigger; sc->sc_default_calib[ucode_type].event_trigger = def_calib->calib.event_trigger; return 0; } static int iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data, struct iwm_ucode_capabilities *capa) { const struct iwm_ucode_api *ucode_api = (const void *)data; uint32_t api_index = le32toh(ucode_api->api_index); uint32_t api_flags = le32toh(ucode_api->api_flags); int i; if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) { device_printf(sc->sc_dev, "api flags index %d larger than supported by driver\n", api_index); /* don't return an error so we can load FW that has more bits */ return 0; } for (i = 0; i < 32; i++) { if (api_flags & (1U << i)) setbit(capa->enabled_api, i + 32 * api_index); } return 0; } static int iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data, struct iwm_ucode_capabilities *capa) { const struct iwm_ucode_capa *ucode_capa = (const void *)data; uint32_t api_index = le32toh(ucode_capa->api_index); uint32_t api_flags = le32toh(ucode_capa->api_capa); int i; if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) { device_printf(sc->sc_dev, "capa flags index %d larger than supported by driver\n", api_index); /* don't return an error so we can load FW that has more bits */ return 0; } for (i = 0; i < 32; i++) { if (api_flags & (1U << i)) setbit(capa->enabled_capa, i + 32 * api_index); } return 0; } static void iwm_fw_info_free(struct iwm_fw_info *fw) { firmware_put(fw->fw_fp, FIRMWARE_UNLOAD); fw->fw_fp = NULL; /* don't touch fw->fw_status */ memset(fw->fw_sects, 0, sizeof(fw->fw_sects)); } static int iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) { struct iwm_fw_info *fw = &sc->sc_fw; const struct iwm_tlv_ucode_header *uhdr; const struct iwm_ucode_tlv *tlv; struct iwm_ucode_capabilities *capa = &sc->ucode_capa; enum iwm_ucode_tlv_type tlv_type; const struct firmware *fwp; const uint8_t *data; uint32_t tlv_len; uint32_t usniffer_img; const uint8_t *tlv_data; uint32_t paging_mem_size; int num_of_cpus; int error = 0; size_t len; if (fw->fw_status == IWM_FW_STATUS_DONE && ucode_type != IWM_UCODE_INIT) return 0; while (fw->fw_status == IWM_FW_STATUS_INPROGRESS) msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfwp", 0); fw->fw_status = IWM_FW_STATUS_INPROGRESS; if (fw->fw_fp != NULL) iwm_fw_info_free(fw); /* * Load firmware into driver memory. * fw_fp will be set. */ IWM_UNLOCK(sc); fwp = firmware_get(sc->cfg->fw_name); IWM_LOCK(sc); if (fwp == NULL) { device_printf(sc->sc_dev, "could not read firmware %s (error %d)\n", sc->cfg->fw_name, error); goto out; } fw->fw_fp = fwp; /* (Re-)Initialize default values. */ capa->flags = 0; capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH; capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS; memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa)); memset(capa->enabled_api, 0, sizeof(capa->enabled_api)); memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc)); /* * Parse firmware contents */ uhdr = (const void *)fw->fw_fp->data; if (*(const uint32_t *)fw->fw_fp->data != 0 || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) { device_printf(sc->sc_dev, "invalid firmware %s\n", sc->cfg->fw_name); error = EINVAL; goto out; } snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%u.%u (API ver %u)", IWM_UCODE_MAJOR(le32toh(uhdr->ver)), IWM_UCODE_MINOR(le32toh(uhdr->ver)), IWM_UCODE_API(le32toh(uhdr->ver))); data = uhdr->data; len = fw->fw_fp->datasize - sizeof(*uhdr); while (len >= sizeof(*tlv)) { len -= sizeof(*tlv); tlv = (const void *)data; tlv_len = le32toh(tlv->length); tlv_type = le32toh(tlv->type); tlv_data = tlv->data; if (len < tlv_len) { device_printf(sc->sc_dev, "firmware too short: %zu bytes\n", len); error = EINVAL; goto parse_out; } len -= roundup2(tlv_len, 4); data += sizeof(tlv) + roundup2(tlv_len, 4); switch ((int)tlv_type) { case IWM_UCODE_TLV_PROBE_MAX_LEN: if (tlv_len != sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: PROBE_MAX_LEN (%d) != sizeof(uint32_t)\n", __func__, (int) tlv_len); error = EINVAL; goto parse_out; } capa->max_probe_length = le32_to_cpup((const uint32_t *)tlv_data); /* limit it to something sensible */ if (capa->max_probe_length > IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) { IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "%s: IWM_UCODE_TLV_PROBE_MAX_LEN " "ridiculous\n", __func__); error = EINVAL; goto parse_out; } break; case IWM_UCODE_TLV_PAN: if (tlv_len) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_PAN: tlv_len (%d) > 0\n", __func__, (int) tlv_len); error = EINVAL; goto parse_out; } capa->flags |= IWM_UCODE_TLV_FLAGS_PAN; break; case IWM_UCODE_TLV_FLAGS: if (tlv_len < sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%d) < sizeof(uint32_t)\n", __func__, (int) tlv_len); error = EINVAL; goto parse_out; } if (tlv_len % sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%d) %% sizeof(uint32_t)\n", __func__, (int) tlv_len); error = EINVAL; goto parse_out; } /* * Apparently there can be many flags, but Linux driver * parses only the first one, and so do we. * * XXX: why does this override IWM_UCODE_TLV_PAN? * Intentional or a bug? Observations from * current firmware file: * 1) TLV_PAN is parsed first * 2) TLV_FLAGS contains TLV_FLAGS_PAN * ==> this resets TLV_PAN to itself... hnnnk */ capa->flags = le32_to_cpup((const uint32_t *)tlv_data); break; case IWM_UCODE_TLV_CSCHEME: if ((error = iwm_store_cscheme(sc, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: iwm_store_cscheme(): returned %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_NUM_OF_CPU: if (tlv_len != sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%d) != sizeof(uint32_t)\n", __func__, (int) tlv_len); error = EINVAL; goto parse_out; } num_of_cpus = le32_to_cpup((const uint32_t *)tlv_data); if (num_of_cpus == 2) { fw->fw_sects[IWM_UCODE_REGULAR].is_dual_cpus = TRUE; fw->fw_sects[IWM_UCODE_INIT].is_dual_cpus = TRUE; fw->fw_sects[IWM_UCODE_WOWLAN].is_dual_cpus = TRUE; } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) { device_printf(sc->sc_dev, "%s: Driver supports only 1 or 2 CPUs\n", __func__); error = EINVAL; goto parse_out; } break; case IWM_UCODE_TLV_SEC_RT: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_SEC_INIT: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_SEC_WOWLAN: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_DEF_CALIB: if (tlv_len != sizeof(struct iwm_tlv_calib_data)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%d) < sizeof(iwm_tlv_calib_data) (%d)\n", __func__, (int) tlv_len, (int) sizeof(struct iwm_tlv_calib_data)); error = EINVAL; goto parse_out; } if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) { device_printf(sc->sc_dev, "%s: iwm_set_default_calib() failed: %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_PHY_SKU: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%d) < sizeof(uint32_t)\n", __func__, (int) tlv_len); goto parse_out; } sc->sc_fw.phy_config = le32_to_cpup((const uint32_t *)tlv_data); sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config & IWM_FW_PHY_CFG_TX_CHAIN) >> IWM_FW_PHY_CFG_TX_CHAIN_POS; sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config & IWM_FW_PHY_CFG_RX_CHAIN) >> IWM_FW_PHY_CFG_RX_CHAIN_POS; break; case IWM_UCODE_TLV_API_CHANGES_SET: { if (tlv_len != sizeof(struct iwm_ucode_api)) { error = EINVAL; goto parse_out; } if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) { error = EINVAL; goto parse_out; } break; } case IWM_UCODE_TLV_ENABLED_CAPABILITIES: { if (tlv_len != sizeof(struct iwm_ucode_capa)) { error = EINVAL; goto parse_out; } if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) { error = EINVAL; goto parse_out; } break; } case 48: /* undocumented TLV */ case IWM_UCODE_TLV_SDIO_ADMA_ADDR: case IWM_UCODE_TLV_FW_GSCAN_CAPA: /* ignore, not used by current driver */ break; case IWM_UCODE_TLV_SEC_RT_USNIFFER: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_REGULAR_USNIFFER, tlv_data, tlv_len)) != 0) goto parse_out; break; case IWM_UCODE_TLV_PAGING: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; goto parse_out; } paging_mem_size = le32_to_cpup((const uint32_t *)tlv_data); IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "%s: Paging: paging enabled (size = %u bytes)\n", __func__, paging_mem_size); if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) { device_printf(sc->sc_dev, "%s: Paging: driver supports up to %u bytes for paging image\n", __func__, IWM_MAX_PAGING_IMAGE_SIZE); error = EINVAL; goto out; } if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) { device_printf(sc->sc_dev, "%s: Paging: image isn't multiple %u\n", __func__, IWM_FW_PAGING_SIZE); error = EINVAL; goto out; } sc->sc_fw.fw_sects[IWM_UCODE_REGULAR].paging_mem_size = paging_mem_size; usniffer_img = IWM_UCODE_REGULAR_USNIFFER; sc->sc_fw.fw_sects[usniffer_img].paging_mem_size = paging_mem_size; break; case IWM_UCODE_TLV_N_SCAN_CHANNELS: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; goto parse_out; } capa->n_scan_channels = le32_to_cpup((const uint32_t *)tlv_data); break; case IWM_UCODE_TLV_FW_VERSION: if (tlv_len != sizeof(uint32_t) * 3) { error = EINVAL; goto parse_out; } snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d.%d", le32toh(((const uint32_t *)tlv_data)[0]), le32toh(((const uint32_t *)tlv_data)[1]), le32toh(((const uint32_t *)tlv_data)[2])); break; case IWM_UCODE_TLV_FW_MEM_SEG: break; default: device_printf(sc->sc_dev, "%s: unknown firmware section %d, abort\n", __func__, tlv_type); error = EINVAL; goto parse_out; } } KASSERT(error == 0, ("unhandled error")); parse_out: if (error) { device_printf(sc->sc_dev, "firmware parse error %d, " "section type %d\n", error, tlv_type); } out: if (error) { fw->fw_status = IWM_FW_STATUS_NONE; if (fw->fw_fp != NULL) iwm_fw_info_free(fw); } else fw->fw_status = IWM_FW_STATUS_DONE; wakeup(&sc->sc_fw); return error; } /* * DMA resource routines */ /* fwmem is used to load firmware onto the card */ static int iwm_alloc_fwmem(struct iwm_softc *sc) { /* Must be aligned on a 16-byte boundary. */ return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma, IWM_FH_MEM_TB_MAX_LENGTH, 16); } /* tx scheduler rings. not used? */ static int iwm_alloc_sched(struct iwm_softc *sc) { /* TX scheduler rings must be aligned on a 1KB boundary. */ return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma, nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024); } /* keep-warm page is used internally by the card. see iwl-fh.h for more info */ static int iwm_alloc_kw(struct iwm_softc *sc) { return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096); } /* interrupt cause table */ static int iwm_alloc_ict(struct iwm_softc *sc) { return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma, IWM_ICT_SIZE, 1<cur = 0; /* Allocate RX descriptors (256-byte aligned). */ size = IWM_RX_RING_COUNT * sizeof(uint32_t); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256); if (error != 0) { device_printf(sc->sc_dev, "could not allocate RX ring DMA memory\n"); goto fail; } ring->desc = ring->desc_dma.vaddr; /* Allocate RX status area (16-byte aligned). */ error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma, sizeof(*ring->stat), 16); if (error != 0) { device_printf(sc->sc_dev, "could not allocate RX status DMA memory\n"); goto fail; } ring->stat = ring->stat_dma.vaddr; /* Create RX buffer DMA tag. */ error = bus_dma_tag_create(sc->sc_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA tag, error %d\n", __func__, error); goto fail; } /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */ error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA map, error %d\n", __func__, error); goto fail; } /* * Allocate and map RX buffers. */ for (i = 0; i < IWM_RX_RING_COUNT; i++) { struct iwm_rx_data *data = &ring->data[i]; error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA map, error %d\n", __func__, error); goto fail; } data->m = NULL; if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) { goto fail; } } return 0; fail: iwm_free_rx_ring(sc, ring); return error; } static void iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring) { /* Reset the ring state */ ring->cur = 0; /* * The hw rx ring index in shared memory must also be cleared, * otherwise the discrepancy can cause reprocessing chaos. */ memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat)); } static void iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring) { int i; iwm_dma_contig_free(&ring->desc_dma); iwm_dma_contig_free(&ring->stat_dma); for (i = 0; i < IWM_RX_RING_COUNT; i++) { struct iwm_rx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->map != NULL) { bus_dmamap_destroy(ring->data_dmat, data->map); data->map = NULL; } } if (ring->spare_map != NULL) { bus_dmamap_destroy(ring->data_dmat, ring->spare_map); ring->spare_map = NULL; } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } static int iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid) { bus_addr_t paddr; bus_size_t size; size_t maxsize; int nsegments; int i, error; ring->qid = qid; ring->queued = 0; ring->cur = 0; /* Allocate TX descriptors (256-byte aligned). */ size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256); if (error != 0) { device_printf(sc->sc_dev, "could not allocate TX ring DMA memory\n"); goto fail; } ring->desc = ring->desc_dma.vaddr; /* * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need * to allocate commands space for other rings. */ if (qid > IWM_MVM_CMD_QUEUE) return 0; size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4); if (error != 0) { device_printf(sc->sc_dev, "could not allocate TX cmd DMA memory\n"); goto fail; } ring->cmd = ring->cmd_dma.vaddr; /* FW commands may require more mapped space than packets. */ if (qid == IWM_MVM_CMD_QUEUE) { maxsize = IWM_RBUF_SIZE; nsegments = 1; } else { maxsize = MCLBYTES; nsegments = IWM_MAX_SCATTER - 2; } error = bus_dma_tag_create(sc->sc_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize, nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "could not create TX buf DMA tag\n"); goto fail; } paddr = ring->cmd_dma.paddr; for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; data->cmd_paddr = paddr; data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header) + offsetof(struct iwm_tx_cmd, scratch); paddr += sizeof(struct iwm_device_cmd); error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "could not create TX buf DMA map\n"); goto fail; } } KASSERT(paddr == ring->cmd_dma.paddr + size, ("invalid physical address")); return 0; fail: iwm_free_tx_ring(sc, ring); return error; } static void iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring) { int i; for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } } /* Clear TX descriptors. */ memset(ring->desc, 0, ring->desc_dma.size); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); sc->qfullmsk &= ~(1 << ring->qid); ring->queued = 0; ring->cur = 0; if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake) iwm_pcie_clear_cmd_in_flight(sc); } static void iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring) { int i; iwm_dma_contig_free(&ring->desc_dma); iwm_dma_contig_free(&ring->cmd_dma); for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->map != NULL) { bus_dmamap_destroy(ring->data_dmat, data->map); data->map = NULL; } } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } /* * High-level hardware frobbing routines */ static void iwm_enable_interrupts(struct iwm_softc *sc) { sc->sc_intmask = IWM_CSR_INI_SET_MASK; IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } static void iwm_restore_interrupts(struct iwm_softc *sc) { IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } static void iwm_disable_interrupts(struct iwm_softc *sc) { /* disable interrupts */ IWM_WRITE(sc, IWM_CSR_INT_MASK, 0); /* acknowledge all interrupts */ IWM_WRITE(sc, IWM_CSR_INT, ~0); IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0); } static void iwm_ict_reset(struct iwm_softc *sc) { iwm_disable_interrupts(sc); /* Reset ICT table. */ memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE); sc->ict_cur = 0; /* Set physical address of ICT table (4KB aligned). */ IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG, IWM_CSR_DRAM_INT_TBL_ENABLE | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT); /* Switch to ICT interrupt mode in driver. */ sc->sc_flags |= IWM_FLAG_USE_ICT; /* Re-enable interrupts. */ IWM_WRITE(sc, IWM_CSR_INT, ~0); iwm_enable_interrupts(sc); } /* iwlwifi pcie/trans.c */ /* * Since this .. hard-resets things, it's time to actually * mark the first vap (if any) as having no mac context. * It's annoying, but since the driver is potentially being * stop/start'ed whilst active (thanks openbsd port!) we * have to correctly track this. */ static void iwm_stop_device(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); int chnl, qid; uint32_t mask = 0; /* tell the device to stop sending interrupts */ iwm_disable_interrupts(sc); /* * FreeBSD-local: mark the first vap as not-uploaded, * so the next transition through auth/assoc * will correctly populate the MAC context. */ if (vap) { struct iwm_vap *iv = IWM_VAP(vap); iv->phy_ctxt = NULL; iv->is_uploaded = 0; } /* device going down, Stop using ICT table */ sc->sc_flags &= ~IWM_FLAG_USE_ICT; /* stop tx and rx. tx and rx bits, as usual, are from if_iwn */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_SCD_TXFACT, 0); /* Stop each Tx DMA channel */ for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) { IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0); mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl); } /* Wait for DMA channels to be idle */ if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask, 5000)) { device_printf(sc->sc_dev, "Failing on timeout while stopping DMA channel: [0x%08x]\n", IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG)); } iwm_nic_unlock(sc); } iwm_pcie_rx_stop(sc); /* Stop RX ring. */ iwm_reset_rx_ring(sc, &sc->rxq); /* Reset all TX rings. */ for (qid = 0; qid < nitems(sc->txq); qid++) iwm_reset_tx_ring(sc, &sc->txq[qid]); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { /* Power-down device's busmaster DMA clocks */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG, IWM_APMG_CLK_VAL_DMA_CLK_RQT); iwm_nic_unlock(sc); } DELAY(5); } /* Make sure (redundant) we've released our request to stay awake */ IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); /* Stop the device, and put it in low power state */ iwm_apm_stop(sc); /* Upon stop, the APM issues an interrupt if HW RF kill is set. * Clean again the interrupt here */ iwm_disable_interrupts(sc); /* stop and reset the on-board processor */ IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET); /* * Even if we stop the HW, we still want the RF kill * interrupt */ iwm_enable_rfkill_int(sc); iwm_check_rfkill(sc); } /* iwlwifi: mvm/ops.c */ static void iwm_mvm_nic_config(struct iwm_softc *sc) { uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash; uint32_t reg_val = 0; uint32_t phy_config = iwm_mvm_get_phy_config(sc); radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >> IWM_FW_PHY_CFG_RADIO_TYPE_POS; radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >> IWM_FW_PHY_CFG_RADIO_STEP_POS; radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >> IWM_FW_PHY_CFG_RADIO_DASH_POS; /* SKU control */ reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) << IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP; reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) << IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH; /* radio configuration */ reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE; reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP; reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH; IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val); IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type, radio_cfg_step, radio_cfg_dash); /* * W/A : NIC is stuck in a reset state after Early PCIe power off * (PCIe power is lost before PERST# is asserted), causing ME FW * to lose ownership and not being able to obtain it back. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG, IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS, ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS); } } static int iwm_nic_rx_init(struct iwm_softc *sc) { /* * Initialize RX ring. This is from the iwn driver. */ memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat)); /* Stop Rx DMA */ iwm_pcie_rx_stop(sc); if (!iwm_nic_lock(sc)) return EBUSY; /* reset and flush pointers */ IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0); IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); /* Set physical address of RX ring (256-byte aligned). */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8); /* Set physical address of RX status (16-byte aligned). */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4); /* Enable Rx DMA * XXX 5000 HW isn't supported by the iwm(4) driver. * IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in * the credit mechanism in 5000 HW RX FIFO * Direct rx interrupts to hosts * Rx buffer size 4 or 8k or 12k * RB timeout 0x10 * 256 RBDs */ IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG, IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | /* HW bug */ IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K | (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS); IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF); /* W/A for interrupt coalescing bug in 7260 and 3160 */ if (sc->cfg->host_interrupt_operation_mode) IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE); /* * Thus sayeth el jefe (iwlwifi) via a comment: * * This value should initially be 0 (before preparing any * RBs), should be 8 after preparing the first 8 RBs (for example) */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8); iwm_nic_unlock(sc); return 0; } static int iwm_nic_tx_init(struct iwm_softc *sc) { int qid; if (!iwm_nic_lock(sc)) return EBUSY; /* Deactivate TX scheduler. */ iwm_write_prph(sc, IWM_SCD_TXFACT, 0); /* Set physical address of "keep warm" page (16-byte aligned). */ IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4); /* Initialize TX rings. */ for (qid = 0; qid < nitems(sc->txq); qid++) { struct iwm_tx_ring *txq = &sc->txq[qid]; /* Set physical address of TX ring (256-byte aligned). */ IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid), txq->desc_dma.paddr >> 8); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: loading ring %d descriptors (%p) at %lx\n", __func__, qid, txq->desc, (unsigned long) (txq->desc_dma.paddr >> 8)); } iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE); iwm_nic_unlock(sc); return 0; } static int iwm_nic_init(struct iwm_softc *sc) { int error; iwm_apm_init(sc); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) iwm_set_pwr(sc); iwm_mvm_nic_config(sc); if ((error = iwm_nic_rx_init(sc)) != 0) return error; /* * Ditto for TX, from iwn */ if ((error = iwm_nic_tx_init(sc)) != 0) return error; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: shadow registers enabled\n", __func__); IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff); return 0; } int iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo) { if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0); if (qid == IWM_MVM_CMD_QUEUE) { /* unactivate before configuration */ iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN)); iwm_nic_unlock(sc); iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid)); if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0); iwm_nic_unlock(sc); iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0); /* Set scheduler window size and frame limit. */ iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) + sizeof(uint32_t), ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) & IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) | ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) & IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK)); if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) | (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) | IWM_SCD_QUEUE_STTS_REG_MSK); } else { struct iwm_scd_txq_cfg_cmd cmd; int error; iwm_nic_unlock(sc); memset(&cmd, 0, sizeof(cmd)); cmd.scd_queue = qid; cmd.enable = 1; cmd.sta_id = sta_id; cmd.tx_fifo = fifo; cmd.aggregate = 0; cmd.window = IWM_FRAME_LIMIT; error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC, sizeof(cmd), &cmd); if (error) { device_printf(sc->sc_dev, "cannot enable txq %d\n", qid); return error; } if (!iwm_nic_lock(sc)) return EBUSY; } iwm_write_prph(sc, IWM_SCD_EN_CTRL, iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid); iwm_nic_unlock(sc); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n", __func__, qid, fifo); return 0; } static int iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr) { int error, chnl; int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND - IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t); if (!iwm_nic_lock(sc)) return EBUSY; iwm_ict_reset(sc); sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR); if (scd_base_addr != 0 && scd_base_addr != sc->scd_base_addr) { device_printf(sc->sc_dev, "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n", __func__, sc->scd_base_addr, scd_base_addr); } iwm_nic_unlock(sc); /* reset context data, TX status and translation data */ error = iwm_write_mem(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND, NULL, clear_dwords); if (error) return EBUSY; if (!iwm_nic_lock(sc)) return EBUSY; /* Set physical address of TX scheduler rings (1KB aligned). */ iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10); iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0); iwm_nic_unlock(sc); /* enable command channel */ error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7); if (error) return error; if (!iwm_nic_lock(sc)) return EBUSY; iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff); /* Enable DMA channels. */ for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) { IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); } IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG, IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); iwm_nic_unlock(sc); /* Enable L1-Active */ if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); } return error; } /* * NVM read access and content parsing. We do not support * external NVM or writing NVM. * iwlwifi/mvm/nvm.c */ /* Default NVM size to read */ #define IWM_NVM_DEFAULT_CHUNK_SIZE (2*1024) #define IWM_NVM_WRITE_OPCODE 1 #define IWM_NVM_READ_OPCODE 0 /* load nvm chunk response */ enum { IWM_READ_NVM_CHUNK_SUCCEED = 0, IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1 }; static int iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len) { struct iwm_nvm_access_cmd nvm_access_cmd = { .offset = htole16(offset), .length = htole16(length), .type = htole16(section), .op_code = IWM_NVM_READ_OPCODE, }; struct iwm_nvm_access_resp *nvm_resp; struct iwm_rx_packet *pkt; struct iwm_host_cmd cmd = { .id = IWM_NVM_ACCESS_CMD, .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL, .data = { &nvm_access_cmd, }, }; int ret, bytes_read, offset_read; uint8_t *resp_data; cmd.len[0] = sizeof(struct iwm_nvm_access_cmd); ret = iwm_send_cmd(sc, &cmd); if (ret) { device_printf(sc->sc_dev, "Could not send NVM_ACCESS command (error=%d)\n", ret); return ret; } pkt = cmd.resp_pkt; /* Extract NVM response */ nvm_resp = (void *)pkt->data; ret = le16toh(nvm_resp->status); bytes_read = le16toh(nvm_resp->length); offset_read = le16toh(nvm_resp->offset); resp_data = nvm_resp->data; if (ret) { if ((offset != 0) && (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) { /* * meaning of NOT_VALID_ADDRESS: * driver try to read chunk from address that is * multiple of 2K and got an error since addr is empty. * meaning of (offset != 0): driver already * read valid data from another chunk so this case * is not an error. */ IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM access command failed on offset 0x%x since that section size is multiple 2K\n", offset); *len = 0; ret = 0; } else { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM access command failed with status %d\n", ret); ret = EIO; } goto exit; } if (offset_read != offset) { device_printf(sc->sc_dev, "NVM ACCESS response with invalid offset %d\n", offset_read); ret = EINVAL; goto exit; } if (bytes_read > length) { device_printf(sc->sc_dev, "NVM ACCESS response with too much data " "(%d bytes requested, %d bytes received)\n", length, bytes_read); ret = EINVAL; goto exit; } /* Write data to NVM */ memcpy(data + offset, resp_data, bytes_read); *len = bytes_read; exit: iwm_free_resp(sc, &cmd); return ret; } /* * Reads an NVM section completely. * NICs prior to 7000 family don't have a real NVM, but just read * section 0 which is the EEPROM. Because the EEPROM reading is unlimited * by uCode, we need to manually check in this case that we don't * overflow and try to read more than the EEPROM size. * For 7000 family NICs, we supply the maximal size we can read, and * the uCode fills the response with as much data as we can, * without overflowing, so no check is needed. */ static int iwm_nvm_read_section(struct iwm_softc *sc, uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read) { uint16_t seglen, length, offset = 0; int ret; /* Set nvm section read length */ length = IWM_NVM_DEFAULT_CHUNK_SIZE; seglen = length; /* Read the NVM until exhausted (reading less than requested) */ while (seglen == length) { /* Check no memory assumptions fail and cause an overflow */ if ((size_read + offset + length) > sc->cfg->eeprom_size) { device_printf(sc->sc_dev, "EEPROM size is too small for NVM\n"); return ENOBUFS; } ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen); if (ret) { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "Cannot read NVM from section %d offset %d, length %d\n", section, offset, length); return ret; } offset += seglen; } IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM section %d read completed\n", section); *len = offset; return 0; } /* * BEGIN IWM_NVM_PARSE */ /* iwlwifi/iwl-nvm-parse.c */ /* NVM offsets (in words) definitions */ enum iwm_nvm_offsets { /* NVM HW-Section offset (in words) definitions */ IWM_HW_ADDR = 0x15, /* NVM SW-Section offset (in words) definitions */ IWM_NVM_SW_SECTION = 0x1C0, IWM_NVM_VERSION = 0, IWM_RADIO_CFG = 1, IWM_SKU = 2, IWM_N_HW_ADDRS = 3, IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION, /* NVM calibration section offset (in words) definitions */ IWM_NVM_CALIB_SECTION = 0x2B8, IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION }; enum iwm_8000_nvm_offsets { /* NVM HW-Section offset (in words) definitions */ IWM_HW_ADDR0_WFPM_8000 = 0x12, IWM_HW_ADDR1_WFPM_8000 = 0x16, IWM_HW_ADDR0_PCIE_8000 = 0x8A, IWM_HW_ADDR1_PCIE_8000 = 0x8E, IWM_MAC_ADDRESS_OVERRIDE_8000 = 1, /* NVM SW-Section offset (in words) definitions */ IWM_NVM_SW_SECTION_8000 = 0x1C0, IWM_NVM_VERSION_8000 = 0, IWM_RADIO_CFG_8000 = 0, IWM_SKU_8000 = 2, IWM_N_HW_ADDRS_8000 = 3, /* NVM REGULATORY -Section offset (in words) definitions */ IWM_NVM_CHANNELS_8000 = 0, IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7, IWM_NVM_LAR_OFFSET_8000 = 0x507, IWM_NVM_LAR_ENABLED_8000 = 0x7, /* NVM calibration section offset (in words) definitions */ IWM_NVM_CALIB_SECTION_8000 = 0x2B8, IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000 }; /* SKU Capabilities (actual values from NVM definition) */ enum nvm_sku_bits { IWM_NVM_SKU_CAP_BAND_24GHZ = (1 << 0), IWM_NVM_SKU_CAP_BAND_52GHZ = (1 << 1), IWM_NVM_SKU_CAP_11N_ENABLE = (1 << 2), IWM_NVM_SKU_CAP_11AC_ENABLE = (1 << 3), }; /* radio config bits (actual values from NVM definition) */ #define IWM_NVM_RF_CFG_DASH_MSK(x) (x & 0x3) /* bits 0-1 */ #define IWM_NVM_RF_CFG_STEP_MSK(x) ((x >> 2) & 0x3) /* bits 2-3 */ #define IWM_NVM_RF_CFG_TYPE_MSK(x) ((x >> 4) & 0x3) /* bits 4-5 */ #define IWM_NVM_RF_CFG_PNUM_MSK(x) ((x >> 6) & 0x3) /* bits 6-7 */ #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8) & 0xF) /* bits 8-11 */ #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */ #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x) (x & 0xF) #define IWM_NVM_RF_CFG_DASH_MSK_8000(x) ((x >> 4) & 0xF) #define IWM_NVM_RF_CFG_STEP_MSK_8000(x) ((x >> 8) & 0xF) #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x) ((x >> 12) & 0xFFF) #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x) ((x >> 24) & 0xF) #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x) ((x >> 28) & 0xF) #define DEFAULT_MAX_TX_POWER 16 /** * enum iwm_nvm_channel_flags - channel flags in NVM * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed * @IWM_NVM_CHANNEL_RADAR: radar detection required * XXX cannot find this (DFS) flag in iwm-nvm-parse.c * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?) * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?) * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?) * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?) */ enum iwm_nvm_channel_flags { IWM_NVM_CHANNEL_VALID = (1 << 0), IWM_NVM_CHANNEL_IBSS = (1 << 1), IWM_NVM_CHANNEL_ACTIVE = (1 << 3), IWM_NVM_CHANNEL_RADAR = (1 << 4), IWM_NVM_CHANNEL_DFS = (1 << 7), IWM_NVM_CHANNEL_WIDE = (1 << 8), IWM_NVM_CHANNEL_40MHZ = (1 << 9), IWM_NVM_CHANNEL_80MHZ = (1 << 10), IWM_NVM_CHANNEL_160MHZ = (1 << 11), }; /* * Translate EEPROM flags to net80211. */ static uint32_t iwm_eeprom_channel_flags(uint16_t ch_flags) { uint32_t nflags; nflags = 0; if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0) nflags |= IEEE80211_CHAN_PASSIVE; if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0) nflags |= IEEE80211_CHAN_NOADHOC; if (ch_flags & IWM_NVM_CHANNEL_RADAR) { nflags |= IEEE80211_CHAN_DFS; /* Just in case. */ nflags |= IEEE80211_CHAN_NOADHOC; } return (nflags); } static void iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[], int maxchans, int *nchans, int ch_idx, size_t ch_num, const uint8_t bands[]) { const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags; uint32_t nflags; uint16_t ch_flags; uint8_t ieee; int error; for (; ch_idx < ch_num; ch_idx++) { ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) ieee = iwm_nvm_channels[ch_idx]; else ieee = iwm_nvm_channels_8000[ch_idx]; if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Ch. %d Flags %x [%sGHz] - No traffic\n", ieee, ch_flags, (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ? "5.2" : "2.4"); continue; } nflags = iwm_eeprom_channel_flags(ch_flags); error = ieee80211_add_channel(chans, maxchans, nchans, ieee, 0, 0, nflags, bands); if (error != 0) break; IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Ch. %d Flags %x [%sGHz] - Added\n", ieee, ch_flags, (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ? "5.2" : "2.4"); } } static void iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans, struct ieee80211_channel chans[]) { struct iwm_softc *sc = ic->ic_softc; struct iwm_nvm_data *data = sc->nvm_data; uint8_t bands[IEEE80211_MODE_BYTES]; size_t ch_num; memset(bands, 0, sizeof(bands)); /* 1-13: 11b/g channels. */ setbit(bands, IEEE80211_MODE_11B); setbit(bands, IEEE80211_MODE_11G); iwm_add_channel_band(sc, chans, maxchans, nchans, 0, IWM_NUM_2GHZ_CHANNELS - 1, bands); /* 14: 11b channel only. */ clrbit(bands, IEEE80211_MODE_11G); iwm_add_channel_band(sc, chans, maxchans, nchans, IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands); if (data->sku_cap_band_52GHz_enable) { if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) ch_num = nitems(iwm_nvm_channels); else ch_num = nitems(iwm_nvm_channels_8000); memset(bands, 0, sizeof(bands)); setbit(bands, IEEE80211_MODE_11A); iwm_add_channel_band(sc, chans, maxchans, nchans, IWM_NUM_2GHZ_CHANNELS, ch_num, bands); } } static void iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data, const uint16_t *mac_override, const uint16_t *nvm_hw) { const uint8_t *hw_addr; if (mac_override) { static const uint8_t reserved_mac[] = { 0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00 }; hw_addr = (const uint8_t *)(mac_override + IWM_MAC_ADDRESS_OVERRIDE_8000); /* * Store the MAC address from MAO section. * No byte swapping is required in MAO section */ IEEE80211_ADDR_COPY(data->hw_addr, hw_addr); /* * Force the use of the OTP MAC address in case of reserved MAC * address in the NVM, or if address is given but invalid. */ if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) && !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) && iwm_is_valid_ether_addr(data->hw_addr) && !IEEE80211_IS_MULTICAST(data->hw_addr)) return; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: mac address from nvm override section invalid\n", __func__); } if (nvm_hw) { /* read the mac address from WFMP registers */ uint32_t mac_addr0 = htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0)); uint32_t mac_addr1 = htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1)); hw_addr = (const uint8_t *)&mac_addr0; data->hw_addr[0] = hw_addr[3]; data->hw_addr[1] = hw_addr[2]; data->hw_addr[2] = hw_addr[1]; data->hw_addr[3] = hw_addr[0]; hw_addr = (const uint8_t *)&mac_addr1; data->hw_addr[4] = hw_addr[1]; data->hw_addr[5] = hw_addr[0]; return; } device_printf(sc->sc_dev, "%s: mac address not found\n", __func__); memset(data->hw_addr, 0, sizeof(data->hw_addr)); } static int iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw, const uint16_t *phy_sku) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_SKU); return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000)); } static int iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_NVM_VERSION); else return le32_to_cpup((const uint32_t *)(nvm_sw + IWM_NVM_VERSION_8000)); } static int iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw, const uint16_t *phy_sku) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_RADIO_CFG); return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000)); } static int iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw) { int n_hw_addr; if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS); n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000)); return n_hw_addr & IWM_N_HW_ADDR_MASK; } static void iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data, uint32_t radio_cfg) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg); data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg); data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg); data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg); return; } /* set the radio configuration for family 8000 */ data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg); data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg); data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg); data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg); data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg); data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg); } static int iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data, const uint16_t *nvm_hw, const uint16_t *mac_override) { #ifdef notyet /* for FAMILY 9000 */ if (cfg->mac_addr_from_csr) { iwm_set_hw_address_from_csr(sc, data); } else #endif if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR); /* The byte order is little endian 16 bit, meaning 214365 */ data->hw_addr[0] = hw_addr[1]; data->hw_addr[1] = hw_addr[0]; data->hw_addr[2] = hw_addr[3]; data->hw_addr[3] = hw_addr[2]; data->hw_addr[4] = hw_addr[5]; data->hw_addr[5] = hw_addr[4]; } else { iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw); } if (!iwm_is_valid_ether_addr(data->hw_addr)) { device_printf(sc->sc_dev, "no valid mac address was found\n"); return EINVAL; } return 0; } static struct iwm_nvm_data * iwm_parse_nvm_data(struct iwm_softc *sc, const uint16_t *nvm_hw, const uint16_t *nvm_sw, const uint16_t *nvm_calib, const uint16_t *mac_override, const uint16_t *phy_sku, const uint16_t *regulatory) { struct iwm_nvm_data *data; uint32_t sku, radio_cfg; uint16_t lar_config; if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { data = malloc(sizeof(*data) + IWM_NUM_CHANNELS * sizeof(uint16_t), M_DEVBUF, M_NOWAIT | M_ZERO); } else { data = malloc(sizeof(*data) + IWM_NUM_CHANNELS_8000 * sizeof(uint16_t), M_DEVBUF, M_NOWAIT | M_ZERO); } if (!data) return NULL; data->nvm_version = iwm_get_nvm_version(sc, nvm_sw); radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku); iwm_set_radio_cfg(sc, data, radio_cfg); sku = iwm_get_sku(sc, nvm_sw, phy_sku); data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ; data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ; data->sku_cap_11n_enable = 0; data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { uint16_t lar_offset = data->nvm_version < 0xE39 ? IWM_NVM_LAR_OFFSET_8000_OLD : IWM_NVM_LAR_OFFSET_8000; lar_config = le16_to_cpup(regulatory + lar_offset); data->lar_enabled = !!(lar_config & IWM_NVM_LAR_ENABLED_8000); } /* If no valid mac address was found - bail out */ if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) { free(data, M_DEVBUF); return NULL; } if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS], IWM_NUM_CHANNELS * sizeof(uint16_t)); } else { memcpy(data->nvm_ch_flags, ®ulatory[IWM_NVM_CHANNELS_8000], IWM_NUM_CHANNELS_8000 * sizeof(uint16_t)); } return data; } static void iwm_free_nvm_data(struct iwm_nvm_data *data) { if (data != NULL) free(data, M_DEVBUF); } static struct iwm_nvm_data * iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections) { const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku; /* Checking for required sections */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { if (!sections[IWM_NVM_SECTION_TYPE_SW].data || !sections[sc->cfg->nvm_hw_section_num].data) { device_printf(sc->sc_dev, "Can't parse empty OTP/NVM sections\n"); return NULL; } } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { /* SW and REGULATORY sections are mandatory */ if (!sections[IWM_NVM_SECTION_TYPE_SW].data || !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) { device_printf(sc->sc_dev, "Can't parse empty OTP/NVM sections\n"); return NULL; } /* MAC_OVERRIDE or at least HW section must exist */ if (!sections[sc->cfg->nvm_hw_section_num].data && !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) { device_printf(sc->sc_dev, "Can't parse mac_address, empty sections\n"); return NULL; } /* PHY_SKU section is mandatory in B0 */ if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) { device_printf(sc->sc_dev, "Can't parse phy_sku in B0, empty sections\n"); return NULL; } } else { panic("unknown device family %d\n", sc->cfg->device_family); } hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data; sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data; calib = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data; regulatory = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_REGULATORY].data; mac_override = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data; phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data; return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override, phy_sku, regulatory); } static int iwm_nvm_init(struct iwm_softc *sc) { struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS]; int i, ret, section; uint32_t size_read = 0; uint8_t *nvm_buffer, *temp; uint16_t len; memset(nvm_sections, 0, sizeof(nvm_sections)); if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS) return EINVAL; /* load NVM values from nic */ /* Read From FW NVM */ IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n"); nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO); if (!nvm_buffer) return ENOMEM; for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) { /* we override the constness for initial read */ ret = iwm_nvm_read_section(sc, section, nvm_buffer, &len, size_read); if (ret) continue; size_read += len; temp = malloc(len, M_DEVBUF, M_NOWAIT); if (!temp) { ret = ENOMEM; break; } memcpy(temp, nvm_buffer, len); nvm_sections[section].data = temp; nvm_sections[section].length = len; } if (!size_read) device_printf(sc->sc_dev, "OTP is blank\n"); free(nvm_buffer, M_DEVBUF); sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections); if (!sc->nvm_data) return EINVAL; IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "nvm version = %x\n", sc->nvm_data->nvm_version); for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) { if (nvm_sections[i].data != NULL) free(nvm_sections[i].data, M_DEVBUF); } return 0; } static int iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num, const struct iwm_fw_desc *section) { struct iwm_dma_info *dma = &sc->fw_dma; uint8_t *v_addr; bus_addr_t p_addr; uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len); int ret = 0; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: [%d] uCode section being loaded...\n", __func__, section_num); v_addr = dma->vaddr; p_addr = dma->paddr; for (offset = 0; offset < section->len; offset += chunk_sz) { uint32_t copy_size, dst_addr; int extended_addr = FALSE; copy_size = MIN(chunk_sz, section->len - offset); dst_addr = section->offset + offset; if (dst_addr >= IWM_FW_MEM_EXTENDED_START && dst_addr <= IWM_FW_MEM_EXTENDED_END) extended_addr = TRUE; if (extended_addr) iwm_set_bits_prph(sc, IWM_LMPM_CHICK, IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); memcpy(v_addr, (const uint8_t *)section->data + offset, copy_size); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr, copy_size); if (extended_addr) iwm_clear_bits_prph(sc, IWM_LMPM_CHICK, IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); if (ret) { device_printf(sc->sc_dev, "%s: Could not load the [%d] uCode section\n", __func__, section_num); break; } } return ret; } /* * ucode */ static int iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr, bus_addr_t phy_addr, uint32_t byte_cnt) { int ret; sc->sc_fw_chunk_done = 0; if (!iwm_nic_lock(sc)) return EBUSY; IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE); IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL), dst_addr); IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL), phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK); IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL), (iwm_get_dma_hi_addr(phy_addr) << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt); IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL), 1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM | 1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX | IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID); IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD); iwm_nic_unlock(sc); /* wait up to 5s for this segment to load */ ret = 0; while (!sc->sc_fw_chunk_done) { ret = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz); if (ret) break; } if (ret != 0) { device_printf(sc->sc_dev, "fw chunk addr 0x%x len %d failed to load\n", dst_addr, byte_cnt); return ETIMEDOUT; } return 0; } static int iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc, const struct iwm_fw_sects *image, int cpu, int *first_ucode_section) { int shift_param; int i, ret = 0, sec_num = 0x1; uint32_t val, last_read_idx = 0; if (cpu == 1) { shift_param = 0; *first_ucode_section = 0; } else { shift_param = 16; (*first_ucode_section)++; } for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) { last_read_idx = i; /* * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between * CPU1 to CPU2. * PAGING_SEPARATOR_SECTION delimiter - separate between * CPU2 non paged to CPU2 paging sec. */ if (!image->fw_sect[i].data || image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION || image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) { IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Break since Data not valid or Empty section, sec = %d\n", i); break; } ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]); if (ret) return ret; /* Notify the ucode of the loaded section number and status */ if (iwm_nic_lock(sc)) { val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS); val = val | (sec_num << shift_param); IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val); sec_num = (sec_num << 1) | 0x1; iwm_nic_unlock(sc); } } *first_ucode_section = last_read_idx; iwm_enable_interrupts(sc); if (iwm_nic_lock(sc)) { if (cpu == 1) IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF); else IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF); iwm_nic_unlock(sc); } return 0; } static int iwm_pcie_load_cpu_sections(struct iwm_softc *sc, const struct iwm_fw_sects *image, int cpu, int *first_ucode_section) { int shift_param; int i, ret = 0; uint32_t last_read_idx = 0; if (cpu == 1) { shift_param = 0; *first_ucode_section = 0; } else { shift_param = 16; (*first_ucode_section)++; } for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) { last_read_idx = i; /* * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between * CPU1 to CPU2. * PAGING_SEPARATOR_SECTION delimiter - separate between * CPU2 non paged to CPU2 paging sec. */ if (!image->fw_sect[i].data || image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION || image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) { IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Break since Data not valid or Empty section, sec = %d\n", i); break; } ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]); if (ret) return ret; } *first_ucode_section = last_read_idx; return 0; } static int iwm_pcie_load_given_ucode(struct iwm_softc *sc, const struct iwm_fw_sects *image) { int ret = 0; int first_ucode_section; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n", image->is_dual_cpus ? "Dual" : "Single"); /* load to FW the binary non secured sections of CPU1 */ ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section); if (ret) return ret; if (image->is_dual_cpus) { /* set CPU2 header address */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR, IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE); iwm_nic_unlock(sc); } /* load to FW the binary sections of CPU2 */ ret = iwm_pcie_load_cpu_sections(sc, image, 2, &first_ucode_section); if (ret) return ret; } iwm_enable_interrupts(sc); /* release CPU reset */ IWM_WRITE(sc, IWM_CSR_RESET, 0); return 0; } int iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc, const struct iwm_fw_sects *image) { int ret = 0; int first_ucode_section; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n", image->is_dual_cpus ? "Dual" : "Single"); /* configure the ucode to be ready to get the secured image */ /* release CPU reset */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT); iwm_nic_unlock(sc); } /* load to FW the binary Secured sections of CPU1 */ ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1, &first_ucode_section); if (ret) return ret; /* load to FW the binary sections of CPU2 */ return iwm_pcie_load_cpu_sections_8000(sc, image, 2, &first_ucode_section); } /* XXX Get rid of this definition */ static inline void iwm_enable_fw_load_int(struct iwm_softc *sc) { IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n"); sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX; IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } /* XXX Add proper rfkill support code */ static int iwm_start_fw(struct iwm_softc *sc, const struct iwm_fw_sects *fw) { int ret; /* This may fail if AMT took ownership of the device */ if (iwm_prepare_card_hw(sc)) { device_printf(sc->sc_dev, "%s: Exit HW not ready\n", __func__); ret = EIO; goto out; } IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF); iwm_disable_interrupts(sc); /* make sure rfkill handshake bits are cleared */ IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED); /* clear (again), then enable host interrupts */ IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF); ret = iwm_nic_init(sc); if (ret) { device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__); goto out; } /* * Now, we load the firmware and don't want to be interrupted, even * by the RF-Kill interrupt (hence mask all the interrupt besides the * FH_TX interrupt which is needed to load the firmware). If the * RF-Kill switch is toggled, we will find out after having loaded * the firmware and return the proper value to the caller. */ iwm_enable_fw_load_int(sc); /* really make sure rfkill handshake bits are cleared */ /* maybe we should write a few times more? just to make sure */ IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); /* Load the given image to the HW */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) ret = iwm_pcie_load_given_ucode_8000(sc, fw); else ret = iwm_pcie_load_given_ucode(sc, fw); /* XXX re-check RF-Kill state */ out: return ret; } static int iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant) { struct iwm_tx_ant_cfg_cmd tx_ant_cmd = { .valid = htole32(valid_tx_ant), }; return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD, IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd); } /* iwlwifi: mvm/fw.c */ static int iwm_send_phy_cfg_cmd(struct iwm_softc *sc) { struct iwm_phy_cfg_cmd phy_cfg_cmd; enum iwm_ucode_type ucode_type = sc->cur_ucode; /* Set parameters */ phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc)); phy_cfg_cmd.calib_control.event_trigger = sc->sc_default_calib[ucode_type].event_trigger; phy_cfg_cmd.calib_control.flow_trigger = sc->sc_default_calib[ucode_type].flow_trigger; IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET, "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg); return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC, sizeof(phy_cfg_cmd), &phy_cfg_cmd); } static int iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data) { struct iwm_mvm_alive_data *alive_data = data; struct iwm_mvm_alive_resp_ver1 *palive1; struct iwm_mvm_alive_resp_ver2 *palive2; struct iwm_mvm_alive_resp *palive; if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive1)) { palive1 = (void *)pkt->data; sc->support_umac_log = FALSE; sc->error_event_table = le32toh(palive1->error_event_table_ptr); sc->log_event_table = le32toh(palive1->log_event_table_ptr); alive_data->scd_base_addr = le32toh(palive1->scd_base_ptr); alive_data->valid = le16toh(palive1->status) == IWM_ALIVE_STATUS_OK; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Alive VER1 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n", le16toh(palive1->status), palive1->ver_type, palive1->ver_subtype, palive1->flags); } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive2)) { palive2 = (void *)pkt->data; sc->error_event_table = le32toh(palive2->error_event_table_ptr); sc->log_event_table = le32toh(palive2->log_event_table_ptr); alive_data->scd_base_addr = le32toh(palive2->scd_base_ptr); sc->umac_error_event_table = le32toh(palive2->error_info_addr); alive_data->valid = le16toh(palive2->status) == IWM_ALIVE_STATUS_OK; if (sc->umac_error_event_table) sc->support_umac_log = TRUE; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Alive VER2 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n", le16toh(palive2->status), palive2->ver_type, palive2->ver_subtype, palive2->flags); IWM_DPRINTF(sc, IWM_DEBUG_RESET, "UMAC version: Major - 0x%x, Minor - 0x%x\n", palive2->umac_major, palive2->umac_minor); } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) { palive = (void *)pkt->data; sc->error_event_table = le32toh(palive->error_event_table_ptr); sc->log_event_table = le32toh(palive->log_event_table_ptr); alive_data->scd_base_addr = le32toh(palive->scd_base_ptr); sc->umac_error_event_table = le32toh(palive->error_info_addr); alive_data->valid = le16toh(palive->status) == IWM_ALIVE_STATUS_OK; if (sc->umac_error_event_table) sc->support_umac_log = TRUE; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Alive VER3 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n", le16toh(palive->status), palive->ver_type, palive->ver_subtype, palive->flags); IWM_DPRINTF(sc, IWM_DEBUG_RESET, "UMAC version: Major - 0x%x, Minor - 0x%x\n", le32toh(palive->umac_major), le32toh(palive->umac_minor)); } return TRUE; } static int iwm_wait_phy_db_entry(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data) { struct iwm_phy_db *phy_db = data; if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) { if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) { device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n", __func__, pkt->hdr.code); } return TRUE; } if (iwm_phy_db_set_section(phy_db, pkt)) { device_printf(sc->sc_dev, "%s: iwm_phy_db_set_section failed\n", __func__); } return FALSE; } static int iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) { struct iwm_notification_wait alive_wait; struct iwm_mvm_alive_data alive_data; const struct iwm_fw_sects *fw; enum iwm_ucode_type old_type = sc->cur_ucode; int error; static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE }; if ((error = iwm_read_firmware(sc, ucode_type)) != 0) { device_printf(sc->sc_dev, "iwm_read_firmware: failed %d\n", error); return error; } fw = &sc->sc_fw.fw_sects[ucode_type]; sc->cur_ucode = ucode_type; sc->ucode_loaded = FALSE; memset(&alive_data, 0, sizeof(alive_data)); iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait, alive_cmd, nitems(alive_cmd), iwm_alive_fn, &alive_data); error = iwm_start_fw(sc, fw); if (error) { device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error); sc->cur_ucode = old_type; iwm_remove_notification(sc->sc_notif_wait, &alive_wait); return error; } /* * Some things may run in the background now, but we * just wait for the ALIVE notification here. */ IWM_UNLOCK(sc); error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait, IWM_MVM_UCODE_ALIVE_TIMEOUT); IWM_LOCK(sc); if (error) { if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { uint32_t a = 0x5a5a5a5a, b = 0x5a5a5a5a; if (iwm_nic_lock(sc)) { a = iwm_read_prph(sc, IWM_SB_CPU_1_STATUS); b = iwm_read_prph(sc, IWM_SB_CPU_2_STATUS); iwm_nic_unlock(sc); } device_printf(sc->sc_dev, "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n", a, b); } sc->cur_ucode = old_type; return error; } if (!alive_data.valid) { device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n", __func__); sc->cur_ucode = old_type; return EIO; } iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr); /* * configure and operate fw paging mechanism. * driver configures the paging flow only once, CPU2 paging image * included in the IWM_UCODE_INIT image. */ if (fw->paging_mem_size) { error = iwm_save_fw_paging(sc, fw); if (error) { device_printf(sc->sc_dev, "%s: failed to save the FW paging image\n", __func__); return error; } error = iwm_send_paging_cmd(sc, fw); if (error) { device_printf(sc->sc_dev, "%s: failed to send the paging cmd\n", __func__); iwm_free_fw_paging(sc); return error; } } if (!error) sc->ucode_loaded = TRUE; return error; } /* * mvm misc bits */ /* * follows iwlwifi/fw.c */ static int iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm) { struct iwm_notification_wait calib_wait; static const uint16_t init_complete[] = { IWM_INIT_COMPLETE_NOTIF, IWM_CALIB_RES_NOTIF_PHY_DB }; int ret; /* do not operate with rfkill switch turned on */ if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) { device_printf(sc->sc_dev, "radio is disabled by hardware switch\n"); return EPERM; } iwm_init_notification_wait(sc->sc_notif_wait, &calib_wait, init_complete, nitems(init_complete), iwm_wait_phy_db_entry, sc->sc_phy_db); /* Will also start the device */ ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT); if (ret) { device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n", ret); goto error; } if (justnvm) { /* Read nvm */ ret = iwm_nvm_init(sc); if (ret) { device_printf(sc->sc_dev, "failed to read nvm\n"); goto error; } IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr); goto error; } ret = iwm_send_bt_init_conf(sc); if (ret) { device_printf(sc->sc_dev, "failed to send bt coex configuration: %d\n", ret); goto error; } /* Send TX valid antennas before triggering calibrations */ ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc)); if (ret) { device_printf(sc->sc_dev, "failed to send antennas before calibration: %d\n", ret); goto error; } /* * Send phy configurations command to init uCode * to start the 16.0 uCode init image internal calibrations. */ ret = iwm_send_phy_cfg_cmd(sc); if (ret) { device_printf(sc->sc_dev, "%s: Failed to run INIT calibrations: %d\n", __func__, ret); goto error; } /* * Nothing to do but wait for the init complete notification * from the firmware. */ IWM_UNLOCK(sc); ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait, IWM_MVM_UCODE_CALIB_TIMEOUT); IWM_LOCK(sc); goto out; error: iwm_remove_notification(sc->sc_notif_wait, &calib_wait); out: return ret; } /* * receive side */ /* (re)stock rx ring, called at init-time and at runtime */ static int iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx) { struct iwm_rx_ring *ring = &sc->rxq; struct iwm_rx_data *data = &ring->data[idx]; struct mbuf *m; bus_dmamap_t dmamap; bus_dma_segment_t seg; int nsegs, error; m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE); if (m == NULL) return ENOBUFS; m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m, &seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); m_freem(m); return error; } if (data->m != NULL) bus_dmamap_unload(ring->data_dmat, data->map); /* Swap ring->spare_map with data->map */ dmamap = data->map; data->map = ring->spare_map; ring->spare_map = dmamap; bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD); data->m = m; /* Update RX descriptor. */ KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned")); ring->desc[idx] = htole32(seg.ds_addr >> 8); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); return 0; } /* iwlwifi: mvm/rx.c */ /* * iwm_mvm_get_signal_strength - use new rx PHY INFO API * values are reported by the fw as positive values - need to negate * to obtain their dBM. Account for missing antennas by replacing 0 * values by -256dBm: practically 0 power and a non-feasible 8 bit value. */ static int iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info) { int energy_a, energy_b, energy_c, max_energy; uint32_t val; val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]); energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >> IWM_RX_INFO_ENERGY_ANT_A_POS; energy_a = energy_a ? -energy_a : -256; energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >> IWM_RX_INFO_ENERGY_ANT_B_POS; energy_b = energy_b ? -energy_b : -256; energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >> IWM_RX_INFO_ENERGY_ANT_C_POS; energy_c = energy_c ? -energy_c : -256; max_energy = MAX(energy_a, energy_b); max_energy = MAX(max_energy, energy_c); IWM_DPRINTF(sc, IWM_DEBUG_RECV, "energy In A %d B %d C %d , and max %d\n", energy_a, energy_b, energy_c, max_energy); return max_energy; } static void iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_rx_phy_info *phy_info = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n"); memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info)); } /* * Retrieve the average noise (in dBm) among receivers. */ static int iwm_get_noise(struct iwm_softc *sc, const struct iwm_mvm_statistics_rx_non_phy *stats) { int i, total, nbant, noise; total = nbant = noise = 0; for (i = 0; i < 3; i++) { noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n", __func__, i, noise); if (noise) { total += noise; nbant++; } } IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n", __func__, nbant, total); #if 0 /* There should be at least one antenna but check anyway. */ return (nbant == 0) ? -127 : (total / nbant) - 107; #else /* For now, just hard-code it to -96 to be safe */ return (-96); #endif } +static void +iwm_mvm_handle_rx_statistics(struct iwm_softc *sc, struct iwm_rx_packet *pkt) +{ + struct iwm_notif_statistics_v10 *stats = (void *)&pkt->data; + + memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats)); + sc->sc_noise = iwm_get_noise(sc, &stats->rx.general); +} + /* * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler * * Handles the actual data of the Rx packet from the fw */ static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset, boolean_t stolen) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct ieee80211_frame *wh; struct ieee80211_node *ni; struct ieee80211_rx_stats rxs; struct iwm_rx_phy_info *phy_info; struct iwm_rx_mpdu_res_start *rx_res; struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset); uint32_t len; uint32_t rx_pkt_status; int rssi; phy_info = &sc->sc_last_phy_info; rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data; wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res)); len = le16toh(rx_res->byte_count); rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len)); if (__predict_false(phy_info->cfg_phy_cnt > 20)) { device_printf(sc->sc_dev, "dsp size out of range [0,20]: %d\n", phy_info->cfg_phy_cnt); goto fail; } if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) || !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status); goto fail; } rssi = iwm_mvm_get_signal_strength(sc, phy_info); /* Map it to relative value */ rssi = rssi - sc->sc_noise; /* replenish ring for the buffer we're going to feed to the sharks */ if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) { device_printf(sc->sc_dev, "%s: unable to add more buffers\n", __func__); goto fail; } m->m_data = pkt->data + sizeof(*rx_res); m->m_pkthdr.len = m->m_len = len; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise); ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh); IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: phy_info: channel=%d, flags=0x%08x\n", __func__, le16toh(phy_info->channel), le16toh(phy_info->phy_flags)); /* * Populate an RX state struct with the provided information. */ bzero(&rxs, sizeof(rxs)); rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ; rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI; rxs.c_ieee = le16toh(phy_info->channel); if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) { rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ); } else { rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ); } /* rssi is in 1/2db units */ rxs.c_rssi = rssi * 2; rxs.c_nf = sc->sc_noise; if (ieee80211_add_rx_params(m, &rxs) == 0) { if (ni) ieee80211_free_node(ni); goto fail; } if (ieee80211_radiotap_active_vap(vap)) { struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap; tap->wr_flags = 0; if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE)) tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE; tap->wr_chan_freq = htole16(rxs.c_freq); /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */ tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags); tap->wr_dbm_antsignal = (int8_t)rssi; tap->wr_dbm_antnoise = (int8_t)sc->sc_noise; tap->wr_tsft = phy_info->system_timestamp; switch (phy_info->rate) { /* CCK rates. */ case 10: tap->wr_rate = 2; break; case 20: tap->wr_rate = 4; break; case 55: tap->wr_rate = 11; break; case 110: tap->wr_rate = 22; break; /* OFDM rates. */ case 0xd: tap->wr_rate = 12; break; case 0xf: tap->wr_rate = 18; break; case 0x5: tap->wr_rate = 24; break; case 0x7: tap->wr_rate = 36; break; case 0x9: tap->wr_rate = 48; break; case 0xb: tap->wr_rate = 72; break; case 0x1: tap->wr_rate = 96; break; case 0x3: tap->wr_rate = 108; break; /* Unknown rate: should not happen. */ default: tap->wr_rate = 0; } } IWM_UNLOCK(sc); if (ni != NULL) { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m); ieee80211_input_mimo(ni, m); ieee80211_free_node(ni); } else { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m); ieee80211_input_mimo_all(ic, m); } IWM_LOCK(sc); return TRUE; fail: counter_u64_add(ic->ic_ierrors, 1); return FALSE; } static int iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt, struct iwm_node *in) { struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data; struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs; struct ieee80211_node *ni = &in->in_ni; int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK; KASSERT(tx_resp->frame_count == 1, ("too many frames")); /* Update rate control statistics. */ IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n", __func__, (int) le16toh(tx_resp->status.status), (int) le16toh(tx_resp->status.sequence), tx_resp->frame_count, tx_resp->bt_kill_count, tx_resp->failure_rts, tx_resp->failure_frame, le32toh(tx_resp->initial_rate), (int) le16toh(tx_resp->wireless_media_time)); txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY | IEEE80211_RATECTL_STATUS_LONG_RETRY; txs->short_retries = tx_resp->failure_rts; txs->long_retries = tx_resp->failure_frame; if (status != IWM_TX_STATUS_SUCCESS && status != IWM_TX_STATUS_DIRECT_DONE) { switch (status) { case IWM_TX_STATUS_FAIL_SHORT_LIMIT: txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT; break; case IWM_TX_STATUS_FAIL_LONG_LIMIT: txs->status = IEEE80211_RATECTL_TX_FAIL_LONG; break; case IWM_TX_STATUS_FAIL_LIFE_EXPIRE: txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED; break; default: txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED; break; } } else { txs->status = IEEE80211_RATECTL_TX_SUCCESS; } ieee80211_ratectl_tx_complete(ni, txs); return (txs->status != IEEE80211_RATECTL_TX_SUCCESS); } static void iwm_mvm_rx_tx_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_cmd_header *cmd_hdr = &pkt->hdr; int idx = cmd_hdr->idx; int qid = cmd_hdr->qid; struct iwm_tx_ring *ring = &sc->txq[qid]; struct iwm_tx_data *txd = &ring->data[idx]; struct iwm_node *in = txd->in; struct mbuf *m = txd->m; int status; KASSERT(txd->done == 0, ("txd not done")); KASSERT(txd->in != NULL, ("txd without node")); KASSERT(txd->m != NULL, ("txd without mbuf")); sc->sc_tx_timer = 0; status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in); /* Unmap and free mbuf. */ bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, txd->map); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "free txd %p, in %p\n", txd, txd->in); txd->done = 1; txd->m = NULL; txd->in = NULL; ieee80211_tx_complete(&in->in_ni, m, status); if (--ring->queued < IWM_TX_RING_LOMARK) { sc->qfullmsk &= ~(1 << ring->qid); if (sc->qfullmsk == 0) { iwm_start(sc); } } } /* * transmit side */ /* * Process a "command done" firmware notification. This is where we wakeup * processes waiting for a synchronous command completion. * from if_iwn */ static void iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE]; struct iwm_tx_data *data; if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) { return; /* Not a command ack. */ } /* XXX wide commands? */ IWM_DPRINTF(sc, IWM_DEBUG_CMD, "cmd notification type 0x%x qid %d idx %d\n", pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx); data = &ring->data[pkt->hdr.idx]; /* If the command was mapped in an mbuf, free it. */ if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } wakeup(&ring->desc[pkt->hdr.idx]); if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) { device_printf(sc->sc_dev, "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n", __func__, pkt->hdr.idx, ring->queued, ring->cur); /* XXX call iwm_force_nmi() */ } KASSERT(ring->queued > 0, ("ring->queued is empty?")); ring->queued--; if (ring->queued == 0) iwm_pcie_clear_cmd_in_flight(sc); } #if 0 /* * necessary only for block ack mode */ void iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id, uint16_t len) { struct iwm_agn_scd_bc_tbl *scd_bc_tbl; uint16_t w_val; scd_bc_tbl = sc->sched_dma.vaddr; len += 8; /* magic numbers came naturally from paris */ len = roundup(len, 4) / 4; w_val = htole16(sta_id << 12 | len); /* Update TX scheduler. */ scd_bc_tbl[qid].tfd_offset[idx] = w_val; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); /* I really wonder what this is ?!? */ if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) { scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); } } #endif /* * Take an 802.11 (non-n) rate, find the relevant rate * table entry. return the index into in_ridx[]. * * The caller then uses that index back into in_ridx * to figure out the rate index programmed /into/ * the firmware for this given node. */ static int iwm_tx_rateidx_lookup(struct iwm_softc *sc, struct iwm_node *in, uint8_t rate) { int i; uint8_t r; for (i = 0; i < nitems(in->in_ridx); i++) { r = iwm_rates[in->in_ridx[i]].rate; if (rate == r) return (i); } IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: couldn't find an entry for rate=%d\n", __func__, rate); /* XXX Return the first */ /* XXX TODO: have it return the /lowest/ */ return (0); } static int iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate) { int i; for (i = 0; i < nitems(iwm_rates); i++) { if (iwm_rates[i].rate == rate) return (i); } /* XXX error? */ IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: couldn't find an entry for rate=%d\n", __func__, rate); return (0); } /* * Fill in the rate related information for a transmit command. */ static const struct iwm_rate * iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in, struct mbuf *m, struct iwm_tx_cmd *tx) { struct ieee80211_node *ni = &in->in_ni; struct ieee80211_frame *wh; const struct ieee80211_txparam *tp = ni->ni_txparms; const struct iwm_rate *rinfo; int type; int ridx, rate_flags; wh = mtod(m, struct ieee80211_frame *); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT; tx->data_retry_limit = IWM_DEFAULT_TX_RETRY; if (type == IEEE80211_FC0_TYPE_MGT || type == IEEE80211_FC0_TYPE_CTL || (m->m_flags & M_EAPOL) != 0) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: MGT (%d)\n", __func__, tp->mgmtrate); } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: MCAST (%d)\n", __func__, tp->mcastrate); } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate); } else { int i; /* for data frames, use RS table */ IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__); /* XXX pass pktlen */ (void) ieee80211_ratectl_rate(ni, NULL, 0); i = iwm_tx_rateidx_lookup(sc, in, ni->ni_txrate); ridx = in->in_ridx[i]; /* This is the index into the programmed table */ tx->initial_rate_index = i; tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE); IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: start with i=%d, txrate %d\n", __func__, i, iwm_rates[ridx].rate); } IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: frame type=%d txrate %d\n", __func__, type, iwm_rates[ridx].rate); rinfo = &iwm_rates[ridx]; IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n", __func__, ridx, rinfo->rate, !! (IWM_RIDX_IS_CCK(ridx)) ); /* XXX TODO: hard-coded TX antenna? */ rate_flags = 1 << IWM_RATE_MCS_ANT_POS; if (IWM_RIDX_IS_CCK(ridx)) rate_flags |= IWM_RATE_MCS_CCK_MSK; tx->rate_n_flags = htole32(rate_flags | rinfo->plcp); return rinfo; } #define TB0_SIZE 16 static int iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_node *in = IWM_NODE(ni); struct iwm_tx_ring *ring; struct iwm_tx_data *data; struct iwm_tfd *desc; struct iwm_device_cmd *cmd; struct iwm_tx_cmd *tx; struct ieee80211_frame *wh; struct ieee80211_key *k = NULL; struct mbuf *m1; const struct iwm_rate *rinfo; uint32_t flags; u_int hdrlen; bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER]; int nsegs; uint8_t tid, type; int i, totlen, error, pad; wh = mtod(m, struct ieee80211_frame *); hdrlen = ieee80211_anyhdrsize(wh); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; tid = 0; ring = &sc->txq[ac]; desc = &ring->desc[ring->cur]; memset(desc, 0, sizeof(*desc)); data = &ring->data[ring->cur]; /* Fill out iwm_tx_cmd to send to the firmware */ cmd = &ring->cmd[ring->cur]; cmd->hdr.code = IWM_TX_CMD; cmd->hdr.flags = 0; cmd->hdr.qid = ring->qid; cmd->hdr.idx = ring->cur; tx = (void *)cmd->data; memset(tx, 0, sizeof(*tx)); rinfo = iwm_tx_fill_cmd(sc, in, m, tx); /* Encrypt the frame if need be. */ if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { /* Retrieve key for TX && do software encryption. */ k = ieee80211_crypto_encap(ni, m); if (k == NULL) { m_freem(m); return (ENOBUFS); } /* 802.11 header may have moved. */ wh = mtod(m, struct ieee80211_frame *); } if (ieee80211_radiotap_active_vap(vap)) { struct iwm_tx_radiotap_header *tap = &sc->sc_txtap; tap->wt_flags = 0; tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq); tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags); tap->wt_rate = rinfo->rate; if (k != NULL) tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP; ieee80211_radiotap_tx(vap, m); } totlen = m->m_pkthdr.len; flags = 0; if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= IWM_TX_CMD_FLG_ACK; } if (type == IEEE80211_FC0_TYPE_DATA && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold) && !IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= IWM_TX_CMD_FLG_PROT_REQUIRE; } if (IEEE80211_IS_MULTICAST(wh->i_addr1) || type != IEEE80211_FC0_TYPE_DATA) tx->sta_id = sc->sc_aux_sta.sta_id; else tx->sta_id = IWM_STATION_ID; if (type == IEEE80211_FC0_TYPE_MGT) { uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ || subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC); } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); } else { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT); } } else { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); } if (hdrlen & 3) { /* First segment length must be a multiple of 4. */ flags |= IWM_TX_CMD_FLG_MH_PAD; pad = 4 - (hdrlen & 3); } else pad = 0; tx->driver_txop = 0; tx->next_frame_len = 0; tx->len = htole16(totlen); tx->tid_tspec = tid; tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE); /* Set physical address of "scratch area". */ tx->dram_lsb_ptr = htole32(data->scratch_paddr); tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr); /* Copy 802.11 header in TX command. */ memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen); flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL; tx->sec_ctl = 0; tx->tx_flags |= htole32(flags); /* Trim 802.11 header. */ m_adj(m, hdrlen); error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { if (error != EFBIG) { device_printf(sc->sc_dev, "can't map mbuf (error %d)\n", error); m_freem(m); return error; } /* Too many DMA segments, linearize mbuf. */ m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2); if (m1 == NULL) { device_printf(sc->sc_dev, "%s: could not defrag mbuf\n", __func__); m_freem(m); return (ENOBUFS); } m = m1; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "can't map mbuf (error %d)\n", error); m_freem(m); return error; } } data->m = m; data->in = in; data->done = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "sending txd %p, in %p\n", data, data->in); KASSERT(data->in != NULL, ("node is NULL")); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n", ring->qid, ring->cur, totlen, nsegs, le32toh(tx->tx_flags), le32toh(tx->rate_n_flags), tx->initial_rate_index ); /* Fill TX descriptor. */ desc->num_tbs = 2 + nsegs; desc->tbs[0].lo = htole32(data->cmd_paddr); desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) | (TB0_SIZE << 4); desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE); desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) | ((sizeof(struct iwm_cmd_header) + sizeof(*tx) + hdrlen + pad - TB0_SIZE) << 4); /* Other DMA segments are for data payload. */ for (i = 0; i < nsegs; i++) { seg = &segs[i]; desc->tbs[i+2].lo = htole32(seg->ds_addr); desc->tbs[i+2].hi_n_len = \ htole16(iwm_get_dma_hi_addr(seg->ds_addr)) | ((seg->ds_len) << 4); } bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); #if 0 iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len)); #endif /* Kick TX ring. */ ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT; IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); /* Mark TX ring as full if we reach a certain threshold. */ if (++ring->queued > IWM_TX_RING_HIMARK) { sc->qfullmsk |= 1 << ring->qid; } return 0; } static int iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_bpf_params *params) { struct ieee80211com *ic = ni->ni_ic; struct iwm_softc *sc = ic->ic_softc; int error = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "->%s begin\n", __func__); if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) { m_freem(m); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "<-%s not RUNNING\n", __func__); return (ENETDOWN); } IWM_LOCK(sc); /* XXX fix this */ if (params == NULL) { error = iwm_tx(sc, m, ni, 0); } else { error = iwm_tx(sc, m, ni, 0); } sc->sc_tx_timer = 5; IWM_UNLOCK(sc); return (error); } /* * mvm/tx.c */ /* * Note that there are transports that buffer frames before they reach * the firmware. This means that after flush_tx_path is called, the * queue might not be empty. The race-free way to handle this is to: * 1) set the station as draining * 2) flush the Tx path * 3) wait for the transport queues to be empty */ int iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags) { int ret; struct iwm_tx_path_flush_cmd flush_cmd = { .queues_ctl = htole32(tfd_msk), .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH), }; ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags, sizeof(flush_cmd), &flush_cmd); if (ret) device_printf(sc->sc_dev, "Flushing tx queue failed: %d\n", ret); return ret; } /* * BEGIN mvm/quota.c */ static int iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_vap *ivp) { struct iwm_time_quota_cmd cmd; int i, idx, ret, num_active_macs, quota, quota_rem; int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, }; int n_ifs[IWM_MAX_BINDINGS] = {0, }; uint16_t id; memset(&cmd, 0, sizeof(cmd)); /* currently, PHY ID == binding ID */ if (ivp) { id = ivp->phy_ctxt->id; KASSERT(id < IWM_MAX_BINDINGS, ("invalid id")); colors[id] = ivp->phy_ctxt->color; if (1) n_ifs[id] = 1; } /* * The FW's scheduling session consists of * IWM_MVM_MAX_QUOTA fragments. Divide these fragments * equally between all the bindings that require quota */ num_active_macs = 0; for (i = 0; i < IWM_MAX_BINDINGS; i++) { cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID); num_active_macs += n_ifs[i]; } quota = 0; quota_rem = 0; if (num_active_macs) { quota = IWM_MVM_MAX_QUOTA / num_active_macs; quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs; } for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) { if (colors[i] < 0) continue; cmd.quotas[idx].id_and_color = htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i])); if (n_ifs[i] <= 0) { cmd.quotas[idx].quota = htole32(0); cmd.quotas[idx].max_duration = htole32(0); } else { cmd.quotas[idx].quota = htole32(quota * n_ifs[i]); cmd.quotas[idx].max_duration = htole32(0); } idx++; } /* Give the remainder of the session to the first binding */ cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem); ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC, sizeof(cmd), &cmd); if (ret) device_printf(sc->sc_dev, "%s: Failed to send quota: %d\n", __func__, ret); return ret; } /* * END mvm/quota.c */ /* * ieee80211 routines */ /* * Change to AUTH state in 80211 state machine. Roughly matches what * Linux does in bss_info_changed(). */ static int iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc) { struct ieee80211_node *ni; struct iwm_node *in; struct iwm_vap *iv = IWM_VAP(vap); uint32_t duration; int error; /* * XXX i have a feeling that the vap node is being * freed from underneath us. Grr. */ ni = ieee80211_ref_node(vap->iv_bss); in = IWM_NODE(ni); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE, "%s: called; vap=%p, bss ni=%p\n", __func__, vap, ni); in->in_assoc = 0; /* * Firmware bug - it'll crash if the beacon interval is less * than 16. We can't avoid connecting at all, so refuse the * station state change, this will cause net80211 to abandon * attempts to connect to this AP, and eventually wpa_s will * blacklist the AP... */ if (ni->ni_intval < 16) { device_printf(sc->sc_dev, "AP %s beacon interval is %d, refusing due to firmware bug!\n", ether_sprintf(ni->ni_bssid), ni->ni_intval); error = EINVAL; goto out; } error = iwm_allow_mcast(vap, sc); if (error) { device_printf(sc->sc_dev, "%s: failed to set multicast\n", __func__); goto out; } /* * This is where it deviates from what Linux does. * * Linux iwlwifi doesn't reset the nic each time, nor does it * call ctxt_add() here. Instead, it adds it during vap creation, * and always does a mac_ctx_changed(). * * The openbsd port doesn't attempt to do that - it reset things * at odd states and does the add here. * * So, until the state handling is fixed (ie, we never reset * the NIC except for a firmware failure, which should drag * the NIC back to IDLE, re-setup and re-add all the mac/phy * contexts that are required), let's do a dirty hack here. */ if (iv->is_uploaded) { if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC\n", __func__); goto out; } } else { if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to add MAC\n", __func__); goto out; } } if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0], in->in_ni.ni_chan, 1, 1)) != 0) { device_printf(sc->sc_dev, "%s: failed update phy ctxt\n", __func__); goto out; } iv->phy_ctxt = &sc->sc_phyctxt[0]; if ((error = iwm_mvm_binding_add_vif(sc, iv)) != 0) { device_printf(sc->sc_dev, "%s: binding update cmd\n", __func__); goto out; } /* * Authentication becomes unreliable when powersaving is left enabled * here. Powersaving will be activated again when association has * finished or is aborted. */ iv->ps_disabled = TRUE; error = iwm_mvm_power_update_mac(sc); iv->ps_disabled = FALSE; if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update power management\n", __func__); goto out; } if ((error = iwm_mvm_add_sta(sc, in)) != 0) { device_printf(sc->sc_dev, "%s: failed to add sta\n", __func__); goto out; } /* * Prevent the FW from wandering off channel during association * by "protecting" the session with a time event. */ /* XXX duration is in units of TU, not MS */ duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS; iwm_mvm_protect_session(sc, iv, duration, 500 /* XXX magic number */); DELAY(100); error = 0; out: ieee80211_free_node(ni); return (error); } static int iwm_release(struct iwm_softc *sc, struct iwm_node *in) { uint32_t tfd_msk; /* * Ok, so *technically* the proper set of calls for going * from RUN back to SCAN is: * * iwm_mvm_power_mac_disable(sc, in); * iwm_mvm_mac_ctxt_changed(sc, vap); * iwm_mvm_rm_sta(sc, in); * iwm_mvm_update_quotas(sc, NULL); * iwm_mvm_mac_ctxt_changed(sc, in); * iwm_mvm_binding_remove_vif(sc, IWM_VAP(in->in_ni.ni_vap)); * iwm_mvm_mac_ctxt_remove(sc, in); * * However, that freezes the device not matter which permutations * and modifications are attempted. Obviously, this driver is missing * something since it works in the Linux driver, but figuring out what * is missing is a little more complicated. Now, since we're going * back to nothing anyway, we'll just do a complete device reset. * Up your's, device! */ /* * Just using 0xf for the queues mask is fine as long as we only * get here from RUN state. */ tfd_msk = 0xf; - mbufq_drain(&sc->sc_snd); + iwm_xmit_queue_drain(sc); iwm_mvm_flush_tx_path(sc, tfd_msk, IWM_CMD_SYNC); /* * We seem to get away with just synchronously sending the * IWM_TXPATH_FLUSH command. */ // iwm_trans_wait_tx_queue_empty(sc, tfd_msk); iwm_stop_device(sc); iwm_init_hw(sc); if (in) in->in_assoc = 0; return 0; #if 0 int error; iwm_mvm_power_mac_disable(sc, in); if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "mac ctxt change fail 1 %d\n", error); return error; } if ((error = iwm_mvm_rm_sta(sc, in)) != 0) { device_printf(sc->sc_dev, "sta remove fail %d\n", error); return error; } error = iwm_mvm_rm_sta(sc, in); in->in_assoc = 0; iwm_mvm_update_quotas(sc, NULL); if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "mac ctxt change fail 2 %d\n", error); return error; } iwm_mvm_binding_remove_vif(sc, IWM_VAP(in->in_ni.ni_vap)); iwm_mvm_mac_ctxt_remove(sc, in); return error; #endif } static struct ieee80211_node * iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN]) { return malloc(sizeof (struct iwm_node), M_80211_NODE, M_NOWAIT | M_ZERO); } uint8_t iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx) { int i; uint8_t rval; for (i = 0; i < rs->rs_nrates; i++) { rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL); if (rval == iwm_rates[ridx].rate) return rs->rs_rates[i]; } return 0; } static void iwm_setrates(struct iwm_softc *sc, struct iwm_node *in) { struct ieee80211_node *ni = &in->in_ni; struct iwm_lq_cmd *lq = &in->in_lq; int nrates = ni->ni_rates.rs_nrates; int i, ridx, tab = 0; // int txant = 0; if (nrates > nitems(lq->rs_table)) { device_printf(sc->sc_dev, "%s: node supports %d rates, driver handles " "only %zu\n", __func__, nrates, nitems(lq->rs_table)); return; } if (nrates == 0) { device_printf(sc->sc_dev, "%s: node supports 0 rates, odd!\n", __func__); return; } /* * XXX .. and most of iwm_node is not initialised explicitly; * it's all just 0x0 passed to the firmware. */ /* first figure out which rates we should support */ /* XXX TODO: this isn't 11n aware /at all/ */ memset(&in->in_ridx, -1, sizeof(in->in_ridx)); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: nrates=%d\n", __func__, nrates); /* * Loop over nrates and populate in_ridx from the highest * rate to the lowest rate. Remember, in_ridx[] has * IEEE80211_RATE_MAXSIZE entries! */ for (i = 0; i < min(nrates, IEEE80211_RATE_MAXSIZE); i++) { int rate = ni->ni_rates.rs_rates[(nrates - 1) - i] & IEEE80211_RATE_VAL; /* Map 802.11 rate to HW rate index. */ for (ridx = 0; ridx <= IWM_RIDX_MAX; ridx++) if (iwm_rates[ridx].rate == rate) break; if (ridx > IWM_RIDX_MAX) { device_printf(sc->sc_dev, "%s: WARNING: device rate for %d not found!\n", __func__, rate); } else { IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: rate: i: %d, rate=%d, ridx=%d\n", __func__, i, rate, ridx); in->in_ridx[i] = ridx; } } /* then construct a lq_cmd based on those */ memset(lq, 0, sizeof(*lq)); lq->sta_id = IWM_STATION_ID; /* For HT, always enable RTS/CTS to avoid excessive retries. */ if (ni->ni_flags & IEEE80211_NODE_HT) lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK; /* * are these used? (we don't do SISO or MIMO) * need to set them to non-zero, though, or we get an error. */ lq->single_stream_ant_msk = 1; lq->dual_stream_ant_msk = 1; /* * Build the actual rate selection table. * The lowest bits are the rates. Additionally, * CCK needs bit 9 to be set. The rest of the bits * we add to the table select the tx antenna * Note that we add the rates in the highest rate first * (opposite of ni_rates). */ /* * XXX TODO: this should be looping over the min of nrates * and LQ_MAX_RETRY_NUM. Sigh. */ for (i = 0; i < nrates; i++) { int nextant; #if 0 if (txant == 0) txant = iwm_mvm_get_valid_tx_ant(sc); nextant = 1<<(ffs(txant)-1); txant &= ~nextant; #else nextant = iwm_mvm_get_valid_tx_ant(sc); #endif /* * Map the rate id into a rate index into * our hardware table containing the * configuration to use for this rate. */ ridx = in->in_ridx[i]; tab = iwm_rates[ridx].plcp; tab |= nextant << IWM_RATE_MCS_ANT_POS; if (IWM_RIDX_IS_CCK(ridx)) tab |= IWM_RATE_MCS_CCK_MSK; IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "station rate i=%d, rate=%d, hw=%x\n", i, iwm_rates[ridx].rate, tab); lq->rs_table[i] = htole32(tab); } /* then fill the rest with the lowest possible rate */ for (i = nrates; i < nitems(lq->rs_table); i++) { KASSERT(tab != 0, ("invalid tab")); lq->rs_table[i] = htole32(tab); } } static int iwm_media_change(struct ifnet *ifp) { struct ieee80211vap *vap = ifp->if_softc; struct ieee80211com *ic = vap->iv_ic; struct iwm_softc *sc = ic->ic_softc; int error; error = ieee80211_media_change(ifp); if (error != ENETRESET) return error; IWM_LOCK(sc); if (ic->ic_nrunning > 0) { iwm_stop(sc); iwm_init(sc); } IWM_UNLOCK(sc); return error; } static int iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct iwm_vap *ivp = IWM_VAP(vap); struct ieee80211com *ic = vap->iv_ic; struct iwm_softc *sc = ic->ic_softc; struct iwm_node *in; int error; IWM_DPRINTF(sc, IWM_DEBUG_STATE, "switching state %s -> %s\n", ieee80211_state_name[vap->iv_state], ieee80211_state_name[nstate]); IEEE80211_UNLOCK(ic); IWM_LOCK(sc); if (vap->iv_state == IEEE80211_S_SCAN && nstate != vap->iv_state) iwm_led_blink_stop(sc); /* disable beacon filtering if we're hopping out of RUN */ if (vap->iv_state == IEEE80211_S_RUN && nstate != vap->iv_state) { iwm_mvm_disable_beacon_filter(sc); if (((in = IWM_NODE(vap->iv_bss)) != NULL)) in->in_assoc = 0; if (nstate == IEEE80211_S_INIT) { IWM_UNLOCK(sc); IEEE80211_LOCK(ic); error = ivp->iv_newstate(vap, nstate, arg); IEEE80211_UNLOCK(ic); IWM_LOCK(sc); iwm_release(sc, NULL); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return error; } /* * It's impossible to directly go RUN->SCAN. If we iwm_release() * above then the card will be completely reinitialized, * so the driver must do everything necessary to bring the card * from INIT to SCAN. * * Additionally, upon receiving deauth frame from AP, * OpenBSD 802.11 stack puts the driver in IEEE80211_S_AUTH * state. This will also fail with this driver, so bring the FSM * from IEEE80211_S_RUN to IEEE80211_S_SCAN in this case as well. * * XXX TODO: fix this for FreeBSD! */ if (nstate == IEEE80211_S_SCAN || nstate == IEEE80211_S_AUTH || nstate == IEEE80211_S_ASSOC) { IWM_DPRINTF(sc, IWM_DEBUG_STATE, "Force transition to INIT; MGT=%d\n", arg); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); /* Always pass arg as -1 since we can't Tx right now. */ /* * XXX arg is just ignored anyway when transitioning * to IEEE80211_S_INIT. */ vap->iv_newstate(vap, IEEE80211_S_INIT, -1); IWM_DPRINTF(sc, IWM_DEBUG_STATE, "Going INIT->SCAN\n"); nstate = IEEE80211_S_SCAN; IEEE80211_UNLOCK(ic); IWM_LOCK(sc); } } switch (nstate) { case IEEE80211_S_INIT: case IEEE80211_S_SCAN: if (vap->iv_state == IEEE80211_S_AUTH || vap->iv_state == IEEE80211_S_ASSOC) { int myerr; IWM_UNLOCK(sc); IEEE80211_LOCK(ic); myerr = ivp->iv_newstate(vap, nstate, arg); IEEE80211_UNLOCK(ic); IWM_LOCK(sc); error = iwm_mvm_rm_sta(sc, vap, FALSE); if (error) { device_printf(sc->sc_dev, "%s: Failed to remove station: %d\n", __func__, error); } error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error) { device_printf(sc->sc_dev, "%s: Failed to change mac context: %d\n", __func__, error); } error = iwm_mvm_binding_remove_vif(sc, ivp); if (error) { device_printf(sc->sc_dev, "%s: Failed to remove channel ctx: %d\n", __func__, error); } ivp->phy_ctxt = NULL; error = iwm_mvm_power_update_mac(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update power management\n", __func__); } IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return myerr; } break; case IEEE80211_S_AUTH: if ((error = iwm_auth(vap, sc)) != 0) { device_printf(sc->sc_dev, "%s: could not move to auth state: %d\n", __func__, error); } break; case IEEE80211_S_ASSOC: /* * EBS may be disabled due to previous failures reported by FW. * Reset EBS status here assuming environment has been changed. */ sc->last_ebs_successful = TRUE; break; case IEEE80211_S_RUN: - { - struct iwm_host_cmd cmd = { - .id = IWM_LQ_CMD, - .len = { sizeof(in->in_lq), }, - .flags = IWM_CMD_SYNC, - }; - in = IWM_NODE(vap->iv_bss); /* Update the association state, now we have it all */ /* (eg associd comes in at this point */ error = iwm_mvm_update_sta(sc, in); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update STA\n", __func__); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return error; } in->in_assoc = 1; error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC: %d\n", __func__, error); } iwm_mvm_sf_update(sc, vap, FALSE); iwm_mvm_enable_beacon_filter(sc, ivp); iwm_mvm_power_update_mac(sc); iwm_mvm_update_quotas(sc, ivp); iwm_setrates(sc, in); - cmd.data[0] = &in->in_lq; - if ((error = iwm_send_cmd(sc, &cmd)) != 0) { + if ((error = iwm_mvm_send_lq_cmd(sc, &in->in_lq, TRUE)) != 0) { device_printf(sc->sc_dev, - "%s: IWM_LQ_CMD failed\n", __func__); + "%s: IWM_LQ_CMD failed: %d\n", __func__, error); } iwm_mvm_led_enable(sc); break; - } default: break; } IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return (ivp->iv_newstate(vap, nstate, arg)); } void iwm_endscan_cb(void *arg, int pending) { struct iwm_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE, "%s: scan ended\n", __func__); ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps)); } static int iwm_send_bt_init_conf(struct iwm_softc *sc) { struct iwm_bt_coex_cmd bt_cmd; bt_cmd.mode = htole32(IWM_BT_COEX_WIFI); bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET); return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd), &bt_cmd); } static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *sc) { boolean_t nvm_lar = sc->nvm_data->lar_enabled; boolean_t tlv_lar = fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT); if (iwm_lar_disable) return FALSE; /* * Enable LAR only if it is supported by the FW (TLV) && * enabled in the NVM */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) return nvm_lar && tlv_lar; else return tlv_lar; } static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *sc) { return fw_has_api(&sc->ucode_capa, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) || fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC); } static int iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2) { struct iwm_mcc_update_cmd mcc_cmd; struct iwm_host_cmd hcmd = { .id = IWM_MCC_UPDATE_CMD, .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB), .data = { &mcc_cmd }, }; int ret; #ifdef IWM_DEBUG struct iwm_rx_packet *pkt; struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL; struct iwm_mcc_update_resp *mcc_resp; int n_channels; uint16_t mcc; #endif int resp_v2 = fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2); if (!iwm_mvm_is_lar_supported(sc)) { IWM_DPRINTF(sc, IWM_DEBUG_LAR, "%s: no LAR support\n", __func__); return 0; } memset(&mcc_cmd, 0, sizeof(mcc_cmd)); mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]); if (iwm_mvm_is_wifi_mcc_supported(sc)) mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT; else mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW; if (resp_v2) hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd); else hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1); IWM_DPRINTF(sc, IWM_DEBUG_LAR, "send MCC update to FW with '%c%c' src = %d\n", alpha2[0], alpha2[1], mcc_cmd.source_id); ret = iwm_send_cmd(sc, &hcmd); if (ret) return ret; #ifdef IWM_DEBUG pkt = hcmd.resp_pkt; /* Extract MCC response */ if (resp_v2) { mcc_resp = (void *)pkt->data; mcc = mcc_resp->mcc; n_channels = le32toh(mcc_resp->n_channels); } else { mcc_resp_v1 = (void *)pkt->data; mcc = mcc_resp_v1->mcc; n_channels = le32toh(mcc_resp_v1->n_channels); } /* W/A for a FW/NVM issue - returns 0x00 for the world domain */ if (mcc == 0) mcc = 0x3030; /* "00" - world */ IWM_DPRINTF(sc, IWM_DEBUG_LAR, "regulatory domain '%c%c' (%d channels available)\n", mcc >> 8, mcc & 0xff, n_channels); #endif iwm_free_resp(sc, &hcmd); return 0; } static void iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff) { struct iwm_host_cmd cmd = { .id = IWM_REPLY_THERMAL_MNG_BACKOFF, .len = { sizeof(uint32_t), }, .data = { &backoff, }, }; if (iwm_send_cmd(sc, &cmd) != 0) { device_printf(sc->sc_dev, "failed to change thermal tx backoff\n"); } } static int iwm_init_hw(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; int error, i, ac; sc->sf_state = IWM_SF_UNINIT; if ((error = iwm_start_hw(sc)) != 0) { printf("iwm_start_hw: failed %d\n", error); return error; } if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) { printf("iwm_run_init_mvm_ucode: failed %d\n", error); return error; } /* * should stop and start HW since that INIT * image just loaded */ iwm_stop_device(sc); sc->sc_ps_disabled = FALSE; if ((error = iwm_start_hw(sc)) != 0) { device_printf(sc->sc_dev, "could not initialize hardware\n"); return error; } /* omstart, this time with the regular firmware */ error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR); if (error) { device_printf(sc->sc_dev, "could not load firmware\n"); goto error; } error = iwm_mvm_sf_update(sc, NULL, FALSE); if (error) device_printf(sc->sc_dev, "Failed to initialize Smart Fifo\n"); if ((error = iwm_send_bt_init_conf(sc)) != 0) { device_printf(sc->sc_dev, "bt init conf failed\n"); goto error; } error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc)); if (error != 0) { device_printf(sc->sc_dev, "antenna config failed\n"); goto error; } /* Send phy db control command and then phy db calibration */ if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0) goto error; if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) { device_printf(sc->sc_dev, "phy_cfg_cmd failed\n"); goto error; } /* Add auxiliary station for scanning */ if ((error = iwm_mvm_add_aux_sta(sc)) != 0) { device_printf(sc->sc_dev, "add_aux_sta failed\n"); goto error; } for (i = 0; i < IWM_NUM_PHY_CTX; i++) { /* * The channel used here isn't relevant as it's * going to be overwritten in the other flows. * For now use the first channel we have. */ if ((error = iwm_mvm_phy_ctxt_add(sc, &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0) goto error; } /* Initialize tx backoffs to the minimum. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) iwm_mvm_tt_tx_backoff(sc, 0); error = iwm_mvm_power_update_device(sc); if (error) goto error; if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0) goto error; if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) { if ((error = iwm_mvm_config_umac_scan(sc)) != 0) goto error; } /* Enable Tx queues. */ for (ac = 0; ac < WME_NUM_AC; ac++) { error = iwm_enable_txq(sc, IWM_STATION_ID, ac, iwm_mvm_ac_to_tx_fifo[ac]); if (error) goto error; } if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) { device_printf(sc->sc_dev, "failed to disable beacon filter\n"); goto error; } return 0; error: iwm_stop_device(sc); return error; } /* Allow multicast from our BSSID. */ static int iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc) { struct ieee80211_node *ni = vap->iv_bss; struct iwm_mcast_filter_cmd *cmd; size_t size; int error; size = roundup(sizeof(*cmd), 4); cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); if (cmd == NULL) return ENOMEM; cmd->filter_own = 1; cmd->port_id = 0; cmd->count = 0; cmd->pass_all = 1; IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid); error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD, IWM_CMD_SYNC, size, cmd); free(cmd, M_DEVBUF); return (error); } /* * ifnet interfaces */ static void iwm_init(struct iwm_softc *sc) { int error; if (sc->sc_flags & IWM_FLAG_HW_INITED) { return; } sc->sc_generation++; sc->sc_flags &= ~IWM_FLAG_STOPPED; if ((error = iwm_init_hw(sc)) != 0) { printf("iwm_init_hw failed %d\n", error); iwm_stop(sc); return; } /* * Ok, firmware loaded and we are jogging */ sc->sc_flags |= IWM_FLAG_HW_INITED; callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); } static int iwm_transmit(struct ieee80211com *ic, struct mbuf *m) { struct iwm_softc *sc; int error; sc = ic->ic_softc; IWM_LOCK(sc); if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) { IWM_UNLOCK(sc); return (ENXIO); } error = mbufq_enqueue(&sc->sc_snd, m); if (error) { IWM_UNLOCK(sc); return (error); } iwm_start(sc); IWM_UNLOCK(sc); return (0); } /* * Dequeue packets from sendq and call send. */ static void iwm_start(struct iwm_softc *sc) { struct ieee80211_node *ni; struct mbuf *m; int ac = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__); while (sc->qfullmsk == 0 && (m = mbufq_dequeue(&sc->sc_snd)) != NULL) { ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; if (iwm_tx(sc, m, ni, ac) != 0) { if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); ieee80211_free_node(ni); continue; } sc->sc_tx_timer = 15; } IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__); } static void iwm_stop(struct iwm_softc *sc) { sc->sc_flags &= ~IWM_FLAG_HW_INITED; sc->sc_flags |= IWM_FLAG_STOPPED; sc->sc_generation++; iwm_led_blink_stop(sc); sc->sc_tx_timer = 0; iwm_stop_device(sc); sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; } static void iwm_watchdog(void *arg) { struct iwm_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; if (sc->sc_tx_timer > 0) { if (--sc->sc_tx_timer == 0) { device_printf(sc->sc_dev, "device timeout\n"); #ifdef IWM_DEBUG iwm_nic_error(sc); #endif ieee80211_restart_all(ic); counter_u64_add(sc->sc_ic.ic_oerrors, 1); return; } } callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); } static void iwm_parent(struct ieee80211com *ic) { struct iwm_softc *sc = ic->ic_softc; int startall = 0; IWM_LOCK(sc); if (ic->ic_nrunning > 0) { if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) { iwm_init(sc); startall = 1; } } else if (sc->sc_flags & IWM_FLAG_HW_INITED) iwm_stop(sc); IWM_UNLOCK(sc); if (startall) ieee80211_start_all(ic); } /* * The interrupt side of things */ /* * error dumping routines are from iwlwifi/mvm/utils.c */ /* * Note: This structure is read from the device with IO accesses, * and the reading already does the endian conversion. As it is * read with uint32_t-sized accesses, any members with a different size * need to be ordered correctly though! */ struct iwm_error_event_table { uint32_t valid; /* (nonzero) valid, (0) log is empty */ uint32_t error_id; /* type of error */ uint32_t trm_hw_status0; /* TRM HW status */ uint32_t trm_hw_status1; /* TRM HW status */ uint32_t blink2; /* branch link */ uint32_t ilink1; /* interrupt link */ uint32_t ilink2; /* interrupt link */ uint32_t data1; /* error-specific data */ uint32_t data2; /* error-specific data */ uint32_t data3; /* error-specific data */ uint32_t bcon_time; /* beacon timer */ uint32_t tsf_low; /* network timestamp function timer */ uint32_t tsf_hi; /* network timestamp function timer */ uint32_t gp1; /* GP1 timer register */ uint32_t gp2; /* GP2 timer register */ uint32_t fw_rev_type; /* firmware revision type */ uint32_t major; /* uCode version major */ uint32_t minor; /* uCode version minor */ uint32_t hw_ver; /* HW Silicon version */ uint32_t brd_ver; /* HW board version */ uint32_t log_pc; /* log program counter */ uint32_t frame_ptr; /* frame pointer */ uint32_t stack_ptr; /* stack pointer */ uint32_t hcmd; /* last host command header */ uint32_t isr0; /* isr status register LMPM_NIC_ISR0: * rxtx_flag */ uint32_t isr1; /* isr status register LMPM_NIC_ISR1: * host_flag */ uint32_t isr2; /* isr status register LMPM_NIC_ISR2: * enc_flag */ uint32_t isr3; /* isr status register LMPM_NIC_ISR3: * time_flag */ uint32_t isr4; /* isr status register LMPM_NIC_ISR4: * wico interrupt */ uint32_t last_cmd_id; /* last HCMD id handled by the firmware */ uint32_t wait_event; /* wait event() caller address */ uint32_t l2p_control; /* L2pControlField */ uint32_t l2p_duration; /* L2pDurationField */ uint32_t l2p_mhvalid; /* L2pMhValidBits */ uint32_t l2p_addr_match; /* L2pAddrMatchStat */ uint32_t lmpm_pmg_sel; /* indicate which clocks are turned on * (LMPM_PMG_SEL) */ uint32_t u_timestamp; /* indicate when the date and time of the * compilation */ uint32_t flow_handler; /* FH read/write pointers, RX credit */ } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */; /* * UMAC error struct - relevant starting from family 8000 chip. * Note: This structure is read from the device with IO accesses, * and the reading already does the endian conversion. As it is * read with u32-sized accesses, any members with a different size * need to be ordered correctly though! */ struct iwm_umac_error_event_table { uint32_t valid; /* (nonzero) valid, (0) log is empty */ uint32_t error_id; /* type of error */ uint32_t blink1; /* branch link */ uint32_t blink2; /* branch link */ uint32_t ilink1; /* interrupt link */ uint32_t ilink2; /* interrupt link */ uint32_t data1; /* error-specific data */ uint32_t data2; /* error-specific data */ uint32_t data3; /* error-specific data */ uint32_t umac_major; uint32_t umac_minor; uint32_t frame_pointer; /* core register 27*/ uint32_t stack_pointer; /* core register 28 */ uint32_t cmd_header; /* latest host cmd sent to UMAC */ uint32_t nic_isr_pref; /* ISR status register */ } __packed; #define ERROR_START_OFFSET (1 * sizeof(uint32_t)) #define ERROR_ELEM_SIZE (7 * sizeof(uint32_t)) #ifdef IWM_DEBUG struct { const char *name; uint8_t num; } advanced_lookup[] = { { "NMI_INTERRUPT_WDG", 0x34 }, { "SYSASSERT", 0x35 }, { "UCODE_VERSION_MISMATCH", 0x37 }, { "BAD_COMMAND", 0x38 }, { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C }, { "FATAL_ERROR", 0x3D }, { "NMI_TRM_HW_ERR", 0x46 }, { "NMI_INTERRUPT_TRM", 0x4C }, { "NMI_INTERRUPT_BREAK_POINT", 0x54 }, { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C }, { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 }, { "NMI_INTERRUPT_HOST", 0x66 }, { "NMI_INTERRUPT_ACTION_PT", 0x7C }, { "NMI_INTERRUPT_UNKNOWN", 0x84 }, { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 }, { "ADVANCED_SYSASSERT", 0 }, }; static const char * iwm_desc_lookup(uint32_t num) { int i; for (i = 0; i < nitems(advanced_lookup) - 1; i++) if (advanced_lookup[i].num == num) return advanced_lookup[i].name; /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */ return advanced_lookup[i].name; } static void iwm_nic_umac_error(struct iwm_softc *sc) { struct iwm_umac_error_event_table table; uint32_t base; base = sc->umac_error_event_table; if (base < 0x800000) { device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n", base); return; } if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { device_printf(sc->sc_dev, "reading errlog failed\n"); return; } if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n"); device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", sc->sc_flags, table.valid); } device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id, iwm_desc_lookup(table.error_id)); device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1); device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2); device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n", table.ilink1); device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n", table.ilink2); device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1); device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2); device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3); device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major); device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor); device_printf(sc->sc_dev, "0x%08X | frame pointer\n", table.frame_pointer); device_printf(sc->sc_dev, "0x%08X | stack pointer\n", table.stack_pointer); device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header); device_printf(sc->sc_dev, "0x%08X | isr status reg\n", table.nic_isr_pref); } /* * Support for dumping the error log seemed like a good idea ... * but it's mostly hex junk and the only sensible thing is the * hw/ucode revision (which we know anyway). Since it's here, * I'll just leave it in, just in case e.g. the Intel guys want to * help us decipher some "ADVANCED_SYSASSERT" later. */ static void iwm_nic_error(struct iwm_softc *sc) { struct iwm_error_event_table table; uint32_t base; device_printf(sc->sc_dev, "dumping device error log\n"); base = sc->error_event_table; if (base < 0x800000) { device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n", base); return; } if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { device_printf(sc->sc_dev, "reading errlog failed\n"); return; } if (!table.valid) { device_printf(sc->sc_dev, "errlog not found, skipping\n"); return; } if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { device_printf(sc->sc_dev, "Start Error Log Dump:\n"); device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", sc->sc_flags, table.valid); } device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id, iwm_desc_lookup(table.error_id)); device_printf(sc->sc_dev, "%08X | trm_hw_status0\n", table.trm_hw_status0); device_printf(sc->sc_dev, "%08X | trm_hw_status1\n", table.trm_hw_status1); device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2); device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1); device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2); device_printf(sc->sc_dev, "%08X | data1\n", table.data1); device_printf(sc->sc_dev, "%08X | data2\n", table.data2); device_printf(sc->sc_dev, "%08X | data3\n", table.data3); device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time); device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low); device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi); device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1); device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2); device_printf(sc->sc_dev, "%08X | uCode revision type\n", table.fw_rev_type); device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major); device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor); device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver); device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver); device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd); device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0); device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1); device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2); device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3); device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4); device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id); device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event); device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control); device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration); device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid); device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match); device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel); device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp); device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler); if (sc->umac_error_event_table) iwm_nic_umac_error(sc); } #endif static void iwm_handle_rxb(struct iwm_softc *sc, struct mbuf *m) { struct ieee80211com *ic = &sc->sc_ic; struct iwm_cmd_response *cresp; struct mbuf *m1; uint32_t offset = 0; uint32_t maxoff = IWM_RBUF_SIZE; uint32_t nextoff; boolean_t stolen = FALSE; #define HAVEROOM(a) \ ((a) + sizeof(uint32_t) + sizeof(struct iwm_cmd_header) < maxoff) while (HAVEROOM(offset)) { struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset); int qid, idx, code, len; qid = pkt->hdr.qid; idx = pkt->hdr.idx; code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code); /* * randomly get these from the firmware, no idea why. * they at least seem harmless, so just ignore them for now */ if ((pkt->hdr.code == 0 && (qid & ~0x80) == 0 && idx == 0) || pkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID)) { break; } IWM_DPRINTF(sc, IWM_DEBUG_INTR, "rx packet qid=%d idx=%d type=%x\n", qid & ~0x80, pkt->hdr.idx, code); len = iwm_rx_packet_len(pkt); len += sizeof(uint32_t); /* account for status word */ nextoff = offset + roundup2(len, IWM_FH_RSCSR_FRAME_ALIGN); iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt); switch (code) { case IWM_REPLY_RX_PHY_CMD: iwm_mvm_rx_rx_phy_cmd(sc, pkt); break; case IWM_REPLY_RX_MPDU_CMD: { /* * If this is the last frame in the RX buffer, we * can directly feed the mbuf to the sharks here. */ struct iwm_rx_packet *nextpkt = mtodoff(m, struct iwm_rx_packet *, nextoff); if (!HAVEROOM(nextoff) || (nextpkt->hdr.code == 0 && (nextpkt->hdr.qid & ~0x80) == 0 && nextpkt->hdr.idx == 0) || (nextpkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID))) { if (iwm_mvm_rx_rx_mpdu(sc, m, offset, stolen)) { stolen = FALSE; /* Make sure we abort the loop */ nextoff = maxoff; } break; } /* * Use m_copym instead of m_split, because that * makes it easier to keep a valid rx buffer in * the ring, when iwm_mvm_rx_rx_mpdu() fails. * * We need to start m_copym() at offset 0, to get the * M_PKTHDR flag preserved. */ m1 = m_copym(m, 0, M_COPYALL, M_NOWAIT); if (m1) { if (iwm_mvm_rx_rx_mpdu(sc, m1, offset, stolen)) stolen = TRUE; else m_freem(m1); } break; } case IWM_TX_CMD: iwm_mvm_rx_tx_cmd(sc, pkt); break; case IWM_MISSED_BEACONS_NOTIFICATION: { struct iwm_missed_beacons_notif *resp; int missed; /* XXX look at mac_id to determine interface ID */ struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); resp = (void *)pkt->data; missed = le32toh(resp->consec_missed_beacons); IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE, "%s: MISSED_BEACON: mac_id=%d, " "consec_since_last_rx=%d, consec=%d, num_expect=%d " "num_rx=%d\n", __func__, le32toh(resp->mac_id), le32toh(resp->consec_missed_beacons_since_last_rx), le32toh(resp->consec_missed_beacons), le32toh(resp->num_expected_beacons), le32toh(resp->num_recvd_beacons)); /* Be paranoid */ if (vap == NULL) break; /* XXX no net80211 locking? */ if (vap->iv_state == IEEE80211_S_RUN && (ic->ic_flags & IEEE80211_F_SCAN) == 0) { if (missed > vap->iv_bmissthreshold) { /* XXX bad locking; turn into task */ IWM_UNLOCK(sc); ieee80211_beacon_miss(ic); IWM_LOCK(sc); } } break; } case IWM_MFUART_LOAD_NOTIFICATION: break; case IWM_MVM_ALIVE: break; case IWM_CALIB_RES_NOTIF_PHY_DB: break; - case IWM_STATISTICS_NOTIFICATION: { - struct iwm_notif_statistics *stats; - stats = (void *)pkt->data; - memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats)); - sc->sc_noise = iwm_get_noise(sc, &stats->rx.general); + case IWM_STATISTICS_NOTIFICATION: + iwm_mvm_handle_rx_statistics(sc, pkt); break; - } case IWM_NVM_ACCESS_CMD: case IWM_MCC_UPDATE_CMD: if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) { memcpy(sc->sc_cmd_resp, pkt, sizeof(sc->sc_cmd_resp)); } break; case IWM_MCC_CHUB_UPDATE_CMD: { struct iwm_mcc_chub_notif *notif; notif = (void *)pkt->data; sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8; sc->sc_fw_mcc[1] = notif->mcc & 0xff; sc->sc_fw_mcc[2] = '\0'; IWM_DPRINTF(sc, IWM_DEBUG_LAR, "fw source %d sent CC '%s'\n", notif->source_id, sc->sc_fw_mcc); break; } case IWM_DTS_MEASUREMENT_NOTIFICATION: case IWM_WIDE_ID(IWM_PHY_OPS_GROUP, IWM_DTS_MEASUREMENT_NOTIF_WIDE): { struct iwm_dts_measurement_notif_v1 *notif; if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) { device_printf(sc->sc_dev, "Invalid DTS_MEASUREMENT_NOTIFICATION\n"); break; } notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_TEMP, "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n", notif->temp); break; } case IWM_PHY_CONFIGURATION_CMD: case IWM_TX_ANT_CONFIGURATION_CMD: case IWM_ADD_STA: case IWM_MAC_CONTEXT_CMD: case IWM_REPLY_SF_CFG_CMD: case IWM_POWER_TABLE_CMD: case IWM_PHY_CONTEXT_CMD: case IWM_BINDING_CONTEXT_CMD: case IWM_TIME_EVENT_CMD: case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD): case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC): case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_ABORT_UMAC): case IWM_SCAN_OFFLOAD_REQUEST_CMD: case IWM_SCAN_OFFLOAD_ABORT_CMD: case IWM_REPLY_BEACON_FILTERING_CMD: case IWM_MAC_PM_POWER_TABLE: case IWM_TIME_QUOTA_CMD: case IWM_REMOVE_STA: case IWM_TXPATH_FLUSH: case IWM_LQ_CMD: case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_FW_PAGING_BLOCK_CMD): case IWM_BT_CONFIG: case IWM_REPLY_THERMAL_MNG_BACKOFF: cresp = (void *)pkt->data; if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) { memcpy(sc->sc_cmd_resp, pkt, sizeof(*pkt)+sizeof(*cresp)); } break; /* ignore */ case IWM_PHY_DB_CMD: break; case IWM_INIT_COMPLETE_NOTIF: break; case IWM_SCAN_OFFLOAD_COMPLETE: iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; ieee80211_runtask(ic, &sc->sc_es_task); } break; case IWM_SCAN_ITERATION_COMPLETE: { struct iwm_lmac_scan_complete_notif *notif; notif = (void *)pkt->data; break; } case IWM_SCAN_COMPLETE_UMAC: iwm_mvm_rx_umac_scan_complete_notif(sc, pkt); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; ieee80211_runtask(ic, &sc->sc_es_task); } break; case IWM_SCAN_ITERATION_COMPLETE_UMAC: { struct iwm_umac_scan_iter_complete_notif *notif; notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration " "complete, status=0x%x, %d channels scanned\n", notif->status, notif->scanned_channels); break; } case IWM_REPLY_ERROR: { struct iwm_error_resp *resp; resp = (void *)pkt->data; device_printf(sc->sc_dev, "firmware error 0x%x, cmd 0x%x\n", le32toh(resp->error_type), resp->cmd_id); break; } case IWM_TIME_EVENT_NOTIFICATION: { struct iwm_time_event_notif *notif; notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_INTR, "TE notif status = 0x%x action = 0x%x\n", notif->status, notif->action); break; } /* * Firmware versions 21 and 22 generate some DEBUG_LOG_MSG * messages. Just ignore them for now. */ case IWM_DEBUG_LOG_MSG: break; case IWM_MCAST_FILTER_CMD: break; case IWM_SCD_QUEUE_CFG: { struct iwm_scd_txq_cfg_rsp *rsp; rsp = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_CMD, "queue cfg token=0x%x sta_id=%d " "tid=%d scd_queue=%d\n", rsp->token, rsp->sta_id, rsp->tid, rsp->scd_queue); break; } default: device_printf(sc->sc_dev, "frame %d/%d %x UNHANDLED (this should " "not happen)\n", qid & ~0x80, idx, pkt->len_n_flags); break; } /* * Why test bit 0x80? The Linux driver: * * There is one exception: uCode sets bit 15 when it * originates the response/notification, i.e. when the * response/notification is not a direct response to a * command sent by the driver. For example, uCode issues * IWM_REPLY_RX when it sends a received frame to the driver; * it is not a direct response to any driver command. * * Ok, so since when is 7 == 15? Well, the Linux driver * uses a slightly different format for pkt->hdr, and "qid" * is actually the upper byte of a two-byte field. */ if (!(qid & (1 << 7))) iwm_cmd_done(sc, pkt); offset = nextoff; } if (stolen) m_freem(m); #undef HAVEROOM } /* * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt. * Basic structure from if_iwn */ static void iwm_notif_intr(struct iwm_softc *sc) { uint16_t hw; bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map, BUS_DMASYNC_POSTREAD); hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff; /* * Process responses */ while (sc->rxq.cur != hw) { struct iwm_rx_ring *ring = &sc->rxq; struct iwm_rx_data *data = &ring->data[ring->cur]; bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); IWM_DPRINTF(sc, IWM_DEBUG_INTR, "%s: hw = %d cur = %d\n", __func__, hw, ring->cur); iwm_handle_rxb(sc, data->m); ring->cur = (ring->cur + 1) % IWM_RX_RING_COUNT; } /* * Tell the firmware that it can reuse the ring entries that * we have just processed. * Seems like the hardware gets upset unless we align * the write by 8?? */ hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1; IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, rounddown2(hw, 8)); } static void iwm_intr(void *arg) { struct iwm_softc *sc = arg; int handled = 0; int r1, r2, rv = 0; int isperiodic = 0; IWM_LOCK(sc); IWM_WRITE(sc, IWM_CSR_INT_MASK, 0); if (sc->sc_flags & IWM_FLAG_USE_ICT) { uint32_t *ict = sc->ict_dma.vaddr; int tmp; tmp = htole32(ict[sc->ict_cur]); if (!tmp) goto out_ena; /* * ok, there was something. keep plowing until we have all. */ r1 = r2 = 0; while (tmp) { r1 |= tmp; ict[sc->ict_cur] = 0; sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT; tmp = htole32(ict[sc->ict_cur]); } /* this is where the fun begins. don't ask */ if (r1 == 0xffffffff) r1 = 0; /* i am not expected to understand this */ if (r1 & 0xc0000) r1 |= 0x8000; r1 = (0xff & r1) | ((0xff00 & r1) << 16); } else { r1 = IWM_READ(sc, IWM_CSR_INT); /* "hardware gone" (where, fishing?) */ if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0) goto out; r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS); } if (r1 == 0 && r2 == 0) { goto out_ena; } IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask); /* Safely ignore these bits for debug checks below */ r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD); if (r1 & IWM_CSR_INT_BIT_SW_ERR) { int i; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); #ifdef IWM_DEBUG iwm_nic_error(sc); #endif /* Dump driver status (TX and RX rings) while we're here. */ device_printf(sc->sc_dev, "driver status:\n"); for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) { struct iwm_tx_ring *ring = &sc->txq[i]; device_printf(sc->sc_dev, " tx ring %2d: qid=%-2d cur=%-3d " "queued=%-3d\n", i, ring->qid, ring->cur, ring->queued); } device_printf(sc->sc_dev, " rx ring: cur=%d\n", sc->rxq.cur); device_printf(sc->sc_dev, " 802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state); /* Don't stop the device; just do a VAP restart */ IWM_UNLOCK(sc); if (vap == NULL) { printf("%s: null vap\n", __func__); return; } device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; " "restarting\n", __func__, vap->iv_state); ieee80211_restart_all(ic); return; } if (r1 & IWM_CSR_INT_BIT_HW_ERR) { handled |= IWM_CSR_INT_BIT_HW_ERR; device_printf(sc->sc_dev, "hardware error, stopping device\n"); iwm_stop(sc); rv = 1; goto out; } /* firmware chunk loaded */ if (r1 & IWM_CSR_INT_BIT_FH_TX) { IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK); handled |= IWM_CSR_INT_BIT_FH_TX; sc->sc_fw_chunk_done = 1; wakeup(&sc->sc_fw); } if (r1 & IWM_CSR_INT_BIT_RF_KILL) { handled |= IWM_CSR_INT_BIT_RF_KILL; if (iwm_check_rfkill(sc)) { device_printf(sc->sc_dev, "%s: rfkill switch, disabling interface\n", __func__); iwm_stop(sc); } } /* * The Linux driver uses periodic interrupts to avoid races. * We cargo-cult like it's going out of fashion. */ if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) { handled |= IWM_CSR_INT_BIT_RX_PERIODIC; IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC); if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0) IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS); isperiodic = 1; } if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) { handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX); IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK); iwm_notif_intr(sc); /* enable periodic interrupt, see above */ if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic) IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_ENA); } if (__predict_false(r1 & ~handled)) IWM_DPRINTF(sc, IWM_DEBUG_INTR, "%s: unhandled interrupts: %x\n", __func__, r1); rv = 1; out_ena: iwm_restore_interrupts(sc); out: IWM_UNLOCK(sc); return; } /* * Autoconf glue-sniffing */ #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_WL_3160_1 0x08b3 #define PCI_PRODUCT_INTEL_WL_3160_2 0x08b4 #define PCI_PRODUCT_INTEL_WL_3165_1 0x3165 #define PCI_PRODUCT_INTEL_WL_3165_2 0x3166 #define PCI_PRODUCT_INTEL_WL_7260_1 0x08b1 #define PCI_PRODUCT_INTEL_WL_7260_2 0x08b2 #define PCI_PRODUCT_INTEL_WL_7265_1 0x095a #define PCI_PRODUCT_INTEL_WL_7265_2 0x095b #define PCI_PRODUCT_INTEL_WL_8260_1 0x24f3 #define PCI_PRODUCT_INTEL_WL_8260_2 0x24f4 static const struct iwm_devices { uint16_t device; const struct iwm_cfg *cfg; } iwm_devices[] = { { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg }, { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg }, { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg }, { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg }, { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg }, { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg }, { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg }, { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg }, { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg }, { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg }, }; static int iwm_probe(device_t dev) { int i; for (i = 0; i < nitems(iwm_devices); i++) { if (pci_get_vendor(dev) == PCI_VENDOR_INTEL && pci_get_device(dev) == iwm_devices[i].device) { device_set_desc(dev, iwm_devices[i].cfg->name); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int iwm_dev_check(device_t dev) { struct iwm_softc *sc; uint16_t devid; int i; sc = device_get_softc(dev); devid = pci_get_device(dev); for (i = 0; i < nitems(iwm_devices); i++) { if (iwm_devices[i].device == devid) { sc->cfg = iwm_devices[i].cfg; return (0); } } device_printf(dev, "unknown adapter type\n"); return ENXIO; } /* PCI registers */ #define PCI_CFG_RETRY_TIMEOUT 0x041 static int iwm_pci_attach(device_t dev) { struct iwm_softc *sc; int count, error, rid; uint16_t reg; sc = device_get_softc(dev); /* We disable the RETRY_TIMEOUT register (0x41) to keep * PCI Tx retries from interfering with C3 CPU state */ pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1); /* Enable bus-mastering and hardware bug workaround. */ pci_enable_busmaster(dev); reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg)); /* if !MSI */ if (reg & PCIM_STATUS_INTxSTATE) { reg &= ~PCIM_STATUS_INTxSTATE; } pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg)); rid = PCIR_BAR(0); sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(sc->sc_dev, "can't map mem space\n"); return (ENXIO); } sc->sc_st = rman_get_bustag(sc->sc_mem); sc->sc_sh = rman_get_bushandle(sc->sc_mem); /* Install interrupt handler. */ count = 1; rid = 0; if (pci_alloc_msi(dev, &count) == 0) rid = 1; sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | (rid != 0 ? 0 : RF_SHAREABLE)); if (sc->sc_irq == NULL) { device_printf(dev, "can't map interrupt\n"); return (ENXIO); } error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, iwm_intr, sc, &sc->sc_ih); if (sc->sc_ih == NULL) { device_printf(dev, "can't establish interrupt"); return (ENXIO); } sc->sc_dmat = bus_get_dma_tag(sc->sc_dev); return (0); } static void iwm_pci_detach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); if (sc->sc_irq != NULL) { bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_irq), sc->sc_irq); pci_release_msi(dev); } if (sc->sc_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem), sc->sc_mem); } static int iwm_attach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); struct ieee80211com *ic = &sc->sc_ic; int error; int txq_i, i; sc->sc_dev = dev; sc->sc_attached = 1; IWM_LOCK_INIT(sc); mbufq_init(&sc->sc_snd, ifqmaxlen); callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0); callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0); TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc); sc->sc_notif_wait = iwm_notification_wait_init(sc); if (sc->sc_notif_wait == NULL) { device_printf(dev, "failed to init notification wait struct\n"); goto fail; } sc->sf_state = IWM_SF_UNINIT; /* Init phy db */ sc->sc_phy_db = iwm_phy_db_init(sc); if (!sc->sc_phy_db) { device_printf(dev, "Cannot init phy_db\n"); goto fail; } /* Set EBS as successful as long as not stated otherwise by the FW. */ sc->last_ebs_successful = TRUE; /* PCI attach */ error = iwm_pci_attach(dev); if (error != 0) goto fail; sc->sc_wantresp = -1; /* Check device type */ error = iwm_dev_check(dev); if (error != 0) goto fail; sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV); /* * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have * changed, and now the revision step also includes bit 0-1 (no more * "dash" value). To keep hw_rev backwards compatible - we'll store it * in the old format. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) | (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2); if (iwm_prepare_card_hw(sc) != 0) { device_printf(dev, "could not initialize hardware\n"); goto fail; } if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { int ret; uint32_t hw_step; /* * In order to recognize C step the driver should read the * chip version id located at the AUX bus MISC address. */ IWM_SETBITS(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE); DELAY(2); ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000); if (!ret) { device_printf(sc->sc_dev, "Failed to wake up the nic\n"); goto fail; } if (iwm_nic_lock(sc)) { hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG); hw_step |= IWM_ENABLE_WFPM; iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step); hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG); hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF; if (hw_step == 0x3) sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) | (IWM_SILICON_C_STEP << 2); iwm_nic_unlock(sc); } else { device_printf(sc->sc_dev, "Failed to lock the nic\n"); goto fail; } } /* special-case 7265D, it has the same PCI IDs. */ if (sc->cfg == &iwm7265_cfg && (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) { sc->cfg = &iwm7265d_cfg; } /* Allocate DMA memory for firmware transfers. */ if ((error = iwm_alloc_fwmem(sc)) != 0) { device_printf(dev, "could not allocate memory for firmware\n"); goto fail; } /* Allocate "Keep Warm" page. */ if ((error = iwm_alloc_kw(sc)) != 0) { device_printf(dev, "could not allocate keep warm page\n"); goto fail; } /* We use ICT interrupts */ if ((error = iwm_alloc_ict(sc)) != 0) { device_printf(dev, "could not allocate ICT table\n"); goto fail; } /* Allocate TX scheduler "rings". */ if ((error = iwm_alloc_sched(sc)) != 0) { device_printf(dev, "could not allocate TX scheduler rings\n"); goto fail; } /* Allocate TX rings */ for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) { if ((error = iwm_alloc_tx_ring(sc, &sc->txq[txq_i], txq_i)) != 0) { device_printf(dev, "could not allocate TX ring %d\n", txq_i); goto fail; } } /* Allocate RX ring. */ if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) { device_printf(dev, "could not allocate RX ring\n"); goto fail; } /* Clear pending interrupts. */ IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff); ic->ic_softc = sc; ic->ic_name = device_get_nameunit(sc->sc_dev); ic->ic_phytype = IEEE80211_T_OFDM; /* not only, but not used */ ic->ic_opmode = IEEE80211_M_STA; /* default to BSS mode */ /* Set device capabilities. */ ic->ic_caps = IEEE80211_C_STA | IEEE80211_C_WPA | /* WPA/RSN */ IEEE80211_C_WME | IEEE80211_C_PMGT | IEEE80211_C_SHSLOT | /* short slot time supported */ IEEE80211_C_SHPREAMBLE /* short preamble supported */ // IEEE80211_C_BGSCAN /* capable of bg scanning */ ; /* Advertise full-offload scanning */ ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD; for (i = 0; i < nitems(sc->sc_phyctxt); i++) { sc->sc_phyctxt[i].id = i; sc->sc_phyctxt[i].color = 0; sc->sc_phyctxt[i].ref = 0; sc->sc_phyctxt[i].channel = NULL; } /* Default noise floor */ sc->sc_noise = -96; /* Max RSSI */ sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM; sc->sc_preinit_hook.ich_func = iwm_preinit; sc->sc_preinit_hook.ich_arg = sc; if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) { device_printf(dev, "config_intrhook_establish failed\n"); goto fail; } #ifdef IWM_DEBUG SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLFLAG_RW, &sc->sc_debug, 0, "control debugging"); #endif IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "<-%s\n", __func__); return 0; /* Free allocated memory if something failed during attachment. */ fail: iwm_detach_local(sc, 0); return ENXIO; } static int iwm_is_valid_ether_addr(uint8_t *addr) { char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr)) return (FALSE); return (TRUE); } static int iwm_wme_update(struct ieee80211com *ic) { #define IWM_EXP2(x) ((1 << (x)) - 1) /* CWmin = 2^ECWmin - 1 */ struct iwm_softc *sc = ic->ic_softc; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_vap *ivp = IWM_VAP(vap); struct iwm_node *in; struct wmeParams tmp[WME_NUM_AC]; int aci, error; if (vap == NULL) return (0); IEEE80211_LOCK(ic); for (aci = 0; aci < WME_NUM_AC; aci++) tmp[aci] = ic->ic_wme.wme_chanParams.cap_wmeParams[aci]; IEEE80211_UNLOCK(ic); IWM_LOCK(sc); for (aci = 0; aci < WME_NUM_AC; aci++) { const struct wmeParams *ac = &tmp[aci]; ivp->queue_params[aci].aifsn = ac->wmep_aifsn; ivp->queue_params[aci].cw_min = IWM_EXP2(ac->wmep_logcwmin); ivp->queue_params[aci].cw_max = IWM_EXP2(ac->wmep_logcwmax); ivp->queue_params[aci].edca_txop = IEEE80211_TXOP_TO_US(ac->wmep_txopLimit); } ivp->have_wme = TRUE; if (ivp->is_uploaded && vap->iv_bss != NULL) { in = IWM_NODE(vap->iv_bss); if (in->in_assoc) { if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC\n", __func__); } } } IWM_UNLOCK(sc); return (0); #undef IWM_EXP2 } static void iwm_preinit(void *arg) { struct iwm_softc *sc = arg; device_t dev = sc->sc_dev; struct ieee80211com *ic = &sc->sc_ic; int error; IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s\n", __func__); IWM_LOCK(sc); if ((error = iwm_start_hw(sc)) != 0) { device_printf(dev, "could not initialize hardware\n"); IWM_UNLOCK(sc); goto fail; } error = iwm_run_init_mvm_ucode(sc, 1); iwm_stop_device(sc); if (error) { IWM_UNLOCK(sc); goto fail; } device_printf(dev, "hw rev 0x%x, fw ver %s, address %s\n", sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK, sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr)); /* not all hardware can do 5GHz band */ if (!sc->nvm_data->sku_cap_band_52GHz_enable) memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0, sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A])); IWM_UNLOCK(sc); iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels); /* * At this point we've committed - if we fail to do setup, * we now also have to tear down the net80211 state. */ ieee80211_ifattach(ic); ic->ic_vap_create = iwm_vap_create; ic->ic_vap_delete = iwm_vap_delete; ic->ic_raw_xmit = iwm_raw_xmit; ic->ic_node_alloc = iwm_node_alloc; ic->ic_scan_start = iwm_scan_start; ic->ic_scan_end = iwm_scan_end; ic->ic_update_mcast = iwm_update_mcast; ic->ic_getradiocaps = iwm_init_channel_map; ic->ic_set_channel = iwm_set_channel; ic->ic_scan_curchan = iwm_scan_curchan; ic->ic_scan_mindwell = iwm_scan_mindwell; ic->ic_wme.wme_update = iwm_wme_update; ic->ic_parent = iwm_parent; ic->ic_transmit = iwm_transmit; iwm_radiotap_attach(sc); if (bootverbose) ieee80211_announce(ic); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "<-%s\n", __func__); config_intrhook_disestablish(&sc->sc_preinit_hook); return; fail: config_intrhook_disestablish(&sc->sc_preinit_hook); iwm_detach_local(sc, 0); } /* * Attach the interface to 802.11 radiotap. */ static void iwm_radiotap_attach(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s begin\n", __func__); ieee80211_radiotap_attach(ic, &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap), IWM_TX_RADIOTAP_PRESENT, &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap), IWM_RX_RADIOTAP_PRESENT); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s end\n", __func__); } static struct ieee80211vap * iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN], const uint8_t mac[IEEE80211_ADDR_LEN]) { struct iwm_vap *ivp; struct ieee80211vap *vap; if (!TAILQ_EMPTY(&ic->ic_vaps)) /* only one at a time */ return NULL; ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO); vap = &ivp->iv_vap; ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid); vap->iv_bmissthreshold = 10; /* override default */ /* Override with driver methods. */ ivp->iv_newstate = vap->iv_newstate; vap->iv_newstate = iwm_newstate; ivp->id = IWM_DEFAULT_MACID; ivp->color = IWM_DEFAULT_COLOR; ivp->have_wme = FALSE; ivp->ps_disabled = FALSE; ieee80211_ratectl_init(vap); /* Complete setup. */ ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status, mac); ic->ic_opmode = opmode; return vap; } static void iwm_vap_delete(struct ieee80211vap *vap) { struct iwm_vap *ivp = IWM_VAP(vap); ieee80211_ratectl_deinit(vap); ieee80211_vap_detach(vap); free(ivp, M_80211_VAP); } static void +iwm_xmit_queue_drain(struct iwm_softc *sc) +{ + struct mbuf *m; + struct ieee80211_node *ni; + + while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) { + ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; + ieee80211_free_node(ni); + m_freem(m); + } +} + +static void iwm_scan_start(struct ieee80211com *ic) { struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_softc *sc = ic->ic_softc; int error; IWM_LOCK(sc); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { /* This should not be possible */ device_printf(sc->sc_dev, "%s: Previous scan not completed yet\n", __func__); } if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) error = iwm_mvm_umac_scan(sc); else error = iwm_mvm_lmac_scan(sc); if (error != 0) { device_printf(sc->sc_dev, "could not initiate scan\n"); IWM_UNLOCK(sc); ieee80211_cancel_scan(vap); } else { sc->sc_flags |= IWM_FLAG_SCAN_RUNNING; iwm_led_blink_start(sc); IWM_UNLOCK(sc); } } static void iwm_scan_end(struct ieee80211com *ic) { struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_softc *sc = ic->ic_softc; IWM_LOCK(sc); iwm_led_blink_stop(sc); if (vap->iv_state == IEEE80211_S_RUN) iwm_mvm_led_enable(sc); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { /* * Removing IWM_FLAG_SCAN_RUNNING now, is fine because * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq * taskqueue. */ sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; iwm_mvm_scan_stop_wait(sc); } IWM_UNLOCK(sc); /* * Make sure we don't race, if sc_es_task is still enqueued here. * This is to make sure that it won't call ieee80211_scan_done * when we have already started the next scan. */ taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL); } static void iwm_update_mcast(struct ieee80211com *ic) { } static void iwm_set_channel(struct ieee80211com *ic) { } static void iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell) { } static void iwm_scan_mindwell(struct ieee80211_scan_state *ss) { return; } void iwm_init_task(void *arg1) { struct iwm_softc *sc = arg1; IWM_LOCK(sc); while (sc->sc_flags & IWM_FLAG_BUSY) msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0); sc->sc_flags |= IWM_FLAG_BUSY; iwm_stop(sc); if (sc->sc_ic.ic_nrunning > 0) iwm_init(sc); sc->sc_flags &= ~IWM_FLAG_BUSY; wakeup(&sc->sc_flags); IWM_UNLOCK(sc); } static int iwm_resume(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); int do_reinit = 0; /* * We disable the RETRY_TIMEOUT register (0x41) to keep * PCI Tx retries from interfering with C3 CPU state. */ pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1); iwm_init_task(device_get_softc(dev)); IWM_LOCK(sc); if (sc->sc_flags & IWM_FLAG_SCANNING) { sc->sc_flags &= ~IWM_FLAG_SCANNING; do_reinit = 1; } IWM_UNLOCK(sc); if (do_reinit) ieee80211_resume_all(&sc->sc_ic); return 0; } static int iwm_suspend(device_t dev) { int do_stop = 0; struct iwm_softc *sc = device_get_softc(dev); do_stop = !! (sc->sc_ic.ic_nrunning > 0); ieee80211_suspend_all(&sc->sc_ic); if (do_stop) { IWM_LOCK(sc); iwm_stop(sc); sc->sc_flags |= IWM_FLAG_SCANNING; IWM_UNLOCK(sc); } return (0); } static int iwm_detach_local(struct iwm_softc *sc, int do_net80211) { struct iwm_fw_info *fw = &sc->sc_fw; device_t dev = sc->sc_dev; int i; if (!sc->sc_attached) return 0; sc->sc_attached = 0; if (do_net80211) ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task); callout_drain(&sc->sc_led_blink_to); callout_drain(&sc->sc_watchdog_to); iwm_stop_device(sc); if (do_net80211) { + IWM_LOCK(sc); + iwm_xmit_queue_drain(sc); + IWM_UNLOCK(sc); ieee80211_ifdetach(&sc->sc_ic); } iwm_phy_db_free(sc->sc_phy_db); sc->sc_phy_db = NULL; iwm_free_nvm_data(sc->nvm_data); /* Free descriptor rings */ iwm_free_rx_ring(sc, &sc->rxq); for (i = 0; i < nitems(sc->txq); i++) iwm_free_tx_ring(sc, &sc->txq[i]); /* Free firmware */ if (fw->fw_fp != NULL) iwm_fw_info_free(fw); /* Free scheduler */ iwm_dma_contig_free(&sc->sched_dma); iwm_dma_contig_free(&sc->ict_dma); iwm_dma_contig_free(&sc->kw_dma); iwm_dma_contig_free(&sc->fw_dma); iwm_free_fw_paging(sc); /* Finished with the hardware - detach things */ iwm_pci_detach(dev); if (sc->sc_notif_wait != NULL) { iwm_notification_wait_free(sc->sc_notif_wait); sc->sc_notif_wait = NULL; } - mbufq_drain(&sc->sc_snd); IWM_LOCK_DESTROY(sc); return (0); } static int iwm_detach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); return (iwm_detach_local(sc, 1)); } static device_method_t iwm_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, iwm_probe), DEVMETHOD(device_attach, iwm_attach), DEVMETHOD(device_detach, iwm_detach), DEVMETHOD(device_suspend, iwm_suspend), DEVMETHOD(device_resume, iwm_resume), DEVMETHOD_END }; static driver_t iwm_pci_driver = { "iwm", iwm_pci_methods, sizeof (struct iwm_softc) }; static devclass_t iwm_devclass; DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL); MODULE_DEPEND(iwm, firmware, 1, 1, 1); MODULE_DEPEND(iwm, pci, 1, 1, 1); MODULE_DEPEND(iwm, wlan, 1, 1, 1); Index: projects/runtime-coverage/sys/dev/iwm/if_iwm_scan.c =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwm_scan.c (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwm_scan.c (revision 321585) @@ -1,885 +1,882 @@ /* $OpenBSD: if_iwm.c,v 1.39 2015/03/23 00:35:19 jsg Exp $ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * BEGIN mvm/scan.c */ #define IWM_DENSE_EBS_SCAN_RATIO 5 #define IWM_SPARSE_EBS_SCAN_RATIO 1 static uint16_t iwm_mvm_scan_rx_chain(struct iwm_softc *sc) { uint16_t rx_chain; uint8_t rx_ant; rx_ant = iwm_mvm_get_valid_rx_ant(sc); rx_chain = rx_ant << IWM_PHY_RX_CHAIN_VALID_POS; rx_chain |= rx_ant << IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS; rx_chain |= rx_ant << IWM_PHY_RX_CHAIN_FORCE_SEL_POS; rx_chain |= 0x1 << IWM_PHY_RX_CHAIN_DRIVER_FORCE_POS; return htole16(rx_chain); } static uint32_t iwm_mvm_scan_rxon_flags(struct ieee80211_channel *c) { if (IEEE80211_IS_CHAN_2GHZ(c)) return htole32(IWM_PHY_BAND_24); else return htole32(IWM_PHY_BAND_5); } static uint32_t iwm_mvm_scan_rate_n_flags(struct iwm_softc *sc, int flags, int no_cck) { uint32_t tx_ant; int i, ind; for (i = 0, ind = sc->sc_scan_last_antenna; i < IWM_RATE_MCS_ANT_NUM; i++) { ind = (ind + 1) % IWM_RATE_MCS_ANT_NUM; if (iwm_mvm_get_valid_tx_ant(sc) & (1 << ind)) { sc->sc_scan_last_antenna = ind; break; } } tx_ant = (1 << sc->sc_scan_last_antenna) << IWM_RATE_MCS_ANT_POS; if ((flags & IEEE80211_CHAN_2GHZ) && !no_cck) return htole32(IWM_RATE_1M_PLCP | IWM_RATE_MCS_CCK_MSK | tx_ant); else return htole32(IWM_RATE_6M_PLCP | tx_ant); } static inline boolean_t iwm_mvm_rrm_scan_needed(struct iwm_softc *sc) { /* require rrm scan whenever the fw supports it */ return fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT); } #ifdef IWM_DEBUG static const char * iwm_mvm_ebs_status_str(enum iwm_scan_ebs_status status) { switch (status) { case IWM_SCAN_EBS_SUCCESS: return "successful"; case IWM_SCAN_EBS_INACTIVE: return "inactive"; case IWM_SCAN_EBS_FAILED: case IWM_SCAN_EBS_CHAN_NOT_FOUND: default: return "failed"; } } static const char * iwm_mvm_offload_status_str(enum iwm_scan_offload_complete_status status) { return (status == IWM_SCAN_OFFLOAD_ABORTED) ? "aborted" : "completed"; } #endif void iwm_mvm_rx_lmac_scan_complete_notif(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_periodic_scan_complete *scan_notif = (void *)pkt->data; /* If this happens, the firmware has mistakenly sent an LMAC * notification during UMAC scans -- warn and ignore it. */ if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) { device_printf(sc->sc_dev, "%s: Mistakenly got LMAC notification during UMAC scan\n", __func__); return; } IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Regular scan %s, EBS status %s (FW)\n", iwm_mvm_offload_status_str(scan_notif->status), iwm_mvm_ebs_status_str(scan_notif->ebs_status)); sc->last_ebs_successful = scan_notif->ebs_status == IWM_SCAN_EBS_SUCCESS || scan_notif->ebs_status == IWM_SCAN_EBS_INACTIVE; } void iwm_mvm_rx_umac_scan_complete_notif(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_umac_scan_complete *notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Scan completed, uid %u, status %s, EBS status %s\n", le32toh(notif->uid), iwm_mvm_offload_status_str(notif->status), iwm_mvm_ebs_status_str(notif->ebs_status)); if (notif->ebs_status != IWM_SCAN_EBS_SUCCESS && notif->ebs_status != IWM_SCAN_EBS_INACTIVE) sc->last_ebs_successful = FALSE; } static int iwm_mvm_scan_skip_channel(struct ieee80211_channel *c) { if (IEEE80211_IS_CHAN_2GHZ(c) && IEEE80211_IS_CHAN_B(c)) return 0; else if (IEEE80211_IS_CHAN_5GHZ(c) && IEEE80211_IS_CHAN_A(c)) return 0; else return 1; } static uint8_t iwm_mvm_lmac_scan_fill_channels(struct iwm_softc *sc, struct iwm_scan_channel_cfg_lmac *chan, int n_ssids) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_scan_state *ss = ic->ic_scan; struct ieee80211_channel *c; uint8_t nchan; int j; for (nchan = j = 0; - j < ic->ic_nchans && nchan < sc->ucode_capa.n_scan_channels; j++) { - c = &ic->ic_channels[j]; - /* For 2GHz, only populate 11b channels */ - /* For 5GHz, only populate 11a channels */ + j < ss->ss_last && nchan < sc->ucode_capa.n_scan_channels; j++) { + c = ss->ss_chans[j]; /* * Catch other channels, in case we have 900MHz channels or * something in the chanlist. */ - if (iwm_mvm_scan_skip_channel(c)) { + if (!IEEE80211_IS_CHAN_2GHZ(c) && !IEEE80211_IS_CHAN_5GHZ(c)) { IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "%s: skipping channel (freq=%d, ieee=%d, flags=0x%08x)\n", __func__, c->ic_freq, c->ic_ieee, c->ic_flags); continue; } IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "Adding channel %d (%d Mhz) to the list\n", nchan, c->ic_freq); chan->channel_num = htole16(ieee80211_mhz2ieee(c->ic_freq, 0)); chan->iter_count = htole16(1); chan->iter_interval = htole32(0); chan->flags = htole32(IWM_UNIFIED_SCAN_CHANNEL_PARTIAL); chan->flags |= htole32(IWM_SCAN_CHANNEL_NSSIDS(n_ssids)); /* XXX IEEE80211_SCAN_NOBCAST flag is never set. */ if (!IEEE80211_IS_CHAN_PASSIVE(c) && (!(ss->ss_flags & IEEE80211_SCAN_NOBCAST) || n_ssids != 0)) chan->flags |= htole32(IWM_SCAN_CHANNEL_TYPE_ACTIVE); chan++; nchan++; } return nchan; } static uint8_t iwm_mvm_umac_scan_fill_channels(struct iwm_softc *sc, struct iwm_scan_channel_cfg_umac *chan, int n_ssids) { struct ieee80211com *ic = &sc->sc_ic; + struct ieee80211_scan_state *ss = ic->ic_scan; struct ieee80211_channel *c; uint8_t nchan; int j; for (nchan = j = 0; - j < ic->ic_nchans && nchan < sc->ucode_capa.n_scan_channels; j++) { - c = &ic->ic_channels[j]; - /* For 2GHz, only populate 11b channels */ - /* For 5GHz, only populate 11a channels */ + j < ss->ss_last && nchan < sc->ucode_capa.n_scan_channels; j++) { + c = ss->ss_chans[j]; /* * Catch other channels, in case we have 900MHz channels or * something in the chanlist. */ - if (iwm_mvm_scan_skip_channel(c)) { + if (!IEEE80211_IS_CHAN_2GHZ(c) && !IEEE80211_IS_CHAN_5GHZ(c)) { IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "%s: skipping channel (freq=%d, ieee=%d, flags=0x%08x)\n", __func__, c->ic_freq, c->ic_ieee, c->ic_flags); continue; } IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "Adding channel %d (%d Mhz) to the list\n", nchan, c->ic_freq); chan->channel_num = ieee80211_mhz2ieee(c->ic_freq, 0); chan->iter_count = 1; chan->iter_interval = htole16(0); chan->flags = htole32(IWM_SCAN_CHANNEL_UMAC_NSSIDS(n_ssids)); chan++; nchan++; } return nchan; } static int iwm_mvm_fill_probe_req(struct iwm_softc *sc, struct iwm_scan_probe_req *preq) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct ieee80211_frame *wh = (struct ieee80211_frame *)preq->buf; struct ieee80211_rateset *rs; size_t remain = sizeof(preq->buf); uint8_t *frm, *pos; memset(preq, 0, sizeof(*preq)); /* Ensure enough space for header and SSID IE. */ if (remain < sizeof(*wh) + 2) return ENOBUFS; /* * Build a probe request frame. Most of the following code is a * copy & paste of what is done in net80211. */ wh->i_fc[0] = IEEE80211_FC0_VERSION_0 | IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_PROBE_REQ; wh->i_fc[1] = IEEE80211_FC1_DIR_NODS; IEEE80211_ADDR_COPY(wh->i_addr1, ieee80211broadcastaddr); IEEE80211_ADDR_COPY(wh->i_addr2, vap ? vap->iv_myaddr : ic->ic_macaddr); IEEE80211_ADDR_COPY(wh->i_addr3, ieee80211broadcastaddr); *(uint16_t *)&wh->i_dur[0] = 0; /* filled by HW */ *(uint16_t *)&wh->i_seq[0] = 0; /* filled by HW */ frm = (uint8_t *)(wh + 1); frm = ieee80211_add_ssid(frm, NULL, 0); /* Tell the firmware where the MAC header is. */ preq->mac_header.offset = 0; preq->mac_header.len = htole16(frm - (uint8_t *)wh); remain -= frm - (uint8_t *)wh; /* Fill in 2GHz IEs and tell firmware where they are. */ rs = &ic->ic_sup_rates[IEEE80211_MODE_11G]; if (rs->rs_nrates > IEEE80211_RATE_SIZE) { if (remain < 4 + rs->rs_nrates) return ENOBUFS; } else if (remain < 2 + rs->rs_nrates) { return ENOBUFS; } preq->band_data[0].offset = htole16(frm - (uint8_t *)wh); pos = frm; frm = ieee80211_add_rates(frm, rs); if (rs->rs_nrates > IEEE80211_RATE_SIZE) frm = ieee80211_add_xrates(frm, rs); preq->band_data[0].len = htole16(frm - pos); remain -= frm - pos; if (iwm_mvm_rrm_scan_needed(sc)) { if (remain < 3) return ENOBUFS; *frm++ = IEEE80211_ELEMID_DSPARMS; *frm++ = 1; *frm++ = 0; remain -= 3; } if (sc->nvm_data->sku_cap_band_52GHz_enable) { /* Fill in 5GHz IEs. */ rs = &ic->ic_sup_rates[IEEE80211_MODE_11A]; if (rs->rs_nrates > IEEE80211_RATE_SIZE) { if (remain < 4 + rs->rs_nrates) return ENOBUFS; } else if (remain < 2 + rs->rs_nrates) { return ENOBUFS; } preq->band_data[1].offset = htole16(frm - (uint8_t *)wh); pos = frm; frm = ieee80211_add_rates(frm, rs); if (rs->rs_nrates > IEEE80211_RATE_SIZE) frm = ieee80211_add_xrates(frm, rs); preq->band_data[1].len = htole16(frm - pos); remain -= frm - pos; } /* Send 11n IEs on both 2GHz and 5GHz bands. */ preq->common_data.offset = htole16(frm - (uint8_t *)wh); pos = frm; #if 0 if (ic->ic_flags & IEEE80211_F_HTON) { if (remain < 28) return ENOBUFS; frm = ieee80211_add_htcaps(frm, ic); /* XXX add WME info? */ } #endif preq->common_data.len = htole16(frm - pos); return 0; } int iwm_mvm_config_umac_scan(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_scan_config *scan_config; int ret, j, nchan; size_t cmd_size; struct ieee80211_channel *c; struct iwm_host_cmd hcmd = { .id = iwm_cmd_id(IWM_SCAN_CFG_CMD, IWM_ALWAYS_LONG_GROUP, 0), .flags = IWM_CMD_SYNC, }; static const uint32_t rates = (IWM_SCAN_CONFIG_RATE_1M | IWM_SCAN_CONFIG_RATE_2M | IWM_SCAN_CONFIG_RATE_5M | IWM_SCAN_CONFIG_RATE_11M | IWM_SCAN_CONFIG_RATE_6M | IWM_SCAN_CONFIG_RATE_9M | IWM_SCAN_CONFIG_RATE_12M | IWM_SCAN_CONFIG_RATE_18M | IWM_SCAN_CONFIG_RATE_24M | IWM_SCAN_CONFIG_RATE_36M | IWM_SCAN_CONFIG_RATE_48M | IWM_SCAN_CONFIG_RATE_54M); cmd_size = sizeof(*scan_config) + sc->ucode_capa.n_scan_channels; scan_config = malloc(cmd_size, M_DEVBUF, M_NOWAIT | M_ZERO); if (scan_config == NULL) return ENOMEM; scan_config->tx_chains = htole32(iwm_mvm_get_valid_tx_ant(sc)); scan_config->rx_chains = htole32(iwm_mvm_get_valid_rx_ant(sc)); scan_config->legacy_rates = htole32(rates | IWM_SCAN_CONFIG_SUPPORTED_RATE(rates)); /* These timings correspond to iwlwifi's UNASSOC scan. */ scan_config->dwell_active = 10; scan_config->dwell_passive = 110; scan_config->dwell_fragmented = 44; scan_config->dwell_extended = 90; scan_config->out_of_channel_time = htole32(0); scan_config->suspend_time = htole32(0); IEEE80211_ADDR_COPY(scan_config->mac_addr, vap ? vap->iv_myaddr : ic->ic_macaddr); scan_config->bcast_sta_id = sc->sc_aux_sta.sta_id; scan_config->channel_flags = IWM_CHANNEL_FLAG_EBS | IWM_CHANNEL_FLAG_ACCURATE_EBS | IWM_CHANNEL_FLAG_EBS_ADD | IWM_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE; for (nchan = j = 0; j < ic->ic_nchans && nchan < sc->ucode_capa.n_scan_channels; j++) { c = &ic->ic_channels[j]; /* For 2GHz, only populate 11b channels */ /* For 5GHz, only populate 11a channels */ /* * Catch other channels, in case we have 900MHz channels or * something in the chanlist. */ if (iwm_mvm_scan_skip_channel(c)) continue; scan_config->channel_array[nchan++] = ieee80211_mhz2ieee(c->ic_freq, 0); } scan_config->flags = htole32(IWM_SCAN_CONFIG_FLAG_ACTIVATE | IWM_SCAN_CONFIG_FLAG_ALLOW_CHUB_REQS | IWM_SCAN_CONFIG_FLAG_SET_TX_CHAINS | IWM_SCAN_CONFIG_FLAG_SET_RX_CHAINS | IWM_SCAN_CONFIG_FLAG_SET_AUX_STA_ID | IWM_SCAN_CONFIG_FLAG_SET_ALL_TIMES | IWM_SCAN_CONFIG_FLAG_SET_LEGACY_RATES | IWM_SCAN_CONFIG_FLAG_SET_MAC_ADDR | IWM_SCAN_CONFIG_FLAG_SET_CHANNEL_FLAGS| IWM_SCAN_CONFIG_N_CHANNELS(nchan) | IWM_SCAN_CONFIG_FLAG_CLEAR_FRAGMENTED); hcmd.data[0] = scan_config; hcmd.len[0] = cmd_size; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Sending UMAC scan config\n"); ret = iwm_send_cmd(sc, &hcmd); if (!ret) IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan config was sent successfully\n"); free(scan_config, M_DEVBUF); return ret; } static boolean_t iwm_mvm_scan_use_ebs(struct iwm_softc *sc) { const struct iwm_ucode_capabilities *capa = &sc->ucode_capa; /* We can only use EBS if: * 1. the feature is supported; * 2. the last EBS was successful; * 3. if only single scan, the single scan EBS API is supported; * 4. it's not a p2p find operation. */ return ((capa->flags & IWM_UCODE_TLV_FLAGS_EBS_SUPPORT) && sc->last_ebs_successful); } int iwm_mvm_umac_scan(struct iwm_softc *sc) { struct iwm_host_cmd hcmd = { .id = iwm_cmd_id(IWM_SCAN_REQ_UMAC, IWM_ALWAYS_LONG_GROUP, 0), .len = { 0, }, .data = { NULL, }, .flags = IWM_CMD_SYNC, }; struct ieee80211_scan_state *ss = sc->sc_ic.ic_scan; struct iwm_scan_req_umac *req; struct iwm_scan_req_umac_tail *tail; size_t req_len; uint8_t i, nssid; int ret; req_len = sizeof(struct iwm_scan_req_umac) + (sizeof(struct iwm_scan_channel_cfg_umac) * sc->ucode_capa.n_scan_channels) + sizeof(struct iwm_scan_req_umac_tail); if (req_len > IWM_MAX_CMD_PAYLOAD_SIZE) return ENOMEM; req = malloc(req_len, M_DEVBUF, M_NOWAIT | M_ZERO); if (req == NULL) return ENOMEM; hcmd.len[0] = (uint16_t)req_len; hcmd.data[0] = (void *)req; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Handling ieee80211 scan request\n"); /* These timings correspond to iwlwifi's UNASSOC scan. */ req->active_dwell = 10; req->passive_dwell = 110; req->fragmented_dwell = 44; req->extended_dwell = 90; req->max_out_time = 0; req->suspend_time = 0; req->scan_priority = htole32(IWM_SCAN_PRIORITY_HIGH); req->ooc_priority = htole32(IWM_SCAN_PRIORITY_HIGH); nssid = MIN(ss->ss_nssid, IWM_PROBE_OPTION_MAX); req->n_channels = iwm_mvm_umac_scan_fill_channels(sc, (struct iwm_scan_channel_cfg_umac *)req->data, nssid); req->general_flags = htole32(IWM_UMAC_SCAN_GEN_FLAGS_PASS_ALL | IWM_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE | IWM_UMAC_SCAN_GEN_FLAGS_EXTENDED_DWELL); tail = (void *)((char *)&req->data + sizeof(struct iwm_scan_channel_cfg_umac) * sc->ucode_capa.n_scan_channels); /* Check if we're doing an active directed scan. */ for (i = 0; i < nssid; i++) { tail->direct_scan[i].id = IEEE80211_ELEMID_SSID; tail->direct_scan[i].len = MIN(ss->ss_ssid[i].len, IEEE80211_NWID_LEN); memcpy(tail->direct_scan[i].ssid, ss->ss_ssid[i].ssid, tail->direct_scan[i].len); /* XXX debug */ } if (nssid != 0) { req->general_flags |= htole32(IWM_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT); } else req->general_flags |= htole32(IWM_UMAC_SCAN_GEN_FLAGS_PASSIVE); if (iwm_mvm_scan_use_ebs(sc)) req->channel_flags = IWM_SCAN_CHANNEL_FLAG_EBS | IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE | IWM_SCAN_CHANNEL_FLAG_CACHE_ADD; if (iwm_mvm_rrm_scan_needed(sc)) req->general_flags |= htole32(IWM_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED); ret = iwm_mvm_fill_probe_req(sc, &tail->preq); if (ret) { free(req, M_DEVBUF); return ret; } /* Specify the scan plan: We'll do one iteration. */ tail->schedule[0].interval = 0; tail->schedule[0].iter_count = 1; ret = iwm_send_cmd(sc, &hcmd); if (!ret) IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Scan request was sent successfully\n"); free(req, M_DEVBUF); return ret; } int iwm_mvm_lmac_scan(struct iwm_softc *sc) { struct iwm_host_cmd hcmd = { .id = IWM_SCAN_OFFLOAD_REQUEST_CMD, .len = { 0, }, .data = { NULL, }, .flags = IWM_CMD_SYNC, }; struct ieee80211_scan_state *ss = sc->sc_ic.ic_scan; struct iwm_scan_req_lmac *req; size_t req_len; uint8_t i, nssid; int ret; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Handling ieee80211 scan request\n"); req_len = sizeof(struct iwm_scan_req_lmac) + (sizeof(struct iwm_scan_channel_cfg_lmac) * sc->ucode_capa.n_scan_channels) + sizeof(struct iwm_scan_probe_req); if (req_len > IWM_MAX_CMD_PAYLOAD_SIZE) return ENOMEM; req = malloc(req_len, M_DEVBUF, M_NOWAIT | M_ZERO); if (req == NULL) return ENOMEM; hcmd.len[0] = (uint16_t)req_len; hcmd.data[0] = (void *)req; /* These timings correspond to iwlwifi's UNASSOC scan. */ req->active_dwell = 10; req->passive_dwell = 110; req->fragmented_dwell = 44; req->extended_dwell = 90; req->max_out_time = 0; req->suspend_time = 0; req->scan_prio = htole32(IWM_SCAN_PRIORITY_HIGH); req->rx_chain_select = iwm_mvm_scan_rx_chain(sc); req->iter_num = htole32(1); req->delay = 0; req->scan_flags = htole32(IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL | IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE | IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL); if (iwm_mvm_rrm_scan_needed(sc)) req->scan_flags |= htole32(IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED); - req->flags = iwm_mvm_scan_rxon_flags(&sc->sc_ic.ic_channels[0]); + req->flags = iwm_mvm_scan_rxon_flags(sc->sc_ic.ic_scan->ss_chans[0]); req->filter_flags = htole32(IWM_MAC_FILTER_ACCEPT_GRP | IWM_MAC_FILTER_IN_BEACON); /* Tx flags 2 GHz. */ req->tx_cmd[0].tx_flags = htole32(IWM_TX_CMD_FLG_SEQ_CTL | IWM_TX_CMD_FLG_BT_DIS); req->tx_cmd[0].rate_n_flags = iwm_mvm_scan_rate_n_flags(sc, IEEE80211_CHAN_2GHZ, 1/*XXX*/); req->tx_cmd[0].sta_id = sc->sc_aux_sta.sta_id; /* Tx flags 5 GHz. */ req->tx_cmd[1].tx_flags = htole32(IWM_TX_CMD_FLG_SEQ_CTL | IWM_TX_CMD_FLG_BT_DIS); req->tx_cmd[1].rate_n_flags = iwm_mvm_scan_rate_n_flags(sc, IEEE80211_CHAN_5GHZ, 1/*XXX*/); req->tx_cmd[1].sta_id = sc->sc_aux_sta.sta_id; /* Check if we're doing an active directed scan. */ nssid = MIN(ss->ss_nssid, IWM_PROBE_OPTION_MAX); for (i = 0; i < nssid; i++) { req->direct_scan[i].id = IEEE80211_ELEMID_SSID; req->direct_scan[i].len = MIN(ss->ss_ssid[i].len, IEEE80211_NWID_LEN); memcpy(req->direct_scan[i].ssid, ss->ss_ssid[i].ssid, req->direct_scan[i].len); /* XXX debug */ } if (nssid != 0) { req->scan_flags |= htole32(IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION); } else req->scan_flags |= htole32(IWM_MVM_LMAC_SCAN_FLAG_PASSIVE); req->n_channels = iwm_mvm_lmac_scan_fill_channels(sc, (struct iwm_scan_channel_cfg_lmac *)req->data, nssid); ret = iwm_mvm_fill_probe_req(sc, (struct iwm_scan_probe_req *)(req->data + (sizeof(struct iwm_scan_channel_cfg_lmac) * sc->ucode_capa.n_scan_channels))); if (ret) { free(req, M_DEVBUF); return ret; } /* Specify the scan plan: We'll do one iteration. */ req->schedule[0].iterations = 1; req->schedule[0].full_scan_mul = 1; if (iwm_mvm_scan_use_ebs(sc)) { req->channel_opt[0].flags = htole16(IWM_SCAN_CHANNEL_FLAG_EBS | IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE | IWM_SCAN_CHANNEL_FLAG_CACHE_ADD); req->channel_opt[0].non_ebs_ratio = htole16(IWM_DENSE_EBS_SCAN_RATIO); req->channel_opt[1].flags = htole16(IWM_SCAN_CHANNEL_FLAG_EBS | IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE | IWM_SCAN_CHANNEL_FLAG_CACHE_ADD); req->channel_opt[1].non_ebs_ratio = htole16(IWM_SPARSE_EBS_SCAN_RATIO); } ret = iwm_send_cmd(sc, &hcmd); if (!ret) { IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Scan request was sent successfully\n"); } free(req, M_DEVBUF); return ret; } static int iwm_mvm_lmac_scan_abort(struct iwm_softc *sc) { int ret; struct iwm_host_cmd hcmd = { .id = IWM_SCAN_OFFLOAD_ABORT_CMD, .len = { 0, }, .data = { NULL, }, .flags = IWM_CMD_SYNC, }; uint32_t status; ret = iwm_mvm_send_cmd_status(sc, &hcmd, &status); if (ret) return ret; if (status != IWM_CAN_ABORT_STATUS) { /* * The scan abort will return 1 for success or * 2 for "failure". A failure condition can be * due to simply not being in an active scan which * can occur if we send the scan abort before the * microcode has notified us that a scan is completed. */ IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "SCAN OFFLOAD ABORT ret %d.\n", status); ret = ENOENT; } return ret; } static int iwm_mvm_umac_scan_abort(struct iwm_softc *sc) { struct iwm_umac_scan_abort cmd = {}; int uid, ret; uid = 0; cmd.uid = htole32(uid); IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Sending scan abort, uid %u\n", uid); ret = iwm_mvm_send_cmd_pdu(sc, iwm_cmd_id(IWM_SCAN_ABORT_UMAC, IWM_ALWAYS_LONG_GROUP, 0), 0, sizeof(cmd), &cmd); return ret; } int iwm_mvm_scan_stop_wait(struct iwm_softc *sc) { struct iwm_notification_wait wait_scan_done; static const uint16_t scan_done_notif[] = { IWM_SCAN_COMPLETE_UMAC, IWM_SCAN_OFFLOAD_COMPLETE, }; int ret; iwm_init_notification_wait(sc->sc_notif_wait, &wait_scan_done, scan_done_notif, nitems(scan_done_notif), NULL, NULL); IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Preparing to stop scan\n"); if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) ret = iwm_mvm_umac_scan_abort(sc); else ret = iwm_mvm_lmac_scan_abort(sc); if (ret) { IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "couldn't stop scan\n"); iwm_remove_notification(sc->sc_notif_wait, &wait_scan_done); return ret; } IWM_UNLOCK(sc); ret = iwm_wait_notification(sc->sc_notif_wait, &wait_scan_done, hz); IWM_LOCK(sc); return ret; } Index: projects/runtime-coverage/sys/dev/iwm/if_iwm_sta.c =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwm_sta.c (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwm_sta.c (revision 321585) @@ -1,384 +1,383 @@ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 4.7.3 (tag id d7f6728f57e3ecbb7ef34eb7d9f564d514775d75) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2012 - 2015 Intel Corporation. All rights reserved. * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH * Copyright(c) 2016 Intel Deutschland GmbH * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * BSD LICENSE * * Copyright(c) 2012 - 2015 Intel Corporation. All rights reserved. * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH * Copyright(c) 2016 Intel Deutschland GmbH * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * *****************************************************************************/ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * New version of ADD_STA_sta command added new fields at the end of the * structure, so sending the size of the relevant API's structure is enough to * support both API versions. */ static inline int iwm_mvm_add_sta_cmd_size(struct iwm_softc *sc) { #ifdef notyet return iwm_mvm_has_new_rx_api(mvm) ? sizeof(struct iwm_mvm_add_sta_cmd) : sizeof(struct iwm_mvm_add_sta_cmd_v7); #else return sizeof(struct iwm_mvm_add_sta_cmd); #endif } /* send station add/update command to firmware */ int iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, boolean_t update) { struct iwm_vap *ivp = IWM_VAP(in->in_ni.ni_vap); struct iwm_mvm_add_sta_cmd add_sta_cmd = { .sta_id = IWM_STATION_ID, .mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(ivp->id, ivp->color)), .add_modify = update ? 1 : 0, .station_flags_msk = htole32(IWM_STA_FLG_FAT_EN_MSK | IWM_STA_FLG_MIMO_EN_MSK), .tid_disable_tx = htole16(0xffff), }; int ret; uint32_t status; uint32_t agg_size = 0, mpdu_dens = 0; if (!update) { int ac; for (ac = 0; ac < WME_NUM_AC; ac++) { add_sta_cmd.tfd_queue_msk |= htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]); } IEEE80211_ADDR_COPY(&add_sta_cmd.addr, in->in_ni.ni_bssid); } add_sta_cmd.station_flags |= htole32(agg_size << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT); add_sta_cmd.station_flags |= htole32(mpdu_dens << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT); status = IWM_ADD_STA_SUCCESS; ret = iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, iwm_mvm_add_sta_cmd_size(sc), &add_sta_cmd, &status); if (ret) return ret; switch (status & IWM_ADD_STA_STATUS_MASK) { case IWM_ADD_STA_SUCCESS: IWM_DPRINTF(sc, IWM_DEBUG_NODE, "IWM_ADD_STA PASSED\n"); break; default: ret = EIO; device_printf(sc->sc_dev, "IWM_ADD_STA failed\n"); break; } return ret; } int iwm_mvm_add_sta(struct iwm_softc *sc, struct iwm_node *in) { return iwm_mvm_sta_send_to_fw(sc, in, FALSE); } int iwm_mvm_update_sta(struct iwm_softc *sc, struct iwm_node *in) { return iwm_mvm_sta_send_to_fw(sc, in, TRUE); } int iwm_mvm_drain_sta(struct iwm_softc *sc, struct iwm_vap *ivp, boolean_t drain) { struct iwm_mvm_add_sta_cmd cmd = {}; int ret; uint32_t status; cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(ivp->id, ivp->color)); cmd.sta_id = IWM_STATION_ID; cmd.add_modify = IWM_STA_MODE_MODIFY; cmd.station_flags = drain ? htole32(IWM_STA_FLG_DRAIN_FLOW) : 0; cmd.station_flags_msk = htole32(IWM_STA_FLG_DRAIN_FLOW); status = IWM_ADD_STA_SUCCESS; ret = iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, iwm_mvm_add_sta_cmd_size(sc), &cmd, &status); if (ret) return ret; switch (status & IWM_ADD_STA_STATUS_MASK) { case IWM_ADD_STA_SUCCESS: IWM_DPRINTF(sc, IWM_DEBUG_NODE, "Frames for staid %d will drained in fw\n", IWM_STATION_ID); break; default: ret = EIO; device_printf(sc->sc_dev, "Couldn't drain frames for staid %d\n", IWM_STATION_ID); break; } return ret; } /* * Remove a station from the FW table. Before sending the command to remove * the station validate that the station is indeed known to the driver (sanity * only). */ static int iwm_mvm_rm_sta_common(struct iwm_softc *sc) { struct iwm_mvm_rm_sta_cmd rm_sta_cmd = { .sta_id = IWM_STATION_ID, }; int ret; ret = iwm_mvm_send_cmd_pdu(sc, IWM_REMOVE_STA, 0, sizeof(rm_sta_cmd), &rm_sta_cmd); if (ret) { device_printf(sc->sc_dev, "Failed to remove station. Id=%d\n", IWM_STATION_ID); return ret; } return 0; } int iwm_mvm_rm_sta(struct iwm_softc *sc, struct ieee80211vap *vap, boolean_t is_assoc) { uint32_t tfd_queue_msk = 0; int ret; int ac; ret = iwm_mvm_drain_sta(sc, IWM_VAP(vap), TRUE); if (ret) return ret; - mbufq_drain(&sc->sc_snd); /* XXX needed ? */ for (ac = 0; ac < WME_NUM_AC; ac++) { tfd_queue_msk |= htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]); } ret = iwm_mvm_flush_tx_path(sc, tfd_queue_msk, 0); if (ret) return ret; #ifdef notyet /* function not yet implemented */ ret = iwl_trans_wait_tx_queue_empty(mvm->trans, mvm_sta->tfd_queue_msk); if (ret) return ret; #endif ret = iwm_mvm_drain_sta(sc, IWM_VAP(vap), FALSE); /* if we are associated - we can't remove the AP STA now */ if (is_assoc) return ret; /* XXX wait until STA is drained */ ret = iwm_mvm_rm_sta_common(sc); return ret; } int iwm_mvm_rm_sta_id(struct iwm_softc *sc, struct ieee80211vap *vap) { /* XXX wait until STA is drained */ return iwm_mvm_rm_sta_common(sc); } static int iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta, const uint8_t *addr, uint16_t mac_id, uint16_t color) { struct iwm_mvm_add_sta_cmd cmd; int ret; uint32_t status; memset(&cmd, 0, sizeof(cmd)); cmd.sta_id = sta->sta_id; cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(mac_id, color)); cmd.tfd_queue_msk = htole32(sta->tfd_queue_msk); cmd.tid_disable_tx = htole16(0xffff); if (addr) IEEE80211_ADDR_COPY(cmd.addr, addr); ret = iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, iwm_mvm_add_sta_cmd_size(sc), &cmd, &status); if (ret) return ret; switch (status & IWM_ADD_STA_STATUS_MASK) { case IWM_ADD_STA_SUCCESS: IWM_DPRINTF(sc, IWM_DEBUG_NODE, "Internal station added.\n"); return 0; default: ret = EIO; device_printf(sc->sc_dev, "Add internal station failed, status=0x%x\n", status); break; } return ret; } int iwm_mvm_add_aux_sta(struct iwm_softc *sc) { int ret; sc->sc_aux_sta.sta_id = IWM_AUX_STA_ID; sc->sc_aux_sta.tfd_queue_msk = (1 << IWM_MVM_AUX_QUEUE); /* Map Aux queue to fifo - needs to happen before adding Aux station */ ret = iwm_enable_txq(sc, 0, IWM_MVM_AUX_QUEUE, IWM_MVM_TX_FIFO_MCAST); if (ret) return ret; ret = iwm_mvm_add_int_sta_common(sc, &sc->sc_aux_sta, NULL, IWM_MAC_INDEX_AUX, 0); if (ret) { memset(&sc->sc_aux_sta, 0, sizeof(sc->sc_aux_sta)); sc->sc_aux_sta.sta_id = IWM_MVM_STATION_COUNT; } return ret; } void iwm_mvm_del_aux_sta(struct iwm_softc *sc) { memset(&sc->sc_aux_sta, 0, sizeof(sc->sc_aux_sta)); sc->sc_aux_sta.sta_id = IWM_MVM_STATION_COUNT; } Index: projects/runtime-coverage/sys/dev/iwm/if_iwm_util.c =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwm_util.c (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwm_util.c (revision 321585) @@ -1,504 +1,530 @@ /* $OpenBSD: if_iwm.c,v 1.39 2015/03/23 00:35:19 jsg Exp $ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Send a command to the firmware. We try to implement the Linux * driver interface for the routine. * mostly from if_iwn (iwn_cmd()). * * For now, we always copy the first part and map the second one (if it exists). */ int iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) { struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE]; struct iwm_tfd *desc; struct iwm_tx_data *txdata = NULL; struct iwm_device_cmd *cmd; struct mbuf *m; bus_dma_segment_t seg; bus_addr_t paddr; uint32_t addr_lo; int error = 0, i, paylen, off; int code; int async, wantresp; int group_id; int nsegs; size_t hdrlen, datasz; uint8_t *data; code = hcmd->id; async = hcmd->flags & IWM_CMD_ASYNC; wantresp = hcmd->flags & IWM_CMD_WANT_SKB; data = NULL; for (i = 0, paylen = 0; i < nitems(hcmd->len); i++) { paylen += hcmd->len[i]; } /* if the command wants an answer, busy sc_cmd_resp */ if (wantresp) { KASSERT(!async, ("invalid async parameter")); while (sc->sc_wantresp != -1) msleep(&sc->sc_wantresp, &sc->sc_mtx, 0, "iwmcmdsl", 0); sc->sc_wantresp = ring->qid << 16 | ring->cur; IWM_DPRINTF(sc, IWM_DEBUG_CMD, "wantresp is %x\n", sc->sc_wantresp); } /* * Is the hardware still available? (after e.g. above wait). */ if (sc->sc_flags & IWM_FLAG_STOPPED) { error = ENXIO; goto out; } desc = &ring->desc[ring->cur]; txdata = &ring->data[ring->cur]; group_id = iwm_cmd_groupid(code); if (group_id != 0) { hdrlen = sizeof(cmd->hdr_wide); datasz = sizeof(cmd->data_wide); } else { hdrlen = sizeof(cmd->hdr); datasz = sizeof(cmd->data); } if (paylen > datasz) { IWM_DPRINTF(sc, IWM_DEBUG_CMD, "large command paylen=%u len0=%u\n", paylen, hcmd->len[0]); /* Command is too large */ size_t totlen = hdrlen + paylen; if (paylen > IWM_MAX_CMD_PAYLOAD_SIZE) { device_printf(sc->sc_dev, "firmware command too long (%zd bytes)\n", totlen); error = EINVAL; goto out; } m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE); if (m == NULL) { error = ENOBUFS; goto out; } m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, txdata->map, m, &seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); m_freem(m); goto out; } txdata->m = m; /* mbuf will be freed in iwm_cmd_done() */ cmd = mtod(m, struct iwm_device_cmd *); paddr = seg.ds_addr; } else { cmd = &ring->cmd[ring->cur]; paddr = txdata->cmd_paddr; } if (group_id != 0) { cmd->hdr_wide.opcode = iwm_cmd_opcode(code); cmd->hdr_wide.group_id = group_id; cmd->hdr_wide.qid = ring->qid; cmd->hdr_wide.idx = ring->cur; cmd->hdr_wide.length = htole16(paylen); cmd->hdr_wide.version = iwm_cmd_version(code); data = cmd->data_wide; } else { cmd->hdr.code = iwm_cmd_opcode(code); cmd->hdr.flags = 0; cmd->hdr.qid = ring->qid; cmd->hdr.idx = ring->cur; data = cmd->data; } for (i = 0, off = 0; i < nitems(hcmd->data); i++) { if (hcmd->len[i] == 0) continue; memcpy(data + off, hcmd->data[i], hcmd->len[i]); off += hcmd->len[i]; } KASSERT(off == paylen, ("off %d != paylen %d", off, paylen)); /* lo field is not aligned */ addr_lo = htole32((uint32_t)paddr); memcpy(&desc->tbs[0].lo, &addr_lo, sizeof(uint32_t)); desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(paddr) | ((hdrlen + paylen) << 4)); desc->num_tbs = 1; IWM_DPRINTF(sc, IWM_DEBUG_CMD, "iwm_send_cmd 0x%x size=%lu %s\n", code, (unsigned long) (hcmd->len[0] + hcmd->len[1] + hdrlen), async ? " (async)" : ""); if (paylen > datasz) { bus_dmamap_sync(ring->data_dmat, txdata->map, BUS_DMASYNC_PREWRITE); } else { bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); } bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); error = iwm_pcie_set_cmd_in_flight(sc); if (error) goto out; ring->queued++; #if 0 iwm_update_sched(sc, ring->qid, ring->cur, 0, 0); #endif IWM_DPRINTF(sc, IWM_DEBUG_CMD, "sending command 0x%x qid %d, idx %d\n", code, ring->qid, ring->cur); /* Kick command ring. */ ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT; IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); if (!async) { /* m..m-mmyy-mmyyyy-mym-ym m-my generation */ int generation = sc->sc_generation; error = msleep(desc, &sc->sc_mtx, PCATCH, "iwmcmd", hz); if (error == 0) { /* if hardware is no longer up, return error */ if (generation != sc->sc_generation) { error = ENXIO; } else { hcmd->resp_pkt = (void *)sc->sc_cmd_resp; } } } out: if (wantresp && error != 0) { iwm_free_resp(sc, hcmd); } return error; } /* iwlwifi: mvm/utils.c */ int iwm_mvm_send_cmd_pdu(struct iwm_softc *sc, uint32_t id, uint32_t flags, uint16_t len, const void *data) { struct iwm_host_cmd cmd = { .id = id, .len = { len, }, .data = { data, }, .flags = flags, }; return iwm_send_cmd(sc, &cmd); } /* iwlwifi: mvm/utils.c */ int iwm_mvm_send_cmd_status(struct iwm_softc *sc, struct iwm_host_cmd *cmd, uint32_t *status) { struct iwm_rx_packet *pkt; struct iwm_cmd_response *resp; int error, resp_len; KASSERT((cmd->flags & IWM_CMD_WANT_SKB) == 0, ("invalid command")); cmd->flags |= IWM_CMD_SYNC | IWM_CMD_WANT_SKB; if ((error = iwm_send_cmd(sc, cmd)) != 0) return error; pkt = cmd->resp_pkt; /* Can happen if RFKILL is asserted */ if (!pkt) { error = 0; goto out_free_resp; } if (pkt->hdr.flags & IWM_CMD_FAILED_MSK) { error = EIO; goto out_free_resp; } resp_len = iwm_rx_packet_payload_len(pkt); if (resp_len != sizeof(*resp)) { error = EIO; goto out_free_resp; } resp = (void *)pkt->data; *status = le32toh(resp->status); out_free_resp: iwm_free_resp(sc, cmd); return error; } /* iwlwifi/mvm/utils.c */ int iwm_mvm_send_cmd_pdu_status(struct iwm_softc *sc, uint32_t id, uint16_t len, const void *data, uint32_t *status) { struct iwm_host_cmd cmd = { .id = id, .len = { len, }, .data = { data, }, }; return iwm_mvm_send_cmd_status(sc, &cmd, status); } void iwm_free_resp(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) { KASSERT(sc->sc_wantresp != -1, ("already freed")); KASSERT((hcmd->flags & (IWM_CMD_WANT_SKB|IWM_CMD_SYNC)) == (IWM_CMD_WANT_SKB|IWM_CMD_SYNC), ("invalid flags")); sc->sc_wantresp = -1; wakeup(&sc->sc_wantresp); } static void iwm_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; KASSERT(nsegs == 1, ("too many DMA segments, %d should be 1", nsegs)); *(bus_addr_t *)arg = segs[0].ds_addr; } int iwm_dma_contig_alloc(bus_dma_tag_t tag, struct iwm_dma_info *dma, bus_size_t size, bus_size_t alignment) { int error; dma->tag = NULL; dma->map = NULL; dma->size = size; dma->vaddr = NULL; error = bus_dma_tag_create(tag, alignment, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, size, 1, size, 0, NULL, NULL, &dma->tag); if (error != 0) goto fail; error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &dma->map); if (error != 0) goto fail; error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size, iwm_dma_map_addr, &dma->paddr, BUS_DMA_NOWAIT); if (error != 0) { bus_dmamem_free(dma->tag, dma->vaddr, dma->map); dma->vaddr = NULL; goto fail; } bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); return 0; fail: iwm_dma_contig_free(dma); return error; } void iwm_dma_contig_free(struct iwm_dma_info *dma) { if (dma->vaddr != NULL) { bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->tag, dma->map); bus_dmamem_free(dma->tag, dma->vaddr, dma->map); dma->vaddr = NULL; } if (dma->tag != NULL) { bus_dma_tag_destroy(dma->tag); dma->tag = NULL; } } +/** + * iwm_mvm_send_lq_cmd() - Send link quality command + * @init: This command is sent as part of station initialization right + * after station has been added. + * + * The link quality command is sent as the last step of station creation. + * This is the special case in which init is set and we call a callback in + * this case to clear the state indicating that station creation is in + * progress. + */ +int +iwm_mvm_send_lq_cmd(struct iwm_softc *sc, struct iwm_lq_cmd *lq, boolean_t init) +{ + struct iwm_host_cmd cmd = { + .id = IWM_LQ_CMD, + .len = { sizeof(struct iwm_lq_cmd), }, + .flags = init ? 0 : IWM_CMD_ASYNC, + .data = { lq, }, + }; + + if (lq->sta_id == IWM_MVM_STATION_COUNT) + return EINVAL; + + return iwm_send_cmd(sc, &cmd); +} + boolean_t iwm_mvm_rx_diversity_allowed(struct iwm_softc *sc) { if (num_of_ant(iwm_mvm_get_valid_rx_ant(sc)) == 1) return FALSE; /* * XXX Also return FALSE when SMPS (Spatial Multiplexing Powersave) * is used on any vap (in the future). */ return TRUE; } Index: projects/runtime-coverage/sys/dev/iwm/if_iwm_util.h =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwm_util.h (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwm_util.h (revision 321585) @@ -1,160 +1,163 @@ /* $OpenBSD: if_iwm.c,v 1.39 2015/03/23 00:35:19 jsg Exp $ */ /* $FreeBSD$ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #ifndef __IF_IWM_UTIL_H__ #define __IF_IWM_UTIL_H__ extern int iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd); extern int iwm_mvm_send_cmd_pdu(struct iwm_softc *sc, uint32_t id, uint32_t flags, uint16_t len, const void *data); extern int iwm_mvm_send_cmd_status(struct iwm_softc *sc, struct iwm_host_cmd *cmd, uint32_t *status); extern int iwm_mvm_send_cmd_pdu_status(struct iwm_softc *sc, uint32_t id, uint16_t len, const void *data, uint32_t *status); extern void iwm_free_resp(struct iwm_softc *sc, struct iwm_host_cmd *hcmd); extern int iwm_dma_contig_alloc(bus_dma_tag_t tag, struct iwm_dma_info *dma, bus_size_t size, bus_size_t alignment); extern void iwm_dma_contig_free(struct iwm_dma_info *); +extern int iwm_mvm_send_lq_cmd(struct iwm_softc *sc, struct iwm_lq_cmd *lq, + boolean_t init); + extern boolean_t iwm_mvm_rx_diversity_allowed(struct iwm_softc *sc); extern uint8_t iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx); extern int iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo); extern int iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags); static inline uint8_t iwm_mvm_get_valid_tx_ant(struct iwm_softc *sc) { return sc->nvm_data && sc->nvm_data->valid_tx_ant ? sc->sc_fw.valid_tx_ant & sc->nvm_data->valid_tx_ant : sc->sc_fw.valid_tx_ant; } static inline uint8_t iwm_mvm_get_valid_rx_ant(struct iwm_softc *sc) { return sc->nvm_data && sc->nvm_data->valid_rx_ant ? sc->sc_fw.valid_rx_ant & sc->nvm_data->valid_rx_ant : sc->sc_fw.valid_rx_ant; } static inline uint32_t iwm_mvm_get_phy_config(struct iwm_softc *sc) { uint32_t phy_config = ~(IWM_FW_PHY_CFG_TX_CHAIN | IWM_FW_PHY_CFG_RX_CHAIN); uint32_t valid_rx_ant = iwm_mvm_get_valid_rx_ant(sc); uint32_t valid_tx_ant = iwm_mvm_get_valid_tx_ant(sc); phy_config |= valid_tx_ant << IWM_FW_PHY_CFG_TX_CHAIN_POS | valid_rx_ant << IWM_FW_PHY_CFG_RX_CHAIN_POS; return sc->sc_fw.phy_config & phy_config; } #endif /* __IF_IWM_UTIL_H__ */ Index: projects/runtime-coverage/sys/dev/iwm/if_iwmreg.h =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwmreg.h (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwmreg.h (revision 321585) @@ -1,6097 +1,6144 @@ /* $OpenBSD: if_iwmreg.h,v 1.4 2015/06/15 08:06:11 stsp Exp $ */ /* $FreeBSD$ */ /****************************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * BSD LICENSE * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * *****************************************************************************/ #ifndef __IF_IWM_REG_H__ #define __IF_IWM_REG_H__ #define le16_to_cpup(_a_) (le16toh(*(const uint16_t *)(_a_))) #define le32_to_cpup(_a_) (le32toh(*(const uint32_t *)(_a_))) /* * BEGIN iwl-csr.h */ /* * CSR (control and status registers) * * CSR registers are mapped directly into PCI bus space, and are accessible * whenever platform supplies power to device, even when device is in * low power states due to driver-invoked device resets * (e.g. IWM_CSR_RESET_REG_FLAG_SW_RESET) or uCode-driven power-saving modes. * * Use iwl_write32() and iwl_read32() family to access these registers; * these provide simple PCI bus access, without waking up the MAC. * Do not use iwl_write_direct32() family for these registers; * no need to "grab nic access" via IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ. * The MAC (uCode processor, etc.) does not need to be powered up for accessing * the CSR registers. * * NOTE: Device does need to be awake in order to read this memory * via IWM_CSR_EEPROM and IWM_CSR_OTP registers */ #define IWM_CSR_HW_IF_CONFIG_REG (0x000) /* hardware interface config */ #define IWM_CSR_INT_COALESCING (0x004) /* accum ints, 32-usec units */ #define IWM_CSR_INT (0x008) /* host interrupt status/ack */ #define IWM_CSR_INT_MASK (0x00c) /* host interrupt enable */ #define IWM_CSR_FH_INT_STATUS (0x010) /* busmaster int status/ack*/ #define IWM_CSR_GPIO_IN (0x018) /* read external chip pins */ #define IWM_CSR_RESET (0x020) /* busmaster enable, NMI, etc*/ #define IWM_CSR_GP_CNTRL (0x024) /* 2nd byte of IWM_CSR_INT_COALESCING, not accessible via iwl_write32()! */ #define IWM_CSR_INT_PERIODIC_REG (0x005) /* * Hardware revision info * Bit fields: * 31-16: Reserved * 15-4: Type of device: see IWM_CSR_HW_REV_TYPE_xxx definitions * 3-2: Revision step: 0 = A, 1 = B, 2 = C, 3 = D * 1-0: "Dash" (-) value, as in A-1, etc. */ #define IWM_CSR_HW_REV (0x028) /* * EEPROM and OTP (one-time-programmable) memory reads * * NOTE: Device must be awake, initialized via apm_ops.init(), * in order to read. */ #define IWM_CSR_EEPROM_REG (0x02c) #define IWM_CSR_EEPROM_GP (0x030) #define IWM_CSR_OTP_GP_REG (0x034) #define IWM_CSR_GIO_REG (0x03C) #define IWM_CSR_GP_UCODE_REG (0x048) #define IWM_CSR_GP_DRIVER_REG (0x050) /* * UCODE-DRIVER GP (general purpose) mailbox registers. * SET/CLR registers set/clear bit(s) if "1" is written. */ #define IWM_CSR_UCODE_DRV_GP1 (0x054) #define IWM_CSR_UCODE_DRV_GP1_SET (0x058) #define IWM_CSR_UCODE_DRV_GP1_CLR (0x05c) #define IWM_CSR_UCODE_DRV_GP2 (0x060) #define IWM_CSR_MBOX_SET_REG (0x088) #define IWM_CSR_MBOX_SET_REG_OS_ALIVE 0x20 #define IWM_CSR_LED_REG (0x094) #define IWM_CSR_DRAM_INT_TBL_REG (0x0A0) #define IWM_CSR_MAC_SHADOW_REG_CTRL (0x0A8) /* 6000 and up */ /* GIO Chicken Bits (PCI Express bus link power management) */ #define IWM_CSR_GIO_CHICKEN_BITS (0x100) /* Analog phase-lock-loop configuration */ #define IWM_CSR_ANA_PLL_CFG (0x20c) /* * CSR Hardware Revision Workaround Register. Indicates hardware rev; * "step" determines CCK backoff for txpower calculation. Used for 4965 only. * See also IWM_CSR_HW_REV register. * Bit fields: * 3-2: 0 = A, 1 = B, 2 = C, 3 = D step * 1-0: "Dash" (-) value, as in C-1, etc. */ #define IWM_CSR_HW_REV_WA_REG (0x22C) #define IWM_CSR_DBG_HPET_MEM_REG (0x240) #define IWM_CSR_DBG_LINK_PWR_MGMT_REG (0x250) /* Bits for IWM_CSR_HW_IF_CONFIG_REG */ #define IWM_CSR_HW_IF_CONFIG_REG_MSK_MAC_DASH (0x00000003) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_MAC_STEP (0x0000000C) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_BOARD_VER (0x000000C0) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_MAC_SI (0x00000100) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_RADIO_SI (0x00000200) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_TYPE (0x00000C00) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_DASH (0x00003000) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_STEP (0x0000C000) #define IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH (0) #define IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP (2) #define IWM_CSR_HW_IF_CONFIG_REG_POS_BOARD_VER (6) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE (10) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH (12) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP (14) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A (0x00080000) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_EEPROM_OWN_SEM (0x00200000) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY (0x00400000) /* PCI_OWN_SEM */ #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE (0x02000000) /* ME_OWN */ #define IWM_CSR_HW_IF_CONFIG_REG_PREPARE (0x08000000) /* WAKE_ME */ #define IWM_CSR_HW_IF_CONFIG_REG_ENABLE_PME (0x10000000) #define IWM_CSR_HW_IF_CONFIG_REG_PERSIST_MODE (0x40000000) /* PERSISTENCE */ #define IWM_CSR_INT_PERIODIC_DIS (0x00) /* disable periodic int*/ #define IWM_CSR_INT_PERIODIC_ENA (0xFF) /* 255*32 usec ~ 8 msec*/ /* interrupt flags in INTA, set by uCode or hardware (e.g. dma), * acknowledged (reset) by host writing "1" to flagged bits. */ #define IWM_CSR_INT_BIT_FH_RX (1 << 31) /* Rx DMA, cmd responses, FH_INT[17:16] */ #define IWM_CSR_INT_BIT_HW_ERR (1 << 29) /* DMA hardware error FH_INT[31] */ #define IWM_CSR_INT_BIT_RX_PERIODIC (1 << 28) /* Rx periodic */ #define IWM_CSR_INT_BIT_FH_TX (1 << 27) /* Tx DMA FH_INT[1:0] */ #define IWM_CSR_INT_BIT_SCD (1 << 26) /* TXQ pointer advanced */ #define IWM_CSR_INT_BIT_SW_ERR (1 << 25) /* uCode error */ #define IWM_CSR_INT_BIT_RF_KILL (1 << 7) /* HW RFKILL switch GP_CNTRL[27] toggled */ #define IWM_CSR_INT_BIT_CT_KILL (1 << 6) /* Critical temp (chip too hot) rfkill */ #define IWM_CSR_INT_BIT_SW_RX (1 << 3) /* Rx, command responses */ #define IWM_CSR_INT_BIT_WAKEUP (1 << 1) /* NIC controller waking up (pwr mgmt) */ #define IWM_CSR_INT_BIT_ALIVE (1 << 0) /* uCode interrupts once it initializes */ #define IWM_CSR_INI_SET_MASK (IWM_CSR_INT_BIT_FH_RX | \ IWM_CSR_INT_BIT_HW_ERR | \ IWM_CSR_INT_BIT_FH_TX | \ IWM_CSR_INT_BIT_SW_ERR | \ IWM_CSR_INT_BIT_RF_KILL | \ IWM_CSR_INT_BIT_SW_RX | \ IWM_CSR_INT_BIT_WAKEUP | \ IWM_CSR_INT_BIT_ALIVE | \ IWM_CSR_INT_BIT_RX_PERIODIC) /* interrupt flags in FH (flow handler) (PCI busmaster DMA) */ #define IWM_CSR_FH_INT_BIT_ERR (1 << 31) /* Error */ #define IWM_CSR_FH_INT_BIT_HI_PRIOR (1 << 30) /* High priority Rx, bypass coalescing */ #define IWM_CSR_FH_INT_BIT_RX_CHNL1 (1 << 17) /* Rx channel 1 */ #define IWM_CSR_FH_INT_BIT_RX_CHNL0 (1 << 16) /* Rx channel 0 */ #define IWM_CSR_FH_INT_BIT_TX_CHNL1 (1 << 1) /* Tx channel 1 */ #define IWM_CSR_FH_INT_BIT_TX_CHNL0 (1 << 0) /* Tx channel 0 */ #define IWM_CSR_FH_INT_RX_MASK (IWM_CSR_FH_INT_BIT_HI_PRIOR | \ IWM_CSR_FH_INT_BIT_RX_CHNL1 | \ IWM_CSR_FH_INT_BIT_RX_CHNL0) #define IWM_CSR_FH_INT_TX_MASK (IWM_CSR_FH_INT_BIT_TX_CHNL1 | \ IWM_CSR_FH_INT_BIT_TX_CHNL0) /* GPIO */ #define IWM_CSR_GPIO_IN_BIT_AUX_POWER (0x00000200) #define IWM_CSR_GPIO_IN_VAL_VAUX_PWR_SRC (0x00000000) #define IWM_CSR_GPIO_IN_VAL_VMAIN_PWR_SRC (0x00000200) /* RESET */ #define IWM_CSR_RESET_REG_FLAG_NEVO_RESET (0x00000001) #define IWM_CSR_RESET_REG_FLAG_FORCE_NMI (0x00000002) #define IWM_CSR_RESET_REG_FLAG_SW_RESET (0x00000080) #define IWM_CSR_RESET_REG_FLAG_MASTER_DISABLED (0x00000100) #define IWM_CSR_RESET_REG_FLAG_STOP_MASTER (0x00000200) #define IWM_CSR_RESET_LINK_PWR_MGMT_DISABLED (0x80000000) /* * GP (general purpose) CONTROL REGISTER * Bit fields: * 27: HW_RF_KILL_SW * Indicates state of (platform's) hardware RF-Kill switch * 26-24: POWER_SAVE_TYPE * Indicates current power-saving mode: * 000 -- No power saving * 001 -- MAC power-down * 010 -- PHY (radio) power-down * 011 -- Error * 9-6: SYS_CONFIG * Indicates current system configuration, reflecting pins on chip * as forced high/low by device circuit board. * 4: GOING_TO_SLEEP * Indicates MAC is entering a power-saving sleep power-down. * Not a good time to access device-internal resources. * 3: MAC_ACCESS_REQ * Host sets this to request and maintain MAC wakeup, to allow host * access to device-internal resources. Host must wait for * MAC_CLOCK_READY (and !GOING_TO_SLEEP) before accessing non-CSR * device registers. * 2: INIT_DONE * Host sets this to put device into fully operational D0 power mode. * Host resets this after SW_RESET to put device into low power mode. * 0: MAC_CLOCK_READY * Indicates MAC (ucode processor, etc.) is powered up and can run. * Internal resources are accessible. * NOTE: This does not indicate that the processor is actually running. * NOTE: This does not indicate that device has completed * init or post-power-down restore of internal SRAM memory. * Use IWM_CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP as indication that * SRAM is restored and uCode is in normal operation mode. * Later devices (5xxx/6xxx/1xxx) use non-volatile SRAM, and * do not need to save/restore it. * NOTE: After device reset, this bit remains "0" until host sets * INIT_DONE */ #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY (0x00000001) #define IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE (0x00000004) #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ (0x00000008) #define IWM_CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP (0x00000010) #define IWM_CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN (0x00000001) #define IWM_CSR_GP_CNTRL_REG_MSK_POWER_SAVE_TYPE (0x07000000) #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_POWER_SAVE (0x04000000) #define IWM_CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW (0x08000000) /* HW REV */ #define IWM_CSR_HW_REV_DASH(_val) (((_val) & 0x0000003) >> 0) #define IWM_CSR_HW_REV_STEP(_val) (((_val) & 0x000000C) >> 2) /** * hw_rev values */ enum { IWM_SILICON_A_STEP = 0, IWM_SILICON_B_STEP, IWM_SILICON_C_STEP, }; #define IWM_CSR_HW_REV_TYPE_MSK (0x000FFF0) #define IWM_CSR_HW_REV_TYPE_5300 (0x0000020) #define IWM_CSR_HW_REV_TYPE_5350 (0x0000030) #define IWM_CSR_HW_REV_TYPE_5100 (0x0000050) #define IWM_CSR_HW_REV_TYPE_5150 (0x0000040) #define IWM_CSR_HW_REV_TYPE_1000 (0x0000060) #define IWM_CSR_HW_REV_TYPE_6x00 (0x0000070) #define IWM_CSR_HW_REV_TYPE_6x50 (0x0000080) #define IWM_CSR_HW_REV_TYPE_6150 (0x0000084) #define IWM_CSR_HW_REV_TYPE_6x05 (0x00000B0) #define IWM_CSR_HW_REV_TYPE_6x30 IWM_CSR_HW_REV_TYPE_6x05 #define IWM_CSR_HW_REV_TYPE_6x35 IWM_CSR_HW_REV_TYPE_6x05 #define IWM_CSR_HW_REV_TYPE_2x30 (0x00000C0) #define IWM_CSR_HW_REV_TYPE_2x00 (0x0000100) #define IWM_CSR_HW_REV_TYPE_105 (0x0000110) #define IWM_CSR_HW_REV_TYPE_135 (0x0000120) #define IWM_CSR_HW_REV_TYPE_7265D (0x0000210) #define IWM_CSR_HW_REV_TYPE_NONE (0x00001F0) /* EEPROM REG */ #define IWM_CSR_EEPROM_REG_READ_VALID_MSK (0x00000001) #define IWM_CSR_EEPROM_REG_BIT_CMD (0x00000002) #define IWM_CSR_EEPROM_REG_MSK_ADDR (0x0000FFFC) #define IWM_CSR_EEPROM_REG_MSK_DATA (0xFFFF0000) /* EEPROM GP */ #define IWM_CSR_EEPROM_GP_VALID_MSK (0x00000007) /* signature */ #define IWM_CSR_EEPROM_GP_IF_OWNER_MSK (0x00000180) #define IWM_CSR_EEPROM_GP_BAD_SIGNATURE_BOTH_EEP_AND_OTP (0x00000000) #define IWM_CSR_EEPROM_GP_BAD_SIG_EEP_GOOD_SIG_OTP (0x00000001) #define IWM_CSR_EEPROM_GP_GOOD_SIG_EEP_LESS_THAN_4K (0x00000002) #define IWM_CSR_EEPROM_GP_GOOD_SIG_EEP_MORE_THAN_4K (0x00000004) /* One-time-programmable memory general purpose reg */ #define IWM_CSR_OTP_GP_REG_DEVICE_SELECT (0x00010000) /* 0 - EEPROM, 1 - OTP */ #define IWM_CSR_OTP_GP_REG_OTP_ACCESS_MODE (0x00020000) /* 0 - absolute, 1 - relative */ #define IWM_CSR_OTP_GP_REG_ECC_CORR_STATUS_MSK (0x00100000) /* bit 20 */ #define IWM_CSR_OTP_GP_REG_ECC_UNCORR_STATUS_MSK (0x00200000) /* bit 21 */ /* GP REG */ #define IWM_CSR_GP_REG_POWER_SAVE_STATUS_MSK (0x03000000) /* bit 24/25 */ #define IWM_CSR_GP_REG_NO_POWER_SAVE (0x00000000) #define IWM_CSR_GP_REG_MAC_POWER_SAVE (0x01000000) #define IWM_CSR_GP_REG_PHY_POWER_SAVE (0x02000000) #define IWM_CSR_GP_REG_POWER_SAVE_ERROR (0x03000000) /* CSR GIO */ #define IWM_CSR_GIO_REG_VAL_L0S_ENABLED (0x00000002) /* * UCODE-DRIVER GP (general purpose) mailbox register 1 * Host driver and uCode write and/or read this register to communicate with * each other. * Bit fields: * 4: UCODE_DISABLE * Host sets this to request permanent halt of uCode, same as * sending CARD_STATE command with "halt" bit set. * 3: CT_KILL_EXIT * Host sets this to request exit from CT_KILL state, i.e. host thinks * device temperature is low enough to continue normal operation. * 2: CMD_BLOCKED * Host sets this during RF KILL power-down sequence (HW, SW, CT KILL) * to release uCode to clear all Tx and command queues, enter * unassociated mode, and power down. * NOTE: Some devices also use HBUS_TARG_MBX_C register for this bit. * 1: SW_BIT_RFKILL * Host sets this when issuing CARD_STATE command to request * device sleep. * 0: MAC_SLEEP * uCode sets this when preparing a power-saving power-down. * uCode resets this when power-up is complete and SRAM is sane. * NOTE: device saves internal SRAM data to host when powering down, * and must restore this data after powering back up. * MAC_SLEEP is the best indication that restore is complete. * Later devices (5xxx/6xxx/1xxx) use non-volatile SRAM, and * do not need to save/restore it. */ #define IWM_CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP (0x00000001) #define IWM_CSR_UCODE_SW_BIT_RFKILL (0x00000002) #define IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED (0x00000004) #define IWM_CSR_UCODE_DRV_GP1_REG_BIT_CT_KILL_EXIT (0x00000008) #define IWM_CSR_UCODE_DRV_GP1_BIT_D3_CFG_COMPLETE (0x00000020) /* GP Driver */ #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_MSK (0x00000003) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_3x3_HYB (0x00000000) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_2x2_HYB (0x00000001) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_2x2_IPA (0x00000002) #define IWM_CSR_GP_DRIVER_REG_BIT_CALIB_VERSION6 (0x00000004) #define IWM_CSR_GP_DRIVER_REG_BIT_6050_1x2 (0x00000008) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_IQ_INVER (0x00000080) /* GIO Chicken Bits (PCI Express bus link power management) */ #define IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX (0x00800000) #define IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER (0x20000000) /* LED */ #define IWM_CSR_LED_BSM_CTRL_MSK (0xFFFFFFDF) #define IWM_CSR_LED_REG_TURN_ON (0x60) #define IWM_CSR_LED_REG_TURN_OFF (0x20) /* ANA_PLL */ #define IWM_CSR50_ANA_PLL_CFG_VAL (0x00880300) /* HPET MEM debug */ #define IWM_CSR_DBG_HPET_MEM_REG_VAL (0xFFFF0000) /* DRAM INT TABLE */ #define IWM_CSR_DRAM_INT_TBL_ENABLE (1 << 31) #define IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER (1 << 28) #define IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK (1 << 27) /* SECURE boot registers */ #define IWM_CSR_SECURE_BOOT_CONFIG_ADDR (0x100) enum iwm_secure_boot_config_reg { IWM_CSR_SECURE_BOOT_CONFIG_INSPECTOR_BURNED_IN_OTP = 0x00000001, IWM_CSR_SECURE_BOOT_CONFIG_INSPECTOR_NOT_REQ = 0x00000002, }; #define IWM_CSR_SECURE_BOOT_CPU1_STATUS_ADDR (0x100) #define IWM_CSR_SECURE_BOOT_CPU2_STATUS_ADDR (0x100) enum iwm_secure_boot_status_reg { IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_STATUS = 0x00000003, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_COMPLETED = 0x00000002, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_SUCCESS = 0x00000004, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_FAIL = 0x00000008, IWM_CSR_SECURE_BOOT_CPU_STATUS_SIGN_VERF_FAIL = 0x00000010, }; #define IWM_FH_UCODE_LOAD_STATUS 0x1af0 #define IWM_FH_MEM_TB_MAX_LENGTH 0x20000 #define IWM_LMPM_SECURE_UCODE_LOAD_CPU1_HDR_ADDR 0x1e78 #define IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR 0x1e7c #define IWM_LMPM_SECURE_CPU1_HDR_MEM_SPACE 0x420000 #define IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE 0x420400 #define IWM_CSR_SECURE_TIME_OUT (100) /* extended range in FW SRAM */ #define IWM_FW_MEM_EXTENDED_START 0x40000 #define IWM_FW_MEM_EXTENDED_END 0x57FFF /* FW chicken bits */ #define IWM_LMPM_CHICK 0xa01ff8 #define IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE 0x01 #define IWM_FH_TCSR_0_REG0 (0x1D00) /* * HBUS (Host-side Bus) * * HBUS registers are mapped directly into PCI bus space, but are used * to indirectly access device's internal memory or registers that * may be powered-down. * * Use iwl_write_direct32()/iwl_read_direct32() family for these registers; * host must "grab nic access" via CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ * to make sure the MAC (uCode processor, etc.) is powered up for accessing * internal resources. * * Do not use iwl_write32()/iwl_read32() family to access these registers; * these provide only simple PCI bus access, without waking up the MAC. */ #define IWM_HBUS_BASE (0x400) /* * Registers for accessing device's internal SRAM memory (e.g. SCD SRAM * structures, error log, event log, verifying uCode load). * First write to address register, then read from or write to data register * to complete the job. Once the address register is set up, accesses to * data registers auto-increment the address by one dword. * Bit usage for address registers (read or write): * 0-31: memory address within device */ #define IWM_HBUS_TARG_MEM_RADDR (IWM_HBUS_BASE+0x00c) #define IWM_HBUS_TARG_MEM_WADDR (IWM_HBUS_BASE+0x010) #define IWM_HBUS_TARG_MEM_WDAT (IWM_HBUS_BASE+0x018) #define IWM_HBUS_TARG_MEM_RDAT (IWM_HBUS_BASE+0x01c) /* Mailbox C, used as workaround alternative to CSR_UCODE_DRV_GP1 mailbox */ #define IWM_HBUS_TARG_MBX_C (IWM_HBUS_BASE+0x030) #define IWM_HBUS_TARG_MBX_C_REG_BIT_CMD_BLOCKED (0x00000004) /* * Registers for accessing device's internal peripheral registers * (e.g. SCD, BSM, etc.). First write to address register, * then read from or write to data register to complete the job. * Bit usage for address registers (read or write): * 0-15: register address (offset) within device * 24-25: (# bytes - 1) to read or write (e.g. 3 for dword) */ #define IWM_HBUS_TARG_PRPH_WADDR (IWM_HBUS_BASE+0x044) #define IWM_HBUS_TARG_PRPH_RADDR (IWM_HBUS_BASE+0x048) #define IWM_HBUS_TARG_PRPH_WDAT (IWM_HBUS_BASE+0x04c) #define IWM_HBUS_TARG_PRPH_RDAT (IWM_HBUS_BASE+0x050) /* enable the ID buf for read */ #define IWM_WFPM_PS_CTL_CLR 0xa0300c #define IWM_WFMP_MAC_ADDR_0 0xa03080 #define IWM_WFMP_MAC_ADDR_1 0xa03084 #define IWM_LMPM_PMG_EN 0xa01cec #define IWM_RADIO_REG_SYS_MANUAL_DFT_0 0xad4078 #define IWM_RFIC_REG_RD 0xad0470 #define IWM_WFPM_CTRL_REG 0xa03030 #define IWM_WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK 0x08000000 #define IWM_ENABLE_WFPM 0x80000000 #define IWM_AUX_MISC_REG 0xa200b0 #define IWM_HW_STEP_LOCATION_BITS 24 #define IWM_AUX_MISC_MASTER1_EN 0xa20818 #define IWM_AUX_MISC_MASTER1_EN_SBE_MSK 0x1 #define IWM_AUX_MISC_MASTER1_SMPHR_STATUS 0xa20800 #define IWM_RSA_ENABLE 0xa24b08 #define IWM_PREG_AUX_BUS_WPROT_0 0xa04cc0 #define IWM_SB_CFG_OVERRIDE_ADDR 0xa26c78 #define IWM_SB_CFG_OVERRIDE_ENABLE 0x8000 #define IWM_SB_CFG_BASE_OVERRIDE 0xa20000 #define IWM_SB_MODIFY_CFG_FLAG 0xa03088 #define IWM_SB_CPU_1_STATUS 0xa01e30 #define IWM_SB_CPU_2_STATUS 0Xa01e34 /* Used to enable DBGM */ #define IWM_HBUS_TARG_TEST_REG (IWM_HBUS_BASE+0x05c) /* * Per-Tx-queue write pointer (index, really!) * Indicates index to next TFD that driver will fill (1 past latest filled). * Bit usage: * 0-7: queue write index * 11-8: queue selector */ #define IWM_HBUS_TARG_WRPTR (IWM_HBUS_BASE+0x060) /********************************************************** * CSR values **********************************************************/ /* * host interrupt timeout value * used with setting interrupt coalescing timer * the CSR_INT_COALESCING is an 8 bit register in 32-usec unit * * default interrupt coalescing timer is 64 x 32 = 2048 usecs */ #define IWM_HOST_INT_TIMEOUT_MAX (0xFF) #define IWM_HOST_INT_TIMEOUT_DEF (0x40) #define IWM_HOST_INT_TIMEOUT_MIN (0x0) #define IWM_HOST_INT_OPER_MODE (1 << 31) /***************************************************************************** * 7000/3000 series SHR DTS addresses * *****************************************************************************/ /* Diode Results Register Structure: */ enum iwm_dtd_diode_reg { IWM_DTS_DIODE_REG_DIG_VAL = 0x000000FF, /* bits [7:0] */ IWM_DTS_DIODE_REG_VREF_LOW = 0x0000FF00, /* bits [15:8] */ IWM_DTS_DIODE_REG_VREF_HIGH = 0x00FF0000, /* bits [23:16] */ IWM_DTS_DIODE_REG_VREF_ID = 0x03000000, /* bits [25:24] */ IWM_DTS_DIODE_REG_PASS_ONCE = 0x80000000, /* bits [31:31] */ IWM_DTS_DIODE_REG_FLAGS_MSK = 0xFF000000, /* bits [31:24] */ /* Those are the masks INSIDE the flags bit-field: */ IWM_DTS_DIODE_REG_FLAGS_VREFS_ID_POS = 0, IWM_DTS_DIODE_REG_FLAGS_VREFS_ID = 0x00000003, /* bits [1:0] */ IWM_DTS_DIODE_REG_FLAGS_PASS_ONCE_POS = 7, IWM_DTS_DIODE_REG_FLAGS_PASS_ONCE = 0x00000080, /* bits [7:7] */ }; /* * END iwl-csr.h */ /* * BEGIN iwl-fw.h */ /** * enum iwm_ucode_tlv_flag - ucode API flags * @IWM_UCODE_TLV_FLAGS_PAN: This is PAN capable microcode; this previously * was a separate TLV but moved here to save space. * @IWM_UCODE_TLV_FLAGS_NEWSCAN: new uCode scan behaviour on hidden SSID, * treats good CRC threshold as a boolean * @IWM_UCODE_TLV_FLAGS_MFP: This uCode image supports MFP (802.11w). * @IWM_UCODE_TLV_FLAGS_UAPSD: This uCode image supports uAPSD * @IWM_UCODE_TLV_FLAGS_SHORT_BL: 16 entries of black list instead of 64 in scan * offload profile config command. * @IWM_UCODE_TLV_FLAGS_D3_6_IPV6_ADDRS: D3 image supports up to six * (rather than two) IPv6 addresses * @IWM_UCODE_TLV_FLAGS_NO_BASIC_SSID: not sending a probe with the SSID element * from the probe request template. * @IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL: new NS offload (small version) * @IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE: new NS offload (large version) * @IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT: General support for uAPSD * @IWM_UCODE_TLV_FLAGS_EBS_SUPPORT: this uCode image supports EBS. * @IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD: P2P client supports uAPSD power save * @IWM_UCODE_TLV_FLAGS_BCAST_FILTERING: uCode supports broadcast filtering. */ enum iwm_ucode_tlv_flag { IWM_UCODE_TLV_FLAGS_PAN = (1 << 0), IWM_UCODE_TLV_FLAGS_NEWSCAN = (1 << 1), IWM_UCODE_TLV_FLAGS_MFP = (1 << 2), IWM_UCODE_TLV_FLAGS_SHORT_BL = (1 << 7), IWM_UCODE_TLV_FLAGS_D3_6_IPV6_ADDRS = (1 << 10), IWM_UCODE_TLV_FLAGS_NO_BASIC_SSID = (1 << 12), IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL = (1 << 15), IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE = (1 << 16), IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT = (1 << 24), IWM_UCODE_TLV_FLAGS_EBS_SUPPORT = (1 << 25), IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD = (1 << 26), IWM_UCODE_TLV_FLAGS_BCAST_FILTERING = (1 << 29), }; #define IWM_UCODE_TLV_FLAG_BITS \ "\020\1PAN\2NEWSCAN\3MFP\4P2P\5DW_BC_TABLE\6NEWBT_COEX\7PM_CMD\10SHORT_BL\11RX_ENERG \ Y\12TIME_EVENT_V2\13D3_6_IPV6\14BF_UPDATED\15NO_BASIC_SSID\17D3_CONTINUITY\20NEW_NSOFF \ L_S\21NEW_NSOFFL_L\22SCHED_SCAN\24STA_KEY_CMD\25DEVICE_PS_CMD\26P2P_PS\27P2P_PS_DCM\30 \ P2P_PS_SCM\31UAPSD_SUPPORT\32EBS\33P2P_PS_UAPSD\36BCAST_FILTERING\37GO_UAPSD\40LTE_COEX" /** * enum iwm_ucode_tlv_api - ucode api * @IWM_UCODE_TLV_API_FRAGMENTED_SCAN: This ucode supports active dwell time * longer than the passive one, which is essential for fragmented scan. * @IWM_UCODE_TLV_API_WIFI_MCC_UPDATE: ucode supports MCC updates with source. * @IWM_UCODE_TLV_API_LQ_SS_PARAMS: Configure STBC/BFER via LQ CMD ss_params * * @IWM_NUM_UCODE_TLV_API: number of bits used */ enum iwm_ucode_tlv_api { IWM_UCODE_TLV_API_FRAGMENTED_SCAN = 8, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE = 9, IWM_UCODE_TLV_API_LQ_SS_PARAMS = 18, IWM_NUM_UCODE_TLV_API = 32 }; #define IWM_UCODE_TLV_API_BITS \ "\020\10FRAGMENTED_SCAN\11WIFI_MCC_UPDATE\16WIDE_CMD_HDR\22LQ_SS_PARAMS\30EXT_SCAN_PRIO\33TX_POWER_CHAIN" /** * enum iwm_ucode_tlv_capa - ucode capabilities * @IWM_UCODE_TLV_CAPA_D0I3_SUPPORT: supports D0i3 * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT: supports Location Aware Regulatory * @IWM_UCODE_TLV_CAPA_UMAC_SCAN: supports UMAC scan. * @IWM_UCODE_TLV_CAPA_BEAMFORMER: supports Beamformer * @IWM_UCODE_TLV_CAPA_TOF_SUPPORT: supports Time of Flight (802.11mc FTM) * @IWM_UCODE_TLV_CAPA_TDLS_SUPPORT: support basic TDLS functionality * @IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT: supports insertion of current * tx power value into TPC Report action frame and Link Measurement Report * action frame * @IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT: supports updating current * channel in DS parameter set element in probe requests. * @IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT: supports adding TPC Report IE in * probe requests. * @IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT: supports Quiet Period requests * @IWM_UCODE_TLV_CAPA_DQA_SUPPORT: supports dynamic queue allocation (DQA), * which also implies support for the scheduler configuration command * @IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH: supports TDLS channel switching * @IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG: Consolidated D3-D0 image * @IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT: supports Hot Spot Command * @IWM_UCODE_TLV_CAPA_DC2DC_SUPPORT: supports DC2DC Command * @IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT: supports 2G coex Command * @IWM_UCODE_TLV_CAPA_CSUM_SUPPORT: supports TCP Checksum Offload * @IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS: support radio and beacon statistics * @IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD: support p2p standalone U-APSD * @IWM_UCODE_TLV_CAPA_BT_COEX_PLCR: enabled BT Coex packet level co-running * @IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC: ucode supports LAR updates with different * sources for the MCC. This TLV bit is a future replacement to * IWM_UCODE_TLV_API_WIFI_MCC_UPDATE. When either is set, multi-source LAR * is supported. * @IWM_UCODE_TLV_CAPA_BT_COEX_RRC: supports BT Coex RRC * @IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT: supports gscan * @IWM_UCODE_TLV_CAPA_NAN_SUPPORT: supports NAN * @IWM_UCODE_TLV_CAPA_UMAC_UPLOAD: supports upload mode in umac (1=supported, * 0=no support) * @IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE: extended DTS measurement * @IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS: supports short PM timeouts * @IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT: supports bt-coex Multi-priority LUT * @IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION: firmware will decide on what * antenna the beacon should be transmitted * @IWM_UCODE_TLV_CAPA_BEACON_STORING: firmware will store the latest beacon * from AP and will send it upon d0i3 exit. * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2: support LAR API V2 * @IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW: firmware responsible for CT-kill * @IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT: supports temperature * thresholds reporting * @IWM_UCODE_TLV_CAPA_CTDP_SUPPORT: supports cTDP command * @IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED: supports usniffer enabled in * regular image. * @IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG: support getting more shared * memory addresses from the firmware. * @IWM_UCODE_TLV_CAPA_LQM_SUPPORT: supports Link Quality Measurement * @IWM_UCODE_TLV_CAPA_LMAC_UPLOAD: supports upload mode in lmac (1=supported, * 0=no support) * * @IWM_NUM_UCODE_TLV_CAPA: number of bits used */ enum iwm_ucode_tlv_capa { IWM_UCODE_TLV_CAPA_D0I3_SUPPORT = 0, IWM_UCODE_TLV_CAPA_LAR_SUPPORT = 1, IWM_UCODE_TLV_CAPA_UMAC_SCAN = 2, IWM_UCODE_TLV_CAPA_BEAMFORMER = 3, IWM_UCODE_TLV_CAPA_TOF_SUPPORT = 5, IWM_UCODE_TLV_CAPA_TDLS_SUPPORT = 6, IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT = 8, IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT = 9, IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT = 10, IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT = 11, IWM_UCODE_TLV_CAPA_DQA_SUPPORT = 12, IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH = 13, IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG = 17, IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT = 18, IWM_UCODE_TLV_CAPA_DC2DC_CONFIG_SUPPORT = 19, IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT = 20, IWM_UCODE_TLV_CAPA_CSUM_SUPPORT = 21, IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS = 22, IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD = 26, IWM_UCODE_TLV_CAPA_BT_COEX_PLCR = 28, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC = 29, IWM_UCODE_TLV_CAPA_BT_COEX_RRC = 30, IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT = 31, IWM_UCODE_TLV_CAPA_NAN_SUPPORT = 34, IWM_UCODE_TLV_CAPA_UMAC_UPLOAD = 35, IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE = 64, IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS = 65, IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT = 67, IWM_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT = 68, IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION = 71, IWM_UCODE_TLV_CAPA_BEACON_STORING = 72, IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2 = 73, IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW = 74, IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT = 75, IWM_UCODE_TLV_CAPA_CTDP_SUPPORT = 76, IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED = 77, IWM_UCODE_TLV_CAPA_LMAC_UPLOAD = 79, IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG = 80, IWM_UCODE_TLV_CAPA_LQM_SUPPORT = 81, IWM_NUM_UCODE_TLV_CAPA = 128 }; /* The default calibrate table size if not specified by firmware file */ #define IWM_DEFAULT_STANDARD_PHY_CALIBRATE_TBL_SIZE 18 #define IWM_MAX_STANDARD_PHY_CALIBRATE_TBL_SIZE 19 #define IWM_MAX_PHY_CALIBRATE_TBL_SIZE 253 /* The default max probe length if not specified by the firmware file */ #define IWM_DEFAULT_MAX_PROBE_LENGTH 200 /* * enumeration of ucode section. * This enumeration is used directly for older firmware (before 16.0). * For new firmware, there can be up to 4 sections (see below) but the * first one packaged into the firmware file is the DATA section and * some debugging code accesses that. */ enum iwm_ucode_sec { IWM_UCODE_SECTION_DATA, IWM_UCODE_SECTION_INST, }; /* * For 16.0 uCode and above, there is no differentiation between sections, * just an offset to the HW address. */ #define IWM_CPU1_CPU2_SEPARATOR_SECTION 0xFFFFCCCC #define IWM_PAGING_SEPARATOR_SECTION 0xAAAABBBB /* uCode version contains 4 values: Major/Minor/API/Serial */ #define IWM_UCODE_MAJOR(ver) (((ver) & 0xFF000000) >> 24) #define IWM_UCODE_MINOR(ver) (((ver) & 0x00FF0000) >> 16) #define IWM_UCODE_API(ver) (((ver) & 0x0000FF00) >> 8) #define IWM_UCODE_SERIAL(ver) ((ver) & 0x000000FF) /* * Calibration control struct. * Sent as part of the phy configuration command. * @flow_trigger: bitmap for which calibrations to perform according to * flow triggers. * @event_trigger: bitmap for which calibrations to perform according to * event triggers. */ struct iwm_tlv_calib_ctrl { uint32_t flow_trigger; uint32_t event_trigger; } __packed; enum iwm_fw_phy_cfg { IWM_FW_PHY_CFG_RADIO_TYPE_POS = 0, IWM_FW_PHY_CFG_RADIO_TYPE = 0x3 << IWM_FW_PHY_CFG_RADIO_TYPE_POS, IWM_FW_PHY_CFG_RADIO_STEP_POS = 2, IWM_FW_PHY_CFG_RADIO_STEP = 0x3 << IWM_FW_PHY_CFG_RADIO_STEP_POS, IWM_FW_PHY_CFG_RADIO_DASH_POS = 4, IWM_FW_PHY_CFG_RADIO_DASH = 0x3 << IWM_FW_PHY_CFG_RADIO_DASH_POS, IWM_FW_PHY_CFG_TX_CHAIN_POS = 16, IWM_FW_PHY_CFG_TX_CHAIN = 0xf << IWM_FW_PHY_CFG_TX_CHAIN_POS, IWM_FW_PHY_CFG_RX_CHAIN_POS = 20, IWM_FW_PHY_CFG_RX_CHAIN = 0xf << IWM_FW_PHY_CFG_RX_CHAIN_POS, }; #define IWM_UCODE_MAX_CS 1 /** * struct iwm_fw_cipher_scheme - a cipher scheme supported by FW. * @cipher: a cipher suite selector * @flags: cipher scheme flags (currently reserved for a future use) * @hdr_len: a size of MPDU security header * @pn_len: a size of PN * @pn_off: an offset of pn from the beginning of the security header * @key_idx_off: an offset of key index byte in the security header * @key_idx_mask: a bit mask of key_idx bits * @key_idx_shift: bit shift needed to get key_idx * @mic_len: mic length in bytes * @hw_cipher: a HW cipher index used in host commands */ struct iwm_fw_cipher_scheme { uint32_t cipher; uint8_t flags; uint8_t hdr_len; uint8_t pn_len; uint8_t pn_off; uint8_t key_idx_off; uint8_t key_idx_mask; uint8_t key_idx_shift; uint8_t mic_len; uint8_t hw_cipher; } __packed; /** * struct iwm_fw_cscheme_list - a cipher scheme list * @size: a number of entries * @cs: cipher scheme entries */ struct iwm_fw_cscheme_list { uint8_t size; struct iwm_fw_cipher_scheme cs[]; } __packed; /* * END iwl-fw.h */ /* * BEGIN iwl-fw-file.h */ /* v1/v2 uCode file layout */ struct iwm_ucode_header { uint32_t ver; /* major/minor/API/serial */ union { struct { uint32_t inst_size; /* bytes of runtime code */ uint32_t data_size; /* bytes of runtime data */ uint32_t init_size; /* bytes of init code */ uint32_t init_data_size; /* bytes of init data */ uint32_t boot_size; /* bytes of bootstrap code */ uint8_t data[0]; /* in same order as sizes */ } v1; struct { uint32_t build; /* build number */ uint32_t inst_size; /* bytes of runtime code */ uint32_t data_size; /* bytes of runtime data */ uint32_t init_size; /* bytes of init code */ uint32_t init_data_size; /* bytes of init data */ uint32_t boot_size; /* bytes of bootstrap code */ uint8_t data[0]; /* in same order as sizes */ } v2; } u; }; /* * new TLV uCode file layout * * The new TLV file format contains TLVs, that each specify * some piece of data. */ enum iwm_ucode_tlv_type { IWM_UCODE_TLV_INVALID = 0, /* unused */ IWM_UCODE_TLV_INST = 1, IWM_UCODE_TLV_DATA = 2, IWM_UCODE_TLV_INIT = 3, IWM_UCODE_TLV_INIT_DATA = 4, IWM_UCODE_TLV_BOOT = 5, IWM_UCODE_TLV_PROBE_MAX_LEN = 6, /* a uint32_t value */ IWM_UCODE_TLV_PAN = 7, IWM_UCODE_TLV_RUNT_EVTLOG_PTR = 8, IWM_UCODE_TLV_RUNT_EVTLOG_SIZE = 9, IWM_UCODE_TLV_RUNT_ERRLOG_PTR = 10, IWM_UCODE_TLV_INIT_EVTLOG_PTR = 11, IWM_UCODE_TLV_INIT_EVTLOG_SIZE = 12, IWM_UCODE_TLV_INIT_ERRLOG_PTR = 13, IWM_UCODE_TLV_ENHANCE_SENS_TBL = 14, IWM_UCODE_TLV_PHY_CALIBRATION_SIZE = 15, IWM_UCODE_TLV_WOWLAN_INST = 16, IWM_UCODE_TLV_WOWLAN_DATA = 17, IWM_UCODE_TLV_FLAGS = 18, IWM_UCODE_TLV_SEC_RT = 19, IWM_UCODE_TLV_SEC_INIT = 20, IWM_UCODE_TLV_SEC_WOWLAN = 21, IWM_UCODE_TLV_DEF_CALIB = 22, IWM_UCODE_TLV_PHY_SKU = 23, IWM_UCODE_TLV_SECURE_SEC_RT = 24, IWM_UCODE_TLV_SECURE_SEC_INIT = 25, IWM_UCODE_TLV_SECURE_SEC_WOWLAN = 26, IWM_UCODE_TLV_NUM_OF_CPU = 27, IWM_UCODE_TLV_CSCHEME = 28, /* * Following two are not in our base tag, but allow * handling ucode version 9. */ IWM_UCODE_TLV_API_CHANGES_SET = 29, IWM_UCODE_TLV_ENABLED_CAPABILITIES = 30, IWM_UCODE_TLV_N_SCAN_CHANNELS = 31, IWM_UCODE_TLV_PAGING = 32, IWM_UCODE_TLV_SEC_RT_USNIFFER = 34, IWM_UCODE_TLV_SDIO_ADMA_ADDR = 35, IWM_UCODE_TLV_FW_VERSION = 36, IWM_UCODE_TLV_FW_DBG_DEST = 38, IWM_UCODE_TLV_FW_DBG_CONF = 39, IWM_UCODE_TLV_FW_DBG_TRIGGER = 40, IWM_UCODE_TLV_FW_GSCAN_CAPA = 50, IWM_UCODE_TLV_FW_MEM_SEG = 51, }; struct iwm_ucode_tlv { uint32_t type; /* see above */ uint32_t length; /* not including type/length fields */ uint8_t data[0]; }; struct iwm_ucode_api { uint32_t api_index; uint32_t api_flags; } __packed; struct iwm_ucode_capa { uint32_t api_index; uint32_t api_capa; } __packed; #define IWM_TLV_UCODE_MAGIC 0x0a4c5749 struct iwm_tlv_ucode_header { /* * The TLV style ucode header is distinguished from * the v1/v2 style header by first four bytes being * zero, as such is an invalid combination of * major/minor/API/serial versions. */ uint32_t zero; uint32_t magic; uint8_t human_readable[64]; uint32_t ver; /* major/minor/API/serial */ uint32_t build; uint64_t ignore; /* * The data contained herein has a TLV layout, * see above for the TLV header and types. * Note that each TLV is padded to a length * that is a multiple of 4 for alignment. */ uint8_t data[0]; }; /* * END iwl-fw-file.h */ /* * BEGIN iwl-prph.h */ /* * Registers in this file are internal, not PCI bus memory mapped. * Driver accesses these via IWM_HBUS_TARG_PRPH_* registers. */ #define IWM_PRPH_BASE (0x00000) #define IWM_PRPH_END (0xFFFFF) /* APMG (power management) constants */ #define IWM_APMG_BASE (IWM_PRPH_BASE + 0x3000) #define IWM_APMG_CLK_CTRL_REG (IWM_APMG_BASE + 0x0000) #define IWM_APMG_CLK_EN_REG (IWM_APMG_BASE + 0x0004) #define IWM_APMG_CLK_DIS_REG (IWM_APMG_BASE + 0x0008) #define IWM_APMG_PS_CTRL_REG (IWM_APMG_BASE + 0x000c) #define IWM_APMG_PCIDEV_STT_REG (IWM_APMG_BASE + 0x0010) #define IWM_APMG_RFKILL_REG (IWM_APMG_BASE + 0x0014) #define IWM_APMG_RTC_INT_STT_REG (IWM_APMG_BASE + 0x001c) #define IWM_APMG_RTC_INT_MSK_REG (IWM_APMG_BASE + 0x0020) #define IWM_APMG_DIGITAL_SVR_REG (IWM_APMG_BASE + 0x0058) #define IWM_APMG_ANALOG_SVR_REG (IWM_APMG_BASE + 0x006C) #define IWM_APMS_CLK_VAL_MRB_FUNC_MODE (0x00000001) #define IWM_APMG_CLK_VAL_DMA_CLK_RQT (0x00000200) #define IWM_APMG_CLK_VAL_BSM_CLK_RQT (0x00000800) #define IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS (0x00400000) #define IWM_APMG_PS_CTRL_VAL_RESET_REQ (0x04000000) #define IWM_APMG_PS_CTRL_MSK_PWR_SRC (0x03000000) #define IWM_APMG_PS_CTRL_VAL_PWR_SRC_VMAIN (0x00000000) #define IWM_APMG_PS_CTRL_VAL_PWR_SRC_VAUX (0x02000000) #define IWM_APMG_SVR_VOLTAGE_CONFIG_BIT_MSK (0x000001E0) /* bit 8:5 */ #define IWM_APMG_SVR_DIGITAL_VOLTAGE_1_32 (0x00000060) #define IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS (0x00000800) #define IWM_APMG_RTC_INT_STT_RFKILL (0x10000000) /* Device system time */ #define IWM_DEVICE_SYSTEM_TIME_REG 0xA0206C /* Device NMI register */ #define IWM_DEVICE_SET_NMI_REG 0x00a01c30 #define IWM_DEVICE_SET_NMI_VAL_HW 0x01 #define IWM_DEVICE_SET_NMI_VAL_DRV 0x80 #define IWM_DEVICE_SET_NMI_8000_REG 0x00a01c24 #define IWM_DEVICE_SET_NMI_8000_VAL 0x1000000 /* * Device reset for family 8000 * write to bit 24 in order to reset the CPU */ #define IWM_RELEASE_CPU_RESET 0x300c #define IWM_RELEASE_CPU_RESET_BIT 0x1000000 /***************************************************************************** * 7000/3000 series SHR DTS addresses * *****************************************************************************/ #define IWM_SHR_MISC_WFM_DTS_EN (0x00a10024) #define IWM_DTSC_CFG_MODE (0x00a10604) #define IWM_DTSC_VREF_AVG (0x00a10648) #define IWM_DTSC_VREF5_AVG (0x00a1064c) #define IWM_DTSC_CFG_MODE_PERIODIC (0x2) #define IWM_DTSC_PTAT_AVG (0x00a10650) /** * Tx Scheduler * * The Tx Scheduler selects the next frame to be transmitted, choosing TFDs * (Transmit Frame Descriptors) from up to 16 circular Tx queues resident in * host DRAM. It steers each frame's Tx command (which contains the frame * data) into one of up to 7 prioritized Tx DMA FIFO channels within the * device. A queue maps to only one (selectable by driver) Tx DMA channel, * but one DMA channel may take input from several queues. * * Tx DMA FIFOs have dedicated purposes. * * For 5000 series and up, they are used differently * (cf. iwl5000_default_queue_to_tx_fifo in iwl-5000.c): * * 0 -- EDCA BK (background) frames, lowest priority * 1 -- EDCA BE (best effort) frames, normal priority * 2 -- EDCA VI (video) frames, higher priority * 3 -- EDCA VO (voice) and management frames, highest priority * 4 -- unused * 5 -- unused * 6 -- unused * 7 -- Commands * * Driver should normally map queues 0-6 to Tx DMA/FIFO channels 0-6. * In addition, driver can map the remaining queues to Tx DMA/FIFO * channels 0-3 to support 11n aggregation via EDCA DMA channels. * * The driver sets up each queue to work in one of two modes: * * 1) Scheduler-Ack, in which the scheduler automatically supports a * block-ack (BA) window of up to 64 TFDs. In this mode, each queue * contains TFDs for a unique combination of Recipient Address (RA) * and Traffic Identifier (TID), that is, traffic of a given * Quality-Of-Service (QOS) priority, destined for a single station. * * In scheduler-ack mode, the scheduler keeps track of the Tx status of * each frame within the BA window, including whether it's been transmitted, * and whether it's been acknowledged by the receiving station. The device * automatically processes block-acks received from the receiving STA, * and reschedules un-acked frames to be retransmitted (successful * Tx completion may end up being out-of-order). * * The driver must maintain the queue's Byte Count table in host DRAM * for this mode. * This mode does not support fragmentation. * * 2) FIFO (a.k.a. non-Scheduler-ACK), in which each TFD is processed in order. * The device may automatically retry Tx, but will retry only one frame * at a time, until receiving ACK from receiving station, or reaching * retry limit and giving up. * * The command queue (#4/#9) must use this mode! * This mode does not require use of the Byte Count table in host DRAM. * * Driver controls scheduler operation via 3 means: * 1) Scheduler registers * 2) Shared scheduler data base in internal SRAM * 3) Shared data in host DRAM * * Initialization: * * When loading, driver should allocate memory for: * 1) 16 TFD circular buffers, each with space for (typically) 256 TFDs. * 2) 16 Byte Count circular buffers in 16 KBytes contiguous memory * (1024 bytes for each queue). * * After receiving "Alive" response from uCode, driver must initialize * the scheduler (especially for queue #4/#9, the command queue, otherwise * the driver can't issue commands!): */ #define IWM_SCD_MEM_LOWER_BOUND (0x0000) /** * Max Tx window size is the max number of contiguous TFDs that the scheduler * can keep track of at one time when creating block-ack chains of frames. * Note that "64" matches the number of ack bits in a block-ack packet. */ #define IWM_SCD_WIN_SIZE 64 #define IWM_SCD_FRAME_LIMIT 64 #define IWM_SCD_TXFIFO_POS_TID (0) #define IWM_SCD_TXFIFO_POS_RA (4) #define IWM_SCD_QUEUE_RA_TID_MAP_RATID_MSK (0x01FF) /* agn SCD */ #define IWM_SCD_QUEUE_STTS_REG_POS_TXF (0) #define IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE (3) #define IWM_SCD_QUEUE_STTS_REG_POS_WSL (4) #define IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN (19) #define IWM_SCD_QUEUE_STTS_REG_MSK (0x017F0000) #define IWM_SCD_QUEUE_CTX_REG1_CREDIT_POS (8) #define IWM_SCD_QUEUE_CTX_REG1_CREDIT_MSK (0x00FFFF00) #define IWM_SCD_QUEUE_CTX_REG1_SUPER_CREDIT_POS (24) #define IWM_SCD_QUEUE_CTX_REG1_SUPER_CREDIT_MSK (0xFF000000) #define IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS (0) #define IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK (0x0000007F) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS (16) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK (0x007F0000) #define IWM_SCD_GP_CTRL_ENABLE_31_QUEUES (1 << 0) #define IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE (1 << 18) /* Context Data */ #define IWM_SCD_CONTEXT_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x600) #define IWM_SCD_CONTEXT_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x6A0) /* Tx status */ #define IWM_SCD_TX_STTS_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x6A0) #define IWM_SCD_TX_STTS_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x7E0) /* Translation Data */ #define IWM_SCD_TRANS_TBL_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x7E0) #define IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x808) #define IWM_SCD_CONTEXT_QUEUE_OFFSET(x)\ (IWM_SCD_CONTEXT_MEM_LOWER_BOUND + ((x) * 8)) #define IWM_SCD_TX_STTS_QUEUE_OFFSET(x)\ (IWM_SCD_TX_STTS_MEM_LOWER_BOUND + ((x) * 16)) #define IWM_SCD_TRANS_TBL_OFFSET_QUEUE(x) \ ((IWM_SCD_TRANS_TBL_MEM_LOWER_BOUND + ((x) * 2)) & 0xfffc) #define IWM_SCD_BASE (IWM_PRPH_BASE + 0xa02c00) #define IWM_SCD_SRAM_BASE_ADDR (IWM_SCD_BASE + 0x0) #define IWM_SCD_DRAM_BASE_ADDR (IWM_SCD_BASE + 0x8) #define IWM_SCD_AIT (IWM_SCD_BASE + 0x0c) #define IWM_SCD_TXFACT (IWM_SCD_BASE + 0x10) #define IWM_SCD_ACTIVE (IWM_SCD_BASE + 0x14) #define IWM_SCD_QUEUECHAIN_SEL (IWM_SCD_BASE + 0xe8) #define IWM_SCD_CHAINEXT_EN (IWM_SCD_BASE + 0x244) #define IWM_SCD_AGGR_SEL (IWM_SCD_BASE + 0x248) #define IWM_SCD_INTERRUPT_MASK (IWM_SCD_BASE + 0x108) #define IWM_SCD_GP_CTRL (IWM_SCD_BASE + 0x1a8) #define IWM_SCD_EN_CTRL (IWM_SCD_BASE + 0x254) static inline unsigned int IWM_SCD_QUEUE_WRPTR(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x18 + chnl * 4; return IWM_SCD_BASE + 0x284 + (chnl - 20) * 4; } static inline unsigned int IWM_SCD_QUEUE_RDPTR(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x68 + chnl * 4; return IWM_SCD_BASE + 0x2B4 + (chnl - 20) * 4; } static inline unsigned int IWM_SCD_QUEUE_STATUS_BITS(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x10c + chnl * 4; return IWM_SCD_BASE + 0x384 + (chnl - 20) * 4; } /*********************** END TX SCHEDULER *************************************/ /* Oscillator clock */ #define IWM_OSC_CLK (0xa04068) #define IWM_OSC_CLK_FORCE_CONTROL (0x8) /* * END iwl-prph.h */ /* * BEGIN iwl-fh.h */ /****************************/ /* Flow Handler Definitions */ /****************************/ /** * This I/O area is directly read/writable by driver (e.g. Linux uses writel()) * Addresses are offsets from device's PCI hardware base address. */ #define IWM_FH_MEM_LOWER_BOUND (0x1000) #define IWM_FH_MEM_UPPER_BOUND (0x2000) /** * Keep-Warm (KW) buffer base address. * * Driver must allocate a 4KByte buffer that is for keeping the * host DRAM powered on (via dummy accesses to DRAM) to maintain low-latency * DRAM access when doing Txing or Rxing. The dummy accesses prevent host * from going into a power-savings mode that would cause higher DRAM latency, * and possible data over/under-runs, before all Tx/Rx is complete. * * Driver loads IWM_FH_KW_MEM_ADDR_REG with the physical address (bits 35:4) * of the buffer, which must be 4K aligned. Once this is set up, the device * automatically invokes keep-warm accesses when normal accesses might not * be sufficient to maintain fast DRAM response. * * Bit fields: * 31-0: Keep-warm buffer physical base address [35:4], must be 4K aligned */ #define IWM_FH_KW_MEM_ADDR_REG (IWM_FH_MEM_LOWER_BOUND + 0x97C) /** * TFD Circular Buffers Base (CBBC) addresses * * Device has 16 base pointer registers, one for each of 16 host-DRAM-resident * circular buffers (CBs/queues) containing Transmit Frame Descriptors (TFDs) * (see struct iwm_tfd_frame). These 16 pointer registers are offset by 0x04 * bytes from one another. Each TFD circular buffer in DRAM must be 256-byte * aligned (address bits 0-7 must be 0). * Later devices have 20 (5000 series) or 30 (higher) queues, but the registers * for them are in different places. * * Bit fields in each pointer register: * 27-0: TFD CB physical base address [35:8], must be 256-byte aligned */ #define IWM_FH_MEM_CBBC_0_15_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9D0) #define IWM_FH_MEM_CBBC_0_15_UPPER_BOUN (IWM_FH_MEM_LOWER_BOUND + 0xA10) #define IWM_FH_MEM_CBBC_16_19_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xBF0) #define IWM_FH_MEM_CBBC_16_19_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_CBBC_20_31_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xB20) #define IWM_FH_MEM_CBBC_20_31_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xB80) /* Find TFD CB base pointer for given queue */ static inline unsigned int IWM_FH_MEM_CBBC_QUEUE(unsigned int chnl) { if (chnl < 16) return IWM_FH_MEM_CBBC_0_15_LOWER_BOUND + 4 * chnl; if (chnl < 20) return IWM_FH_MEM_CBBC_16_19_LOWER_BOUND + 4 * (chnl - 16); return IWM_FH_MEM_CBBC_20_31_LOWER_BOUND + 4 * (chnl - 20); } /** * Rx SRAM Control and Status Registers (RSCSR) * * These registers provide handshake between driver and device for the Rx queue * (this queue handles *all* command responses, notifications, Rx data, etc. * sent from uCode to host driver). Unlike Tx, there is only one Rx * queue, and only one Rx DMA/FIFO channel. Also unlike Tx, which can * concatenate up to 20 DRAM buffers to form a Tx frame, each Receive Buffer * Descriptor (RBD) points to only one Rx Buffer (RB); there is a 1:1 * mapping between RBDs and RBs. * * Driver must allocate host DRAM memory for the following, and set the * physical address of each into device registers: * * 1) Receive Buffer Descriptor (RBD) circular buffer (CB), typically with 256 * entries (although any power of 2, up to 4096, is selectable by driver). * Each entry (1 dword) points to a receive buffer (RB) of consistent size * (typically 4K, although 8K or 16K are also selectable by driver). * Driver sets up RB size and number of RBDs in the CB via Rx config * register IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG. * * Bit fields within one RBD: * 27-0: Receive Buffer physical address bits [35:8], 256-byte aligned * * Driver sets physical address [35:8] of base of RBD circular buffer * into IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG [27:0]. * * 2) Rx status buffer, 8 bytes, in which uCode indicates which Rx Buffers * (RBs) have been filled, via a "write pointer", actually the index of * the RB's corresponding RBD within the circular buffer. Driver sets * physical address [35:4] into IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG [31:0]. * * Bit fields in lower dword of Rx status buffer (upper dword not used * by driver: * 31-12: Not used by driver * 11- 0: Index of last filled Rx buffer descriptor * (device writes, driver reads this value) * * As the driver prepares Receive Buffers (RBs) for device to fill, driver must * enter pointers to these RBs into contiguous RBD circular buffer entries, * and update the device's "write" index register, * IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG. * * This "write" index corresponds to the *next* RBD that the driver will make * available, i.e. one RBD past the tail of the ready-to-fill RBDs within * the circular buffer. This value should initially be 0 (before preparing any * RBs), should be 8 after preparing the first 8 RBs (for example), and must * wrap back to 0 at the end of the circular buffer (but don't wrap before * "read" index has advanced past 1! See below). * NOTE: DEVICE EXPECTS THE WRITE INDEX TO BE INCREMENTED IN MULTIPLES OF 8. * * As the device fills RBs (referenced from contiguous RBDs within the circular * buffer), it updates the Rx status buffer in host DRAM, 2) described above, * to tell the driver the index of the latest filled RBD. The driver must * read this "read" index from DRAM after receiving an Rx interrupt from device * * The driver must also internally keep track of a third index, which is the * next RBD to process. When receiving an Rx interrupt, driver should process * all filled but unprocessed RBs up to, but not including, the RB * corresponding to the "read" index. For example, if "read" index becomes "1", * driver may process the RB pointed to by RBD 0. Depending on volume of * traffic, there may be many RBs to process. * * If read index == write index, device thinks there is no room to put new data. * Due to this, the maximum number of filled RBs is 255, instead of 256. To * be safe, make sure that there is a gap of at least 2 RBDs between "write" * and "read" indexes; that is, make sure that there are no more than 254 * buffers waiting to be filled. */ #define IWM_FH_MEM_RSCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xBC0) #define IWM_FH_MEM_RSCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_RSCSR_CHNL0 (IWM_FH_MEM_RSCSR_LOWER_BOUND) /** * Physical base address of 8-byte Rx Status buffer. * Bit fields: * 31-0: Rx status buffer physical base address [35:4], must 16-byte aligned. */ #define IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG (IWM_FH_MEM_RSCSR_CHNL0) /** * Physical base address of Rx Buffer Descriptor Circular Buffer. * Bit fields: * 27-0: RBD CD physical base address [35:8], must be 256-byte aligned. */ #define IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x004) /** * Rx write pointer (index, really!). * Bit fields: * 11-0: Index of driver's most recent prepared-to-be-filled RBD, + 1. * NOTE: For 256-entry circular buffer, use only bits [7:0]. */ #define IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x008) #define IWM_FH_RSCSR_CHNL0_WPTR (IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG) #define IWM_FW_RSCSR_CHNL0_RXDCB_RDPTR_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x00c) #define IWM_FH_RSCSR_CHNL0_RDPTR IWM_FW_RSCSR_CHNL0_RXDCB_RDPTR_REG /** * Rx Config/Status Registers (RCSR) * Rx Config Reg for channel 0 (only channel used) * * Driver must initialize IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG as follows for * normal operation (see bit fields). * * Clearing IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG to 0 turns off Rx DMA. * Driver should poll IWM_FH_MEM_RSSR_RX_STATUS_REG for * IWM_FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE (bit 24) before continuing. * * Bit fields: * 31-30: Rx DMA channel enable: '00' off/pause, '01' pause at end of frame, * '10' operate normally * 29-24: reserved * 23-20: # RBDs in circular buffer = 2^value; use "8" for 256 RBDs (normal), * min "5" for 32 RBDs, max "12" for 4096 RBDs. * 19-18: reserved * 17-16: size of each receive buffer; '00' 4K (normal), '01' 8K, * '10' 12K, '11' 16K. * 15-14: reserved * 13-12: IRQ destination; '00' none, '01' host driver (normal operation) * 11- 4: timeout for closing Rx buffer and interrupting host (units 32 usec) * typical value 0x10 (about 1/2 msec) * 3- 0: reserved */ #define IWM_FH_MEM_RCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_RCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xCC0) #define IWM_FH_MEM_RCSR_CHNL0 (IWM_FH_MEM_RCSR_LOWER_BOUND) #define IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG (IWM_FH_MEM_RCSR_CHNL0) #define IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR (IWM_FH_MEM_RCSR_CHNL0 + 0x8) #define IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ (IWM_FH_MEM_RCSR_CHNL0 + 0x10) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RB_TIMEOUT_MSK (0x00000FF0) /* bits 4-11 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_MSK (0x00001000) /* bits 12 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK (0x00008000) /* bit 15 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RB_SIZE_MSK (0x00030000) /* bits 16-17 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RBDBC_SIZE_MSK (0x00F00000) /* bits 20-23 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_DMA_CHNL_EN_MSK (0xC0000000) /* bits 30-31*/ #define IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS (20) #define IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS (4) #define IWM_RX_RB_TIMEOUT (0x11) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_PAUSE_VAL (0x00000000) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_PAUSE_EOF_VAL (0x40000000) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL (0x80000000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K (0x00000000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K (0x00010000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_12K (0x00020000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_16K (0x00030000) #define IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY (0x00000004) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_NO_INT_VAL (0x00000000) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL (0x00001000) /** * Rx Shared Status Registers (RSSR) * * After stopping Rx DMA channel (writing 0 to * IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG), driver must poll * IWM_FH_MEM_RSSR_RX_STATUS_REG until Rx channel is idle. * * Bit fields: * 24: 1 = Channel 0 is idle * * IWM_FH_MEM_RSSR_SHARED_CTRL_REG and IWM_FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV * contain default values that should not be altered by the driver. */ #define IWM_FH_MEM_RSSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC40) #define IWM_FH_MEM_RSSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xD00) #define IWM_FH_MEM_RSSR_SHARED_CTRL_REG (IWM_FH_MEM_RSSR_LOWER_BOUND) #define IWM_FH_MEM_RSSR_RX_STATUS_REG (IWM_FH_MEM_RSSR_LOWER_BOUND + 0x004) #define IWM_FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV\ (IWM_FH_MEM_RSSR_LOWER_BOUND + 0x008) #define IWM_FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE (0x01000000) #define IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT 28 /* TFDB Area - TFDs buffer table */ #define IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK (0xFFFFFFFF) #define IWM_FH_TFDIB_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x900) #define IWM_FH_TFDIB_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x958) #define IWM_FH_TFDIB_CTRL0_REG(_chnl) (IWM_FH_TFDIB_LOWER_BOUND + 0x8 * (_chnl)) #define IWM_FH_TFDIB_CTRL1_REG(_chnl) (IWM_FH_TFDIB_LOWER_BOUND + 0x8 * (_chnl) + 0x4) /** * Transmit DMA Channel Control/Status Registers (TCSR) * * Device has one configuration register for each of 8 Tx DMA/FIFO channels * supported in hardware (don't confuse these with the 16 Tx queues in DRAM, * which feed the DMA/FIFO channels); config regs are separated by 0x20 bytes. * * To use a Tx DMA channel, driver must initialize its * IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl) with: * * IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | * IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE_VAL * * All other bits should be 0. * * Bit fields: * 31-30: Tx DMA channel enable: '00' off/pause, '01' pause at end of frame, * '10' operate normally * 29- 4: Reserved, set to "0" * 3: Enable internal DMA requests (1, normal operation), disable (0) * 2- 0: Reserved, set to "0" */ #define IWM_FH_TCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xD00) #define IWM_FH_TCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xE60) /* Find Control/Status reg for given Tx DMA/FIFO channel */ #define IWM_FH_TCSR_CHNL_NUM (8) /* TCSR: tx_config register values */ #define IWM_FH_TCSR_CHNL_TX_CONFIG_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl)) #define IWM_FH_TCSR_CHNL_TX_CREDIT_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl) + 0x4) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl) + 0x8) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_MSG_MODE_TXF (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_MSG_MODE_DRV (0x00000001) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE (0x00000008) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_NOINT (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD (0x00100000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_IFTFD (0x00200000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_NOINT (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_ENDTFD (0x00400000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_IFTFD (0x00800000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE_EOF (0x40000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE (0x80000000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_EMPTY (0x00000000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_WAIT (0x00002000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID (0x00000003) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM (20) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX (12) /** * Tx Shared Status Registers (TSSR) * * After stopping Tx DMA channel (writing 0 to * IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl)), driver must poll * IWM_FH_TSSR_TX_STATUS_REG until selected Tx channel is idle * (channel's buffers empty | no pending requests). * * Bit fields: * 31-24: 1 = Channel buffers empty (channel 7:0) * 23-16: 1 = No pending requests (channel 7:0) */ #define IWM_FH_TSSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xEA0) #define IWM_FH_TSSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xEC0) #define IWM_FH_TSSR_TX_STATUS_REG (IWM_FH_TSSR_LOWER_BOUND + 0x010) /** * Bit fields for TSSR(Tx Shared Status & Control) error status register: * 31: Indicates an address error when accessed to internal memory * uCode/driver must write "1" in order to clear this flag * 30: Indicates that Host did not send the expected number of dwords to FH * uCode/driver must write "1" in order to clear this flag * 16-9:Each status bit is for one channel. Indicates that an (Error) ActDMA * command was received from the scheduler while the TRB was already full * with previous command * uCode/driver must write "1" in order to clear this flag * 7-0: Each status bit indicates a channel's TxCredit error. When an error * bit is set, it indicates that the FH has received a full indication * from the RTC TxFIFO and the current value of the TxCredit counter was * not equal to zero. This mean that the credit mechanism was not * synchronized to the TxFIFO status * uCode/driver must write "1" in order to clear this flag */ #define IWM_FH_TSSR_TX_ERROR_REG (IWM_FH_TSSR_LOWER_BOUND + 0x018) #define IWM_FH_TSSR_TX_MSG_CONFIG_REG (IWM_FH_TSSR_LOWER_BOUND + 0x008) #define IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(_chnl) ((1 << (_chnl)) << 16) /* Tx service channels */ #define IWM_FH_SRVC_CHNL (9) #define IWM_FH_SRVC_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9C8) #define IWM_FH_SRVC_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9D0) #define IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(_chnl) \ (IWM_FH_SRVC_LOWER_BOUND + ((_chnl) - 9) * 0x4) #define IWM_FH_TX_CHICKEN_BITS_REG (IWM_FH_MEM_LOWER_BOUND + 0xE98) #define IWM_FH_TX_TRB_REG(_chan) (IWM_FH_MEM_LOWER_BOUND + 0x958 + \ (_chan) * 4) /* Instruct FH to increment the retry count of a packet when * it is brought from the memory to TX-FIFO */ #define IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN (0x00000002) #define IWM_RX_QUEUE_SIZE 256 #define IWM_RX_QUEUE_MASK 255 #define IWM_RX_QUEUE_SIZE_LOG 8 /* * RX related structures and functions */ #define IWM_RX_FREE_BUFFERS 64 #define IWM_RX_LOW_WATERMARK 8 /** * struct iwm_rb_status - reseve buffer status * host memory mapped FH registers * @closed_rb_num [0:11] - Indicates the index of the RB which was closed * @closed_fr_num [0:11] - Indicates the index of the RX Frame which was closed * @finished_rb_num [0:11] - Indicates the index of the current RB * in which the last frame was written to * @finished_fr_num [0:11] - Indicates the index of the RX Frame * which was transferred */ struct iwm_rb_status { uint16_t closed_rb_num; uint16_t closed_fr_num; uint16_t finished_rb_num; uint16_t finished_fr_nam; uint32_t unused; } __packed; #define IWM_TFD_QUEUE_SIZE_MAX (256) #define IWM_TFD_QUEUE_SIZE_BC_DUP (64) #define IWM_TFD_QUEUE_BC_SIZE (IWM_TFD_QUEUE_SIZE_MAX + \ IWM_TFD_QUEUE_SIZE_BC_DUP) #define IWM_TX_DMA_MASK DMA_BIT_MASK(36) #define IWM_NUM_OF_TBS 20 static inline uint8_t iwm_get_dma_hi_addr(bus_addr_t addr) { return (sizeof(addr) > sizeof(uint32_t) ? (addr >> 16) >> 16 : 0) & 0xF; } /** * struct iwm_tfd_tb transmit buffer descriptor within transmit frame descriptor * * This structure contains dma address and length of transmission address * * @lo: low [31:0] portion of the dma address of TX buffer * every even is unaligned on 16 bit boundary * @hi_n_len 0-3 [35:32] portion of dma * 4-15 length of the tx buffer */ struct iwm_tfd_tb { uint32_t lo; uint16_t hi_n_len; } __packed; /** * struct iwm_tfd * * Transmit Frame Descriptor (TFD) * * @ __reserved1[3] reserved * @ num_tbs 0-4 number of active tbs * 5 reserved * 6-7 padding (not used) * @ tbs[20] transmit frame buffer descriptors * @ __pad padding * * Each Tx queue uses a circular buffer of 256 TFDs stored in host DRAM. * Both driver and device share these circular buffers, each of which must be * contiguous 256 TFDs x 128 bytes-per-TFD = 32 KBytes * * Driver must indicate the physical address of the base of each * circular buffer via the IWM_FH_MEM_CBBC_QUEUE registers. * * Each TFD contains pointer/size information for up to 20 data buffers * in host DRAM. These buffers collectively contain the (one) frame described * by the TFD. Each buffer must be a single contiguous block of memory within * itself, but buffers may be scattered in host DRAM. Each buffer has max size * of (4K - 4). The concatenates all of a TFD's buffers into a single * Tx frame, up to 8 KBytes in size. * * A maximum of 255 (not 256!) TFDs may be on a queue waiting for Tx. */ struct iwm_tfd { uint8_t __reserved1[3]; uint8_t num_tbs; struct iwm_tfd_tb tbs[IWM_NUM_OF_TBS]; uint32_t __pad; } __packed; /* Keep Warm Size */ #define IWM_KW_SIZE 0x1000 /* 4k */ /* Fixed (non-configurable) rx data from phy */ /** * struct iwm_agn_schedq_bc_tbl scheduler byte count table * base physical address provided by IWM_SCD_DRAM_BASE_ADDR * @tfd_offset 0-12 - tx command byte count * 12-16 - station index */ struct iwm_agn_scd_bc_tbl { uint16_t tfd_offset[IWM_TFD_QUEUE_BC_SIZE]; } __packed; /* * END iwl-fh.h */ /* * BEGIN mvm/fw-api.h */ /* Maximum number of Tx queues. */ #define IWM_MVM_MAX_QUEUES 31 /* Tx queue numbers */ enum { IWM_MVM_OFFCHANNEL_QUEUE = 8, IWM_MVM_CMD_QUEUE = 9, IWM_MVM_AUX_QUEUE = 15, }; enum iwm_mvm_tx_fifo { IWM_MVM_TX_FIFO_BK = 0, IWM_MVM_TX_FIFO_BE, IWM_MVM_TX_FIFO_VI, IWM_MVM_TX_FIFO_VO, IWM_MVM_TX_FIFO_MCAST = 5, IWM_MVM_TX_FIFO_CMD = 7, }; #define IWM_MVM_STATION_COUNT 16 /* commands */ enum { IWM_MVM_ALIVE = 0x1, IWM_REPLY_ERROR = 0x2, IWM_INIT_COMPLETE_NOTIF = 0x4, /* PHY context commands */ IWM_PHY_CONTEXT_CMD = 0x8, IWM_DBG_CFG = 0x9, /* UMAC scan commands */ IWM_SCAN_ITERATION_COMPLETE_UMAC = 0xb5, IWM_SCAN_CFG_CMD = 0xc, IWM_SCAN_REQ_UMAC = 0xd, IWM_SCAN_ABORT_UMAC = 0xe, IWM_SCAN_COMPLETE_UMAC = 0xf, /* station table */ IWM_ADD_STA_KEY = 0x17, IWM_ADD_STA = 0x18, IWM_REMOVE_STA = 0x19, /* TX */ IWM_TX_CMD = 0x1c, IWM_TXPATH_FLUSH = 0x1e, IWM_MGMT_MCAST_KEY = 0x1f, /* scheduler config */ IWM_SCD_QUEUE_CFG = 0x1d, /* global key */ IWM_WEP_KEY = 0x20, /* MAC and Binding commands */ IWM_MAC_CONTEXT_CMD = 0x28, IWM_TIME_EVENT_CMD = 0x29, /* both CMD and response */ IWM_TIME_EVENT_NOTIFICATION = 0x2a, IWM_BINDING_CONTEXT_CMD = 0x2b, IWM_TIME_QUOTA_CMD = 0x2c, IWM_NON_QOS_TX_COUNTER_CMD = 0x2d, IWM_LQ_CMD = 0x4e, /* paging block to FW cpu2 */ IWM_FW_PAGING_BLOCK_CMD = 0x4f, /* Scan offload */ IWM_SCAN_OFFLOAD_REQUEST_CMD = 0x51, IWM_SCAN_OFFLOAD_ABORT_CMD = 0x52, IWM_HOT_SPOT_CMD = 0x53, IWM_SCAN_OFFLOAD_COMPLETE = 0x6d, IWM_SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6e, IWM_SCAN_OFFLOAD_CONFIG_CMD = 0x6f, IWM_MATCH_FOUND_NOTIFICATION = 0xd9, IWM_SCAN_ITERATION_COMPLETE = 0xe7, /* Phy */ IWM_PHY_CONFIGURATION_CMD = 0x6a, IWM_CALIB_RES_NOTIF_PHY_DB = 0x6b, IWM_PHY_DB_CMD = 0x6c, /* Power - legacy power table command */ IWM_POWER_TABLE_CMD = 0x77, IWM_PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78, /* Thermal Throttling*/ IWM_REPLY_THERMAL_MNG_BACKOFF = 0x7e, /* Scanning */ IWM_SCAN_ABORT_CMD = 0x81, IWM_SCAN_START_NOTIFICATION = 0x82, IWM_SCAN_RESULTS_NOTIFICATION = 0x83, /* NVM */ IWM_NVM_ACCESS_CMD = 0x88, IWM_SET_CALIB_DEFAULT_CMD = 0x8e, IWM_BEACON_NOTIFICATION = 0x90, IWM_BEACON_TEMPLATE_CMD = 0x91, IWM_TX_ANT_CONFIGURATION_CMD = 0x98, IWM_BT_CONFIG = 0x9b, IWM_STATISTICS_NOTIFICATION = 0x9d, IWM_REDUCE_TX_POWER_CMD = 0x9f, /* RF-KILL commands and notifications */ IWM_CARD_STATE_CMD = 0xa0, IWM_CARD_STATE_NOTIFICATION = 0xa1, IWM_MISSED_BEACONS_NOTIFICATION = 0xa2, IWM_MFUART_LOAD_NOTIFICATION = 0xb1, /* Power - new power table command */ IWM_MAC_PM_POWER_TABLE = 0xa9, IWM_REPLY_RX_PHY_CMD = 0xc0, IWM_REPLY_RX_MPDU_CMD = 0xc1, IWM_BA_NOTIF = 0xc5, /* Location Aware Regulatory */ IWM_MCC_UPDATE_CMD = 0xc8, IWM_MCC_CHUB_UPDATE_CMD = 0xc9, /* BT Coex */ IWM_BT_COEX_PRIO_TABLE = 0xcc, IWM_BT_COEX_PROT_ENV = 0xcd, IWM_BT_PROFILE_NOTIFICATION = 0xce, IWM_BT_COEX_CI = 0x5d, IWM_REPLY_SF_CFG_CMD = 0xd1, IWM_REPLY_BEACON_FILTERING_CMD = 0xd2, /* DTS measurements */ IWM_CMD_DTS_MEASUREMENT_TRIGGER = 0xdc, IWM_DTS_MEASUREMENT_NOTIFICATION = 0xdd, IWM_REPLY_DEBUG_CMD = 0xf0, IWM_DEBUG_LOG_MSG = 0xf7, IWM_MCAST_FILTER_CMD = 0xd0, /* D3 commands/notifications */ IWM_D3_CONFIG_CMD = 0xd3, IWM_PROT_OFFLOAD_CONFIG_CMD = 0xd4, IWM_OFFLOADS_QUERY_CMD = 0xd5, IWM_REMOTE_WAKE_CONFIG_CMD = 0xd6, /* for WoWLAN in particular */ IWM_WOWLAN_PATTERNS = 0xe0, IWM_WOWLAN_CONFIGURATION = 0xe1, IWM_WOWLAN_TSC_RSC_PARAM = 0xe2, IWM_WOWLAN_TKIP_PARAM = 0xe3, IWM_WOWLAN_KEK_KCK_MATERIAL = 0xe4, IWM_WOWLAN_GET_STATUSES = 0xe5, IWM_WOWLAN_TX_POWER_PER_DB = 0xe6, /* and for NetDetect */ IWM_NET_DETECT_CONFIG_CMD = 0x54, IWM_NET_DETECT_PROFILES_QUERY_CMD = 0x56, IWM_NET_DETECT_PROFILES_CMD = 0x57, IWM_NET_DETECT_HOTSPOTS_CMD = 0x58, IWM_NET_DETECT_HOTSPOTS_QUERY_CMD = 0x59, IWM_REPLY_MAX = 0xff, }; enum iwm_phy_ops_subcmd_ids { IWM_CMD_DTS_MEASUREMENT_TRIGGER_WIDE = 0x0, IWM_CTDP_CONFIG_CMD = 0x03, IWM_TEMP_REPORTING_THRESHOLDS_CMD = 0x04, IWM_CT_KILL_NOTIFICATION = 0xFE, IWM_DTS_MEASUREMENT_NOTIF_WIDE = 0xFF, }; /* command groups */ enum { IWM_LEGACY_GROUP = 0x0, IWM_LONG_GROUP = 0x1, IWM_SYSTEM_GROUP = 0x2, IWM_MAC_CONF_GROUP = 0x3, IWM_PHY_OPS_GROUP = 0x4, IWM_DATA_PATH_GROUP = 0x5, IWM_PROT_OFFLOAD_GROUP = 0xb, }; /** * struct iwm_cmd_response - generic response struct for most commands * @status: status of the command asked, changes for each one */ struct iwm_cmd_response { uint32_t status; }; /* * struct iwm_tx_ant_cfg_cmd * @valid: valid antenna configuration */ struct iwm_tx_ant_cfg_cmd { uint32_t valid; } __packed; /** * struct iwm_reduce_tx_power_cmd - TX power reduction command * IWM_REDUCE_TX_POWER_CMD = 0x9f * @flags: (reserved for future implementation) * @mac_context_id: id of the mac ctx for which we are reducing TX power. * @pwr_restriction: TX power restriction in dBms. */ struct iwm_reduce_tx_power_cmd { uint8_t flags; uint8_t mac_context_id; uint16_t pwr_restriction; } __packed; /* IWM_TX_REDUCED_POWER_API_S_VER_1 */ /* * Calibration control struct. * Sent as part of the phy configuration command. * @flow_trigger: bitmap for which calibrations to perform according to * flow triggers. * @event_trigger: bitmap for which calibrations to perform according to * event triggers. */ struct iwm_calib_ctrl { uint32_t flow_trigger; uint32_t event_trigger; } __packed; /* This enum defines the bitmap of various calibrations to enable in both * init ucode and runtime ucode through IWM_CALIBRATION_CFG_CMD. */ enum iwm_calib_cfg { IWM_CALIB_CFG_XTAL_IDX = (1 << 0), IWM_CALIB_CFG_TEMPERATURE_IDX = (1 << 1), IWM_CALIB_CFG_VOLTAGE_READ_IDX = (1 << 2), IWM_CALIB_CFG_PAPD_IDX = (1 << 3), IWM_CALIB_CFG_TX_PWR_IDX = (1 << 4), IWM_CALIB_CFG_DC_IDX = (1 << 5), IWM_CALIB_CFG_BB_FILTER_IDX = (1 << 6), IWM_CALIB_CFG_LO_LEAKAGE_IDX = (1 << 7), IWM_CALIB_CFG_TX_IQ_IDX = (1 << 8), IWM_CALIB_CFG_TX_IQ_SKEW_IDX = (1 << 9), IWM_CALIB_CFG_RX_IQ_IDX = (1 << 10), IWM_CALIB_CFG_RX_IQ_SKEW_IDX = (1 << 11), IWM_CALIB_CFG_SENSITIVITY_IDX = (1 << 12), IWM_CALIB_CFG_CHAIN_NOISE_IDX = (1 << 13), IWM_CALIB_CFG_DISCONNECTED_ANT_IDX = (1 << 14), IWM_CALIB_CFG_ANT_COUPLING_IDX = (1 << 15), IWM_CALIB_CFG_DAC_IDX = (1 << 16), IWM_CALIB_CFG_ABS_IDX = (1 << 17), IWM_CALIB_CFG_AGC_IDX = (1 << 18), }; /* * Phy configuration command. */ struct iwm_phy_cfg_cmd { uint32_t phy_cfg; struct iwm_calib_ctrl calib_control; } __packed; #define IWM_PHY_CFG_RADIO_TYPE ((1 << 0) | (1 << 1)) #define IWM_PHY_CFG_RADIO_STEP ((1 << 2) | (1 << 3)) #define IWM_PHY_CFG_RADIO_DASH ((1 << 4) | (1 << 5)) #define IWM_PHY_CFG_PRODUCT_NUMBER ((1 << 6) | (1 << 7)) #define IWM_PHY_CFG_TX_CHAIN_A (1 << 8) #define IWM_PHY_CFG_TX_CHAIN_B (1 << 9) #define IWM_PHY_CFG_TX_CHAIN_C (1 << 10) #define IWM_PHY_CFG_RX_CHAIN_A (1 << 12) #define IWM_PHY_CFG_RX_CHAIN_B (1 << 13) #define IWM_PHY_CFG_RX_CHAIN_C (1 << 14) /* Target of the IWM_NVM_ACCESS_CMD */ enum { IWM_NVM_ACCESS_TARGET_CACHE = 0, IWM_NVM_ACCESS_TARGET_OTP = 1, IWM_NVM_ACCESS_TARGET_EEPROM = 2, }; /* Section types for IWM_NVM_ACCESS_CMD */ enum { IWM_NVM_SECTION_TYPE_SW = 1, IWM_NVM_SECTION_TYPE_REGULATORY = 3, IWM_NVM_SECTION_TYPE_CALIBRATION = 4, IWM_NVM_SECTION_TYPE_PRODUCTION = 5, IWM_NVM_SECTION_TYPE_MAC_OVERRIDE = 11, IWM_NVM_SECTION_TYPE_PHY_SKU = 12, IWM_NVM_MAX_NUM_SECTIONS = 13, }; /** * struct iwm_nvm_access_cmd_ver2 - Request the device to send an NVM section * @op_code: 0 - read, 1 - write * @target: IWM_NVM_ACCESS_TARGET_* * @type: IWM_NVM_SECTION_TYPE_* * @offset: offset in bytes into the section * @length: in bytes, to read/write * @data: if write operation, the data to write. On read its empty */ struct iwm_nvm_access_cmd { uint8_t op_code; uint8_t target; uint16_t type; uint16_t offset; uint16_t length; uint8_t data[]; } __packed; /* IWM_NVM_ACCESS_CMD_API_S_VER_2 */ #define IWM_NUM_OF_FW_PAGING_BLOCKS 33 /* 32 for data and 1 block for CSS */ /* * struct iwm_fw_paging_cmd - paging layout * * (IWM_FW_PAGING_BLOCK_CMD = 0x4f) * * Send to FW the paging layout in the driver. * * @flags: various flags for the command * @block_size: the block size in powers of 2 * @block_num: number of blocks specified in the command. * @device_phy_addr: virtual addresses from device side */ struct iwm_fw_paging_cmd { uint32_t flags; uint32_t block_size; uint32_t block_num; uint32_t device_phy_addr[IWM_NUM_OF_FW_PAGING_BLOCKS]; } __packed; /* IWM_FW_PAGING_BLOCK_CMD_API_S_VER_1 */ /* * Fw items ID's * * @IWM_FW_ITEM_ID_PAGING: Address of the pages that the FW will upload * download */ enum iwm_fw_item_id { IWM_FW_ITEM_ID_PAGING = 3, }; /* * struct iwm_fw_get_item_cmd - get an item from the fw */ struct iwm_fw_get_item_cmd { uint32_t item_id; } __packed; /* IWM_FW_GET_ITEM_CMD_API_S_VER_1 */ /** * struct iwm_nvm_access_resp_ver2 - response to IWM_NVM_ACCESS_CMD * @offset: offset in bytes into the section * @length: in bytes, either how much was written or read * @type: IWM_NVM_SECTION_TYPE_* * @status: 0 for success, fail otherwise * @data: if read operation, the data returned. Empty on write. */ struct iwm_nvm_access_resp { uint16_t offset; uint16_t length; uint16_t type; uint16_t status; uint8_t data[]; } __packed; /* IWM_NVM_ACCESS_CMD_RESP_API_S_VER_2 */ /* IWM_MVM_ALIVE 0x1 */ /* alive response is_valid values */ #define IWM_ALIVE_RESP_UCODE_OK (1 << 0) #define IWM_ALIVE_RESP_RFKILL (1 << 1) /* alive response ver_type values */ enum { IWM_FW_TYPE_HW = 0, IWM_FW_TYPE_PROT = 1, IWM_FW_TYPE_AP = 2, IWM_FW_TYPE_WOWLAN = 3, IWM_FW_TYPE_TIMING = 4, IWM_FW_TYPE_WIPAN = 5 }; /* alive response ver_subtype values */ enum { IWM_FW_SUBTYPE_FULL_FEATURE = 0, IWM_FW_SUBTYPE_BOOTSRAP = 1, /* Not valid */ IWM_FW_SUBTYPE_REDUCED = 2, IWM_FW_SUBTYPE_ALIVE_ONLY = 3, IWM_FW_SUBTYPE_WOWLAN = 4, IWM_FW_SUBTYPE_AP_SUBTYPE = 5, IWM_FW_SUBTYPE_WIPAN = 6, IWM_FW_SUBTYPE_INITIALIZE = 9 }; #define IWM_ALIVE_STATUS_ERR 0xDEAD #define IWM_ALIVE_STATUS_OK 0xCAFE #define IWM_ALIVE_FLG_RFKILL (1 << 0) struct iwm_mvm_alive_resp_ver1 { uint16_t status; uint16_t flags; uint8_t ucode_minor; uint8_t ucode_major; uint16_t id; uint8_t api_minor; uint8_t api_major; uint8_t ver_subtype; uint8_t ver_type; uint8_t mac; uint8_t opt; uint16_t reserved2; uint32_t timestamp; uint32_t error_event_table_ptr; /* SRAM address for error log */ uint32_t log_event_table_ptr; /* SRAM address for event log */ uint32_t cpu_register_ptr; uint32_t dbgm_config_ptr; uint32_t alive_counter_ptr; uint32_t scd_base_ptr; /* SRAM address for SCD */ } __packed; /* IWM_ALIVE_RES_API_S_VER_1 */ struct iwm_mvm_alive_resp_ver2 { uint16_t status; uint16_t flags; uint8_t ucode_minor; uint8_t ucode_major; uint16_t id; uint8_t api_minor; uint8_t api_major; uint8_t ver_subtype; uint8_t ver_type; uint8_t mac; uint8_t opt; uint16_t reserved2; uint32_t timestamp; uint32_t error_event_table_ptr; /* SRAM address for error log */ uint32_t log_event_table_ptr; /* SRAM address for LMAC event log */ uint32_t cpu_register_ptr; uint32_t dbgm_config_ptr; uint32_t alive_counter_ptr; uint32_t scd_base_ptr; /* SRAM address for SCD */ uint32_t st_fwrd_addr; /* pointer to Store and forward */ uint32_t st_fwrd_size; uint8_t umac_minor; /* UMAC version: minor */ uint8_t umac_major; /* UMAC version: major */ uint16_t umac_id; /* UMAC version: id */ uint32_t error_info_addr; /* SRAM address for UMAC error log */ uint32_t dbg_print_buff_addr; } __packed; /* ALIVE_RES_API_S_VER_2 */ struct iwm_mvm_alive_resp { uint16_t status; uint16_t flags; uint32_t ucode_minor; uint32_t ucode_major; uint8_t ver_subtype; uint8_t ver_type; uint8_t mac; uint8_t opt; uint32_t timestamp; uint32_t error_event_table_ptr; /* SRAM address for error log */ uint32_t log_event_table_ptr; /* SRAM address for LMAC event log */ uint32_t cpu_register_ptr; uint32_t dbgm_config_ptr; uint32_t alive_counter_ptr; uint32_t scd_base_ptr; /* SRAM address for SCD */ uint32_t st_fwrd_addr; /* pointer to Store and forward */ uint32_t st_fwrd_size; uint32_t umac_minor; /* UMAC version: minor */ uint32_t umac_major; /* UMAC version: major */ uint32_t error_info_addr; /* SRAM address for UMAC error log */ uint32_t dbg_print_buff_addr; } __packed; /* ALIVE_RES_API_S_VER_3 */ /* Error response/notification */ enum { IWM_FW_ERR_UNKNOWN_CMD = 0x0, IWM_FW_ERR_INVALID_CMD_PARAM = 0x1, IWM_FW_ERR_SERVICE = 0x2, IWM_FW_ERR_ARC_MEMORY = 0x3, IWM_FW_ERR_ARC_CODE = 0x4, IWM_FW_ERR_WATCH_DOG = 0x5, IWM_FW_ERR_WEP_GRP_KEY_INDX = 0x10, IWM_FW_ERR_WEP_KEY_SIZE = 0x11, IWM_FW_ERR_OBSOLETE_FUNC = 0x12, IWM_FW_ERR_UNEXPECTED = 0xFE, IWM_FW_ERR_FATAL = 0xFF }; /** * struct iwm_error_resp - FW error indication * ( IWM_REPLY_ERROR = 0x2 ) * @error_type: one of IWM_FW_ERR_* * @cmd_id: the command ID for which the error occurred * @bad_cmd_seq_num: sequence number of the erroneous command * @error_service: which service created the error, applicable only if * error_type = 2, otherwise 0 * @timestamp: TSF in usecs. */ struct iwm_error_resp { uint32_t error_type; uint8_t cmd_id; uint8_t reserved1; uint16_t bad_cmd_seq_num; uint32_t error_service; uint64_t timestamp; } __packed; /* Common PHY, MAC and Bindings definitions */ #define IWM_MAX_MACS_IN_BINDING (3) #define IWM_MAX_BINDINGS (4) #define IWM_AUX_BINDING_INDEX (3) #define IWM_MAX_PHYS (4) /* Used to extract ID and color from the context dword */ #define IWM_FW_CTXT_ID_POS (0) #define IWM_FW_CTXT_ID_MSK (0xff << IWM_FW_CTXT_ID_POS) #define IWM_FW_CTXT_COLOR_POS (8) #define IWM_FW_CTXT_COLOR_MSK (0xff << IWM_FW_CTXT_COLOR_POS) #define IWM_FW_CTXT_INVALID (0xffffffff) #define IWM_FW_CMD_ID_AND_COLOR(_id, _color) ((_id << IWM_FW_CTXT_ID_POS) |\ (_color << IWM_FW_CTXT_COLOR_POS)) /* Possible actions on PHYs, MACs and Bindings */ enum { IWM_FW_CTXT_ACTION_STUB = 0, IWM_FW_CTXT_ACTION_ADD, IWM_FW_CTXT_ACTION_MODIFY, IWM_FW_CTXT_ACTION_REMOVE, IWM_FW_CTXT_ACTION_NUM }; /* COMMON_CONTEXT_ACTION_API_E_VER_1 */ /* Time Events */ /* Time Event types, according to MAC type */ enum iwm_time_event_type { /* BSS Station Events */ IWM_TE_BSS_STA_AGGRESSIVE_ASSOC, IWM_TE_BSS_STA_ASSOC, IWM_TE_BSS_EAP_DHCP_PROT, IWM_TE_BSS_QUIET_PERIOD, /* P2P Device Events */ IWM_TE_P2P_DEVICE_DISCOVERABLE, IWM_TE_P2P_DEVICE_LISTEN, IWM_TE_P2P_DEVICE_ACTION_SCAN, IWM_TE_P2P_DEVICE_FULL_SCAN, /* P2P Client Events */ IWM_TE_P2P_CLIENT_AGGRESSIVE_ASSOC, IWM_TE_P2P_CLIENT_ASSOC, IWM_TE_P2P_CLIENT_QUIET_PERIOD, /* P2P GO Events */ IWM_TE_P2P_GO_ASSOC_PROT, IWM_TE_P2P_GO_REPETITIVE_NOA, IWM_TE_P2P_GO_CT_WINDOW, /* WiDi Sync Events */ IWM_TE_WIDI_TX_SYNC, IWM_TE_MAX }; /* IWM_MAC_EVENT_TYPE_API_E_VER_1 */ /* Time event - defines for command API v1 */ /* * @IWM_TE_V1_FRAG_NONE: fragmentation of the time event is NOT allowed. * @IWM_TE_V1_FRAG_SINGLE: fragmentation of the time event is allowed, but only * the first fragment is scheduled. * @IWM_TE_V1_FRAG_DUAL: fragmentation of the time event is allowed, but only * the first 2 fragments are scheduled. * @IWM_TE_V1_FRAG_ENDLESS: fragmentation of the time event is allowed, and any * number of fragments are valid. * * Other than the constant defined above, specifying a fragmentation value 'x' * means that the event can be fragmented but only the first 'x' will be * scheduled. */ enum { IWM_TE_V1_FRAG_NONE = 0, IWM_TE_V1_FRAG_SINGLE = 1, IWM_TE_V1_FRAG_DUAL = 2, IWM_TE_V1_FRAG_ENDLESS = 0xffffffff }; /* If a Time Event can be fragmented, this is the max number of fragments */ #define IWM_TE_V1_FRAG_MAX_MSK 0x0fffffff /* Repeat the time event endlessly (until removed) */ #define IWM_TE_V1_REPEAT_ENDLESS 0xffffffff /* If a Time Event has bounded repetitions, this is the maximal value */ #define IWM_TE_V1_REPEAT_MAX_MSK_V1 0x0fffffff /* Time Event dependencies: none, on another TE, or in a specific time */ enum { IWM_TE_V1_INDEPENDENT = 0, IWM_TE_V1_DEP_OTHER = (1 << 0), IWM_TE_V1_DEP_TSF = (1 << 1), IWM_TE_V1_EVENT_SOCIOPATHIC = (1 << 2), }; /* IWM_MAC_EVENT_DEPENDENCY_POLICY_API_E_VER_2 */ /* * @IWM_TE_V1_NOTIF_NONE: no notifications * @IWM_TE_V1_NOTIF_HOST_EVENT_START: request/receive notification on event start * @IWM_TE_V1_NOTIF_HOST_EVENT_END:request/receive notification on event end * @IWM_TE_V1_NOTIF_INTERNAL_EVENT_START: internal FW use * @IWM_TE_V1_NOTIF_INTERNAL_EVENT_END: internal FW use. * @IWM_TE_V1_NOTIF_HOST_FRAG_START: request/receive notification on frag start * @IWM_TE_V1_NOTIF_HOST_FRAG_END:request/receive notification on frag end * @IWM_TE_V1_NOTIF_INTERNAL_FRAG_START: internal FW use. * @IWM_TE_V1_NOTIF_INTERNAL_FRAG_END: internal FW use. * * Supported Time event notifications configuration. * A notification (both event and fragment) includes a status indicating weather * the FW was able to schedule the event or not. For fragment start/end * notification the status is always success. There is no start/end fragment * notification for monolithic events. */ enum { IWM_TE_V1_NOTIF_NONE = 0, IWM_TE_V1_NOTIF_HOST_EVENT_START = (1 << 0), IWM_TE_V1_NOTIF_HOST_EVENT_END = (1 << 1), IWM_TE_V1_NOTIF_INTERNAL_EVENT_START = (1 << 2), IWM_TE_V1_NOTIF_INTERNAL_EVENT_END = (1 << 3), IWM_TE_V1_NOTIF_HOST_FRAG_START = (1 << 4), IWM_TE_V1_NOTIF_HOST_FRAG_END = (1 << 5), IWM_TE_V1_NOTIF_INTERNAL_FRAG_START = (1 << 6), IWM_TE_V1_NOTIF_INTERNAL_FRAG_END = (1 << 7), IWM_T2_V2_START_IMMEDIATELY = (1 << 11), }; /* IWM_MAC_EVENT_ACTION_API_E_VER_2 */ /* Time event - defines for command API */ /* * @IWM_TE_V2_FRAG_NONE: fragmentation of the time event is NOT allowed. * @IWM_TE_V2_FRAG_SINGLE: fragmentation of the time event is allowed, but only * the first fragment is scheduled. * @IWM_TE_V2_FRAG_DUAL: fragmentation of the time event is allowed, but only * the first 2 fragments are scheduled. * @IWM_TE_V2_FRAG_ENDLESS: fragmentation of the time event is allowed, and any * number of fragments are valid. * * Other than the constant defined above, specifying a fragmentation value 'x' * means that the event can be fragmented but only the first 'x' will be * scheduled. */ enum { IWM_TE_V2_FRAG_NONE = 0, IWM_TE_V2_FRAG_SINGLE = 1, IWM_TE_V2_FRAG_DUAL = 2, IWM_TE_V2_FRAG_MAX = 0xfe, IWM_TE_V2_FRAG_ENDLESS = 0xff }; /* Repeat the time event endlessly (until removed) */ #define IWM_TE_V2_REPEAT_ENDLESS 0xff /* If a Time Event has bounded repetitions, this is the maximal value */ #define IWM_TE_V2_REPEAT_MAX 0xfe #define IWM_TE_V2_PLACEMENT_POS 12 #define IWM_TE_V2_ABSENCE_POS 15 /* Time event policy values * A notification (both event and fragment) includes a status indicating weather * the FW was able to schedule the event or not. For fragment start/end * notification the status is always success. There is no start/end fragment * notification for monolithic events. * * @IWM_TE_V2_DEFAULT_POLICY: independent, social, present, unoticable * @IWM_TE_V2_NOTIF_HOST_EVENT_START: request/receive notification on event start * @IWM_TE_V2_NOTIF_HOST_EVENT_END:request/receive notification on event end * @IWM_TE_V2_NOTIF_INTERNAL_EVENT_START: internal FW use * @IWM_TE_V2_NOTIF_INTERNAL_EVENT_END: internal FW use. * @IWM_TE_V2_NOTIF_HOST_FRAG_START: request/receive notification on frag start * @IWM_TE_V2_NOTIF_HOST_FRAG_END:request/receive notification on frag end * @IWM_TE_V2_NOTIF_INTERNAL_FRAG_START: internal FW use. * @IWM_TE_V2_NOTIF_INTERNAL_FRAG_END: internal FW use. * @IWM_TE_V2_DEP_OTHER: depends on another time event * @IWM_TE_V2_DEP_TSF: depends on a specific time * @IWM_TE_V2_EVENT_SOCIOPATHIC: can't co-exist with other events of tha same MAC * @IWM_TE_V2_ABSENCE: are we present or absent during the Time Event. */ enum { IWM_TE_V2_DEFAULT_POLICY = 0x0, /* notifications (event start/stop, fragment start/stop) */ IWM_TE_V2_NOTIF_HOST_EVENT_START = (1 << 0), IWM_TE_V2_NOTIF_HOST_EVENT_END = (1 << 1), IWM_TE_V2_NOTIF_INTERNAL_EVENT_START = (1 << 2), IWM_TE_V2_NOTIF_INTERNAL_EVENT_END = (1 << 3), IWM_TE_V2_NOTIF_HOST_FRAG_START = (1 << 4), IWM_TE_V2_NOTIF_HOST_FRAG_END = (1 << 5), IWM_TE_V2_NOTIF_INTERNAL_FRAG_START = (1 << 6), IWM_TE_V2_NOTIF_INTERNAL_FRAG_END = (1 << 7), IWM_TE_V2_NOTIF_MSK = 0xff, /* placement characteristics */ IWM_TE_V2_DEP_OTHER = (1 << IWM_TE_V2_PLACEMENT_POS), IWM_TE_V2_DEP_TSF = (1 << (IWM_TE_V2_PLACEMENT_POS + 1)), IWM_TE_V2_EVENT_SOCIOPATHIC = (1 << (IWM_TE_V2_PLACEMENT_POS + 2)), /* are we present or absent during the Time Event. */ IWM_TE_V2_ABSENCE = (1 << IWM_TE_V2_ABSENCE_POS), }; /** * struct iwm_time_event_cmd_api - configuring Time Events * with struct IWM_MAC_TIME_EVENT_DATA_API_S_VER_2 (see also * with version 1. determined by IWM_UCODE_TLV_FLAGS) * ( IWM_TIME_EVENT_CMD = 0x29 ) * @id_and_color: ID and color of the relevant MAC * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @id: this field has two meanings, depending on the action: * If the action is ADD, then it means the type of event to add. * For all other actions it is the unique event ID assigned when the * event was added by the FW. * @apply_time: When to start the Time Event (in GP2) * @max_delay: maximum delay to event's start (apply time), in TU * @depends_on: the unique ID of the event we depend on (if any) * @interval: interval between repetitions, in TU * @duration: duration of event in TU * @repeat: how many repetitions to do, can be IWM_TE_REPEAT_ENDLESS * @max_frags: maximal number of fragments the Time Event can be divided to * @policy: defines whether uCode shall notify the host or other uCode modules * on event and/or fragment start and/or end * using one of IWM_TE_INDEPENDENT, IWM_TE_DEP_OTHER, IWM_TE_DEP_TSF * IWM_TE_EVENT_SOCIOPATHIC * using IWM_TE_ABSENCE and using IWM_TE_NOTIF_* */ struct iwm_time_event_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; uint32_t id; /* IWM_MAC_TIME_EVENT_DATA_API_S_VER_2 */ uint32_t apply_time; uint32_t max_delay; uint32_t depends_on; uint32_t interval; uint32_t duration; uint8_t repeat; uint8_t max_frags; uint16_t policy; } __packed; /* IWM_MAC_TIME_EVENT_CMD_API_S_VER_2 */ /** * struct iwm_time_event_resp - response structure to iwm_time_event_cmd * @status: bit 0 indicates success, all others specify errors * @id: the Time Event type * @unique_id: the unique ID assigned (in ADD) or given (others) to the TE * @id_and_color: ID and color of the relevant MAC */ struct iwm_time_event_resp { uint32_t status; uint32_t id; uint32_t unique_id; uint32_t id_and_color; } __packed; /* IWM_MAC_TIME_EVENT_RSP_API_S_VER_1 */ /** * struct iwm_time_event_notif - notifications of time event start/stop * ( IWM_TIME_EVENT_NOTIFICATION = 0x2a ) * @timestamp: action timestamp in GP2 * @session_id: session's unique id * @unique_id: unique id of the Time Event itself * @id_and_color: ID and color of the relevant MAC * @action: one of IWM_TE_NOTIF_START or IWM_TE_NOTIF_END * @status: true if scheduled, false otherwise (not executed) */ struct iwm_time_event_notif { uint32_t timestamp; uint32_t session_id; uint32_t unique_id; uint32_t id_and_color; uint32_t action; uint32_t status; } __packed; /* IWM_MAC_TIME_EVENT_NTFY_API_S_VER_1 */ /* Bindings and Time Quota */ /** * struct iwm_binding_cmd - configuring bindings * ( IWM_BINDING_CONTEXT_CMD = 0x2b ) * @id_and_color: ID and color of the relevant Binding * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @macs: array of MAC id and colors which belong to the binding * @phy: PHY id and color which belongs to the binding */ struct iwm_binding_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_BINDING_DATA_API_S_VER_1 */ uint32_t macs[IWM_MAX_MACS_IN_BINDING]; uint32_t phy; } __packed; /* IWM_BINDING_CMD_API_S_VER_1 */ /* The maximal number of fragments in the FW's schedule session */ #define IWM_MVM_MAX_QUOTA 128 /** * struct iwm_time_quota_data - configuration of time quota per binding * @id_and_color: ID and color of the relevant Binding * @quota: absolute time quota in TU. The scheduler will try to divide the * remainig quota (after Time Events) according to this quota. * @max_duration: max uninterrupted context duration in TU */ struct iwm_time_quota_data { uint32_t id_and_color; uint32_t quota; uint32_t max_duration; } __packed; /* IWM_TIME_QUOTA_DATA_API_S_VER_1 */ /** * struct iwm_time_quota_cmd - configuration of time quota between bindings * ( IWM_TIME_QUOTA_CMD = 0x2c ) * @quotas: allocations per binding */ struct iwm_time_quota_cmd { struct iwm_time_quota_data quotas[IWM_MAX_BINDINGS]; } __packed; /* IWM_TIME_QUOTA_ALLOCATION_CMD_API_S_VER_1 */ /* PHY context */ /* Supported bands */ #define IWM_PHY_BAND_5 (0) #define IWM_PHY_BAND_24 (1) /* Supported channel width, vary if there is VHT support */ #define IWM_PHY_VHT_CHANNEL_MODE20 (0x0) #define IWM_PHY_VHT_CHANNEL_MODE40 (0x1) #define IWM_PHY_VHT_CHANNEL_MODE80 (0x2) #define IWM_PHY_VHT_CHANNEL_MODE160 (0x3) /* * Control channel position: * For legacy set bit means upper channel, otherwise lower. * For VHT - bit-2 marks if the control is lower/upper relative to center-freq * bits-1:0 mark the distance from the center freq. for 20Mhz, offset is 0. * center_freq * | * 40Mhz |_______|_______| * 80Mhz |_______|_______|_______|_______| * 160Mhz |_______|_______|_______|_______|_______|_______|_______|_______| * code 011 010 001 000 | 100 101 110 111 */ #define IWM_PHY_VHT_CTRL_POS_1_BELOW (0x0) #define IWM_PHY_VHT_CTRL_POS_2_BELOW (0x1) #define IWM_PHY_VHT_CTRL_POS_3_BELOW (0x2) #define IWM_PHY_VHT_CTRL_POS_4_BELOW (0x3) #define IWM_PHY_VHT_CTRL_POS_1_ABOVE (0x4) #define IWM_PHY_VHT_CTRL_POS_2_ABOVE (0x5) #define IWM_PHY_VHT_CTRL_POS_3_ABOVE (0x6) #define IWM_PHY_VHT_CTRL_POS_4_ABOVE (0x7) /* * @band: IWM_PHY_BAND_* * @channel: channel number * @width: PHY_[VHT|LEGACY]_CHANNEL_* * @ctrl channel: PHY_[VHT|LEGACY]_CTRL_* */ struct iwm_fw_channel_info { uint8_t band; uint8_t channel; uint8_t width; uint8_t ctrl_pos; } __packed; #define IWM_PHY_RX_CHAIN_DRIVER_FORCE_POS (0) #define IWM_PHY_RX_CHAIN_DRIVER_FORCE_MSK \ (0x1 << IWM_PHY_RX_CHAIN_DRIVER_FORCE_POS) #define IWM_PHY_RX_CHAIN_VALID_POS (1) #define IWM_PHY_RX_CHAIN_VALID_MSK \ (0x7 << IWM_PHY_RX_CHAIN_VALID_POS) #define IWM_PHY_RX_CHAIN_FORCE_SEL_POS (4) #define IWM_PHY_RX_CHAIN_FORCE_SEL_MSK \ (0x7 << IWM_PHY_RX_CHAIN_FORCE_SEL_POS) #define IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS (7) #define IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_MSK \ (0x7 << IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS) #define IWM_PHY_RX_CHAIN_CNT_POS (10) #define IWM_PHY_RX_CHAIN_CNT_MSK \ (0x3 << IWM_PHY_RX_CHAIN_CNT_POS) #define IWM_PHY_RX_CHAIN_MIMO_CNT_POS (12) #define IWM_PHY_RX_CHAIN_MIMO_CNT_MSK \ (0x3 << IWM_PHY_RX_CHAIN_MIMO_CNT_POS) #define IWM_PHY_RX_CHAIN_MIMO_FORCE_POS (14) #define IWM_PHY_RX_CHAIN_MIMO_FORCE_MSK \ (0x1 << IWM_PHY_RX_CHAIN_MIMO_FORCE_POS) /* TODO: fix the value, make it depend on firmware at runtime? */ #define IWM_NUM_PHY_CTX 3 /* TODO: complete missing documentation */ /** * struct iwm_phy_context_cmd - config of the PHY context * ( IWM_PHY_CONTEXT_CMD = 0x8 ) * @id_and_color: ID and color of the relevant Binding * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @apply_time: 0 means immediate apply and context switch. * other value means apply new params after X usecs * @tx_param_color: ??? * @channel_info: * @txchain_info: ??? * @rxchain_info: ??? * @acquisition_data: ??? * @dsp_cfg_flags: set to 0 */ struct iwm_phy_context_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_PHY_CONTEXT_DATA_API_S_VER_1 */ uint32_t apply_time; uint32_t tx_param_color; struct iwm_fw_channel_info ci; uint32_t txchain_info; uint32_t rxchain_info; uint32_t acquisition_data; uint32_t dsp_cfg_flags; } __packed; /* IWM_PHY_CONTEXT_CMD_API_VER_1 */ #define IWM_RX_INFO_PHY_CNT 8 #define IWM_RX_INFO_ENERGY_ANT_ABC_IDX 1 #define IWM_RX_INFO_ENERGY_ANT_A_MSK 0x000000ff #define IWM_RX_INFO_ENERGY_ANT_B_MSK 0x0000ff00 #define IWM_RX_INFO_ENERGY_ANT_C_MSK 0x00ff0000 #define IWM_RX_INFO_ENERGY_ANT_A_POS 0 #define IWM_RX_INFO_ENERGY_ANT_B_POS 8 #define IWM_RX_INFO_ENERGY_ANT_C_POS 16 #define IWM_RX_INFO_AGC_IDX 1 #define IWM_RX_INFO_RSSI_AB_IDX 2 #define IWM_OFDM_AGC_A_MSK 0x0000007f #define IWM_OFDM_AGC_A_POS 0 #define IWM_OFDM_AGC_B_MSK 0x00003f80 #define IWM_OFDM_AGC_B_POS 7 #define IWM_OFDM_AGC_CODE_MSK 0x3fe00000 #define IWM_OFDM_AGC_CODE_POS 20 #define IWM_OFDM_RSSI_INBAND_A_MSK 0x00ff #define IWM_OFDM_RSSI_A_POS 0 #define IWM_OFDM_RSSI_ALLBAND_A_MSK 0xff00 #define IWM_OFDM_RSSI_ALLBAND_A_POS 8 #define IWM_OFDM_RSSI_INBAND_B_MSK 0xff0000 #define IWM_OFDM_RSSI_B_POS 16 #define IWM_OFDM_RSSI_ALLBAND_B_MSK 0xff000000 #define IWM_OFDM_RSSI_ALLBAND_B_POS 24 /** * struct iwm_rx_phy_info - phy info * (IWM_REPLY_RX_PHY_CMD = 0xc0) * @non_cfg_phy_cnt: non configurable DSP phy data byte count * @cfg_phy_cnt: configurable DSP phy data byte count * @stat_id: configurable DSP phy data set ID * @reserved1: * @system_timestamp: GP2 at on air rise * @timestamp: TSF at on air rise * @beacon_time_stamp: beacon at on-air rise * @phy_flags: general phy flags: band, modulation, ... * @channel: channel number * @non_cfg_phy_buf: for various implementations of non_cfg_phy * @rate_n_flags: IWM_RATE_MCS_* * @byte_count: frame's byte-count * @frame_time: frame's time on the air, based on byte count and frame rate * calculation * @mac_active_msk: what MACs were active when the frame was received * * Before each Rx, the device sends this data. It contains PHY information * about the reception of the packet. */ struct iwm_rx_phy_info { uint8_t non_cfg_phy_cnt; uint8_t cfg_phy_cnt; uint8_t stat_id; uint8_t reserved1; uint32_t system_timestamp; uint64_t timestamp; uint32_t beacon_time_stamp; uint16_t phy_flags; #define IWM_PHY_INFO_FLAG_SHPREAMBLE (1 << 2) uint16_t channel; uint32_t non_cfg_phy[IWM_RX_INFO_PHY_CNT]; uint8_t rate; uint8_t rflags; uint16_t xrflags; uint32_t byte_count; uint16_t mac_active_msk; uint16_t frame_time; } __packed; struct iwm_rx_mpdu_res_start { uint16_t byte_count; uint16_t reserved; } __packed; /** * enum iwm_rx_phy_flags - to parse %iwm_rx_phy_info phy_flags * @IWM_RX_RES_PHY_FLAGS_BAND_24: true if the packet was received on 2.4 band * @IWM_RX_RES_PHY_FLAGS_MOD_CCK: * @IWM_RX_RES_PHY_FLAGS_SHORT_PREAMBLE: true if packet's preamble was short * @IWM_RX_RES_PHY_FLAGS_NARROW_BAND: * @IWM_RX_RES_PHY_FLAGS_ANTENNA: antenna on which the packet was received * @IWM_RX_RES_PHY_FLAGS_AGG: set if the packet was part of an A-MPDU * @IWM_RX_RES_PHY_FLAGS_OFDM_HT: The frame was an HT frame * @IWM_RX_RES_PHY_FLAGS_OFDM_GF: The frame used GF preamble * @IWM_RX_RES_PHY_FLAGS_OFDM_VHT: The frame was a VHT frame */ enum iwm_rx_phy_flags { IWM_RX_RES_PHY_FLAGS_BAND_24 = (1 << 0), IWM_RX_RES_PHY_FLAGS_MOD_CCK = (1 << 1), IWM_RX_RES_PHY_FLAGS_SHORT_PREAMBLE = (1 << 2), IWM_RX_RES_PHY_FLAGS_NARROW_BAND = (1 << 3), IWM_RX_RES_PHY_FLAGS_ANTENNA = (0x7 << 4), IWM_RX_RES_PHY_FLAGS_ANTENNA_POS = 4, IWM_RX_RES_PHY_FLAGS_AGG = (1 << 7), IWM_RX_RES_PHY_FLAGS_OFDM_HT = (1 << 8), IWM_RX_RES_PHY_FLAGS_OFDM_GF = (1 << 9), IWM_RX_RES_PHY_FLAGS_OFDM_VHT = (1 << 10), }; /** * enum iwm_mvm_rx_status - written by fw for each Rx packet * @IWM_RX_MPDU_RES_STATUS_CRC_OK: CRC is fine * @IWM_RX_MPDU_RES_STATUS_OVERRUN_OK: there was no RXE overflow * @IWM_RX_MPDU_RES_STATUS_SRC_STA_FOUND: * @IWM_RX_MPDU_RES_STATUS_KEY_VALID: * @IWM_RX_MPDU_RES_STATUS_KEY_PARAM_OK: * @IWM_RX_MPDU_RES_STATUS_ICV_OK: ICV is fine, if not, the packet is destroyed * @IWM_RX_MPDU_RES_STATUS_MIC_OK: used for CCM alg only. TKIP MIC is checked * in the driver. * @IWM_RX_MPDU_RES_STATUS_TTAK_OK: TTAK is fine * @IWM_RX_MPDU_RES_STATUS_MNG_FRAME_REPLAY_ERR: valid for alg = CCM_CMAC or * alg = CCM only. Checks replay attack for 11w frames. Relevant only if * %IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME is set. * @IWM_RX_MPDU_RES_STATUS_SEC_NO_ENC: this frame is not encrypted * @IWM_RX_MPDU_RES_STATUS_SEC_WEP_ENC: this frame is encrypted using WEP * @IWM_RX_MPDU_RES_STATUS_SEC_CCM_ENC: this frame is encrypted using CCM * @IWM_RX_MPDU_RES_STATUS_SEC_TKIP_ENC: this frame is encrypted using TKIP * @IWM_RX_MPDU_RES_STATUS_SEC_CCM_CMAC_ENC: this frame is encrypted using CCM_CMAC * @IWM_RX_MPDU_RES_STATUS_SEC_ENC_ERR: this frame couldn't be decrypted * @IWM_RX_MPDU_RES_STATUS_SEC_ENC_MSK: bitmask of the encryption algorithm * @IWM_RX_MPDU_RES_STATUS_DEC_DONE: this frame has been successfully decrypted * @IWM_RX_MPDU_RES_STATUS_PROTECT_FRAME_BIT_CMP: * @IWM_RX_MPDU_RES_STATUS_EXT_IV_BIT_CMP: * @IWM_RX_MPDU_RES_STATUS_KEY_ID_CMP_BIT: * @IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME: this frame is an 11w management frame * @IWM_RX_MPDU_RES_STATUS_HASH_INDEX_MSK: * @IWM_RX_MPDU_RES_STATUS_STA_ID_MSK: * @IWM_RX_MPDU_RES_STATUS_RRF_KILL: * @IWM_RX_MPDU_RES_STATUS_FILTERING_MSK: * @IWM_RX_MPDU_RES_STATUS2_FILTERING_MSK: */ enum iwm_mvm_rx_status { IWM_RX_MPDU_RES_STATUS_CRC_OK = (1 << 0), IWM_RX_MPDU_RES_STATUS_OVERRUN_OK = (1 << 1), IWM_RX_MPDU_RES_STATUS_SRC_STA_FOUND = (1 << 2), IWM_RX_MPDU_RES_STATUS_KEY_VALID = (1 << 3), IWM_RX_MPDU_RES_STATUS_KEY_PARAM_OK = (1 << 4), IWM_RX_MPDU_RES_STATUS_ICV_OK = (1 << 5), IWM_RX_MPDU_RES_STATUS_MIC_OK = (1 << 6), IWM_RX_MPDU_RES_STATUS_TTAK_OK = (1 << 7), IWM_RX_MPDU_RES_STATUS_MNG_FRAME_REPLAY_ERR = (1 << 7), IWM_RX_MPDU_RES_STATUS_SEC_NO_ENC = (0 << 8), IWM_RX_MPDU_RES_STATUS_SEC_WEP_ENC = (1 << 8), IWM_RX_MPDU_RES_STATUS_SEC_CCM_ENC = (2 << 8), IWM_RX_MPDU_RES_STATUS_SEC_TKIP_ENC = (3 << 8), IWM_RX_MPDU_RES_STATUS_SEC_EXT_ENC = (4 << 8), IWM_RX_MPDU_RES_STATUS_SEC_CCM_CMAC_ENC = (6 << 8), IWM_RX_MPDU_RES_STATUS_SEC_ENC_ERR = (7 << 8), IWM_RX_MPDU_RES_STATUS_SEC_ENC_MSK = (7 << 8), IWM_RX_MPDU_RES_STATUS_DEC_DONE = (1 << 11), IWM_RX_MPDU_RES_STATUS_PROTECT_FRAME_BIT_CMP = (1 << 12), IWM_RX_MPDU_RES_STATUS_EXT_IV_BIT_CMP = (1 << 13), IWM_RX_MPDU_RES_STATUS_KEY_ID_CMP_BIT = (1 << 14), IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME = (1 << 15), IWM_RX_MPDU_RES_STATUS_HASH_INDEX_MSK = (0x3F0000), IWM_RX_MPDU_RES_STATUS_STA_ID_MSK = (0x1f000000), IWM_RX_MPDU_RES_STATUS_RRF_KILL = (1 << 29), IWM_RX_MPDU_RES_STATUS_FILTERING_MSK = (0xc00000), IWM_RX_MPDU_RES_STATUS2_FILTERING_MSK = (0xc0000000), }; /** * struct iwm_radio_version_notif - information on the radio version * ( IWM_RADIO_VERSION_NOTIFICATION = 0x68 ) * @radio_flavor: * @radio_step: * @radio_dash: */ struct iwm_radio_version_notif { uint32_t radio_flavor; uint32_t radio_step; uint32_t radio_dash; } __packed; /* IWM_RADIO_VERSION_NOTOFICATION_S_VER_1 */ enum iwm_card_state_flags { IWM_CARD_ENABLED = 0x00, IWM_HW_CARD_DISABLED = 0x01, IWM_SW_CARD_DISABLED = 0x02, IWM_CT_KILL_CARD_DISABLED = 0x04, IWM_HALT_CARD_DISABLED = 0x08, IWM_CARD_DISABLED_MSK = 0x0f, IWM_CARD_IS_RX_ON = 0x10, }; /** * struct iwm_radio_version_notif - information on the radio version * (IWM_CARD_STATE_NOTIFICATION = 0xa1 ) * @flags: %iwm_card_state_flags */ struct iwm_card_state_notif { uint32_t flags; } __packed; /* CARD_STATE_NTFY_API_S_VER_1 */ /** * struct iwm_missed_beacons_notif - information on missed beacons * ( IWM_MISSED_BEACONS_NOTIFICATION = 0xa2 ) * @mac_id: interface ID * @consec_missed_beacons_since_last_rx: number of consecutive missed * beacons since last RX. * @consec_missed_beacons: number of consecutive missed beacons * @num_expected_beacons: * @num_recvd_beacons: */ struct iwm_missed_beacons_notif { uint32_t mac_id; uint32_t consec_missed_beacons_since_last_rx; uint32_t consec_missed_beacons; uint32_t num_expected_beacons; uint32_t num_recvd_beacons; } __packed; /* IWM_MISSED_BEACON_NTFY_API_S_VER_3 */ /** * struct iwm_mfuart_load_notif - mfuart image version & status * ( IWM_MFUART_LOAD_NOTIFICATION = 0xb1 ) * @installed_ver: installed image version * @external_ver: external image version * @status: MFUART loading status * @duration: MFUART loading time */ struct iwm_mfuart_load_notif { uint32_t installed_ver; uint32_t external_ver; uint32_t status; uint32_t duration; } __packed; /*MFU_LOADER_NTFY_API_S_VER_1*/ /** * struct iwm_set_calib_default_cmd - set default value for calibration. * ( IWM_SET_CALIB_DEFAULT_CMD = 0x8e ) * @calib_index: the calibration to set value for * @length: of data * @data: the value to set for the calibration result */ struct iwm_set_calib_default_cmd { uint16_t calib_index; uint16_t length; uint8_t data[0]; } __packed; /* IWM_PHY_CALIB_OVERRIDE_VALUES_S */ #define IWM_MAX_PORT_ID_NUM 2 #define IWM_MAX_MCAST_FILTERING_ADDRESSES 256 /** * struct iwm_mcast_filter_cmd - configure multicast filter. * @filter_own: Set 1 to filter out multicast packets sent by station itself * @port_id: Multicast MAC addresses array specifier. This is a strange way * to identify network interface adopted in host-device IF. * It is used by FW as index in array of addresses. This array has * IWM_MAX_PORT_ID_NUM members. * @count: Number of MAC addresses in the array * @pass_all: Set 1 to pass all multicast packets. * @bssid: current association BSSID. * @addr_list: Place holder for array of MAC addresses. * IMPORTANT: add padding if necessary to ensure DWORD alignment. */ struct iwm_mcast_filter_cmd { uint8_t filter_own; uint8_t port_id; uint8_t count; uint8_t pass_all; uint8_t bssid[6]; uint8_t reserved[2]; uint8_t addr_list[0]; } __packed; /* IWM_MCAST_FILTERING_CMD_API_S_VER_1 */ +/* + * The first MAC indices (starting from 0) + * are available to the driver, AUX follows + */ +#define IWM_MAC_INDEX_AUX 4 +#define IWM_MAC_INDEX_MIN_DRIVER 0 +#define IWM_NUM_MAC_INDEX_DRIVER IWM_MAC_INDEX_AUX +#define IWM_NUM_MAC_INDEX (IWM_MAC_INDEX_AUX + 1) + +/*********************************** + * Statistics API + ***********************************/ struct iwm_mvm_statistics_dbg { uint32_t burst_check; uint32_t burst_count; uint32_t wait_for_silence_timeout_cnt; uint32_t reserved[3]; } __packed; /* IWM_STATISTICS_DEBUG_API_S_VER_2 */ struct iwm_mvm_statistics_div { uint32_t tx_on_a; uint32_t tx_on_b; uint32_t exec_time; uint32_t probe_time; uint32_t rssi_ant; uint32_t reserved2; } __packed; /* IWM_STATISTICS_SLOW_DIV_API_S_VER_2 */ -struct iwm_mvm_statistics_general_common { - uint32_t temperature; /* radio temperature */ - uint32_t temperature_m; /* radio voltage */ - struct iwm_mvm_statistics_dbg dbg; - uint32_t sleep_time; - uint32_t slots_out; - uint32_t slots_idle; - uint32_t ttl_timestamp; - struct iwm_mvm_statistics_div div; - uint32_t rx_enable_counter; - /* - * num_of_sos_states: - * count the number of times we have to re-tune - * in order to get out of bad PHY status - */ - uint32_t num_of_sos_states; -} __packed; /* IWM_STATISTICS_GENERAL_API_S_VER_5 */ - struct iwm_mvm_statistics_rx_non_phy { uint32_t bogus_cts; /* CTS received when not expecting CTS */ uint32_t bogus_ack; /* ACK received when not expecting ACK */ uint32_t non_bssid_frames; /* number of frames with BSSID that * doesn't belong to the STA BSSID */ uint32_t filtered_frames; /* count frames that were dumped in the * filtering process */ uint32_t non_channel_beacons; /* beacons with our bss id but not on * our serving channel */ uint32_t channel_beacons; /* beacons with our bss id and in our * serving channel */ uint32_t num_missed_bcon; /* number of missed beacons */ uint32_t adc_rx_saturation_time; /* count in 0.8us units the time the * ADC was in saturation */ uint32_t ina_detection_search_time;/* total time (in 0.8us) searched * for INA */ uint32_t beacon_silence_rssi[3];/* RSSI silence after beacon frame */ uint32_t interference_data_flag; /* flag for interference data * availability. 1 when data is * available. */ uint32_t channel_load; /* counts RX Enable time in uSec */ uint32_t dsp_false_alarms; /* DSP false alarm (both OFDM * and CCK) counter */ uint32_t beacon_rssi_a; uint32_t beacon_rssi_b; uint32_t beacon_rssi_c; uint32_t beacon_energy_a; uint32_t beacon_energy_b; uint32_t beacon_energy_c; uint32_t num_bt_kills; uint32_t mac_id; uint32_t directed_data_mpdu; } __packed; /* IWM_STATISTICS_RX_NON_PHY_API_S_VER_3 */ struct iwm_mvm_statistics_rx_phy { uint32_t ina_cnt; uint32_t fina_cnt; uint32_t plcp_err; uint32_t crc32_err; uint32_t overrun_err; uint32_t early_overrun_err; uint32_t crc32_good; uint32_t false_alarm_cnt; uint32_t fina_sync_err_cnt; uint32_t sfd_timeout; uint32_t fina_timeout; uint32_t unresponded_rts; uint32_t rxe_frame_limit_overrun; uint32_t sent_ack_cnt; uint32_t sent_cts_cnt; uint32_t sent_ba_rsp_cnt; uint32_t dsp_self_kill; uint32_t mh_format_err; uint32_t re_acq_main_rssi_sum; uint32_t reserved; } __packed; /* IWM_STATISTICS_RX_PHY_API_S_VER_2 */ struct iwm_mvm_statistics_rx_ht_phy { uint32_t plcp_err; uint32_t overrun_err; uint32_t early_overrun_err; uint32_t crc32_good; uint32_t crc32_err; uint32_t mh_format_err; uint32_t agg_crc32_good; uint32_t agg_mpdu_cnt; uint32_t agg_cnt; uint32_t unsupport_mcs; } __packed; /* IWM_STATISTICS_HT_RX_PHY_API_S_VER_1 */ +struct iwm_mvm_statistics_tx_non_phy { + uint32_t preamble_cnt; + uint32_t rx_detected_cnt; + uint32_t bt_prio_defer_cnt; + uint32_t bt_prio_kill_cnt; + uint32_t few_bytes_cnt; + uint32_t cts_timeout; + uint32_t ack_timeout; + uint32_t expected_ack_cnt; + uint32_t actual_ack_cnt; + uint32_t dump_msdu_cnt; + uint32_t burst_abort_next_frame_mismatch_cnt; + uint32_t burst_abort_missing_next_frame_cnt; + uint32_t cts_timeout_collision; + uint32_t ack_or_ba_timeout_collision; +} __packed; /* IWM_STATISTICS_TX_NON_PHY_API_S_VER_3 */ + #define IWM_MAX_CHAINS 3 struct iwm_mvm_statistics_tx_non_phy_agg { uint32_t ba_timeout; uint32_t ba_reschedule_frames; uint32_t scd_query_agg_frame_cnt; uint32_t scd_query_no_agg; uint32_t scd_query_agg; uint32_t scd_query_mismatch; uint32_t frame_not_ready; uint32_t underrun; uint32_t bt_prio_kill; uint32_t rx_ba_rsp_cnt; int8_t txpower[IWM_MAX_CHAINS]; int8_t reserved; uint32_t reserved2; } __packed; /* IWM_STATISTICS_TX_NON_PHY_AGG_API_S_VER_1 */ struct iwm_mvm_statistics_tx_channel_width { uint32_t ext_cca_narrow_ch20[1]; uint32_t ext_cca_narrow_ch40[2]; uint32_t ext_cca_narrow_ch80[3]; uint32_t ext_cca_narrow_ch160[4]; uint32_t last_tx_ch_width_indx; uint32_t rx_detected_per_ch_width[4]; uint32_t success_per_ch_width[4]; uint32_t fail_per_ch_width[4]; }; /* IWM_STATISTICS_TX_CHANNEL_WIDTH_API_S_VER_1 */ struct iwm_mvm_statistics_tx { - uint32_t preamble_cnt; - uint32_t rx_detected_cnt; - uint32_t bt_prio_defer_cnt; - uint32_t bt_prio_kill_cnt; - uint32_t few_bytes_cnt; - uint32_t cts_timeout; - uint32_t ack_timeout; - uint32_t expected_ack_cnt; - uint32_t actual_ack_cnt; - uint32_t dump_msdu_cnt; - uint32_t burst_abort_next_frame_mismatch_cnt; - uint32_t burst_abort_missing_next_frame_cnt; - uint32_t cts_timeout_collision; - uint32_t ack_or_ba_timeout_collision; + struct iwm_mvm_statistics_tx_non_phy general; struct iwm_mvm_statistics_tx_non_phy_agg agg; struct iwm_mvm_statistics_tx_channel_width channel_width; } __packed; /* IWM_STATISTICS_TX_API_S_VER_4 */ struct iwm_mvm_statistics_bt_activity { uint32_t hi_priority_tx_req_cnt; uint32_t hi_priority_tx_denied_cnt; uint32_t lo_priority_tx_req_cnt; uint32_t lo_priority_tx_denied_cnt; uint32_t hi_priority_rx_req_cnt; uint32_t hi_priority_rx_denied_cnt; uint32_t lo_priority_rx_req_cnt; uint32_t lo_priority_rx_denied_cnt; } __packed; /* IWM_STATISTICS_BT_ACTIVITY_API_S_VER_1 */ -struct iwm_mvm_statistics_general { - struct iwm_mvm_statistics_general_common common; +struct iwm_mvm_statistics_general_v8 { + uint32_t radio_temperature; + uint32_t radio_voltage; + struct iwm_mvm_statistics_dbg dbg; + uint32_t sleep_time; + uint32_t slots_out; + uint32_t slots_idle; + uint32_t ttl_timestamp; + struct iwm_mvm_statistics_div slow_div; + uint32_t rx_enable_counter; + /* + * num_of_sos_states: + * count the number of times we have to re-tune + * in order to get out of bad PHY status + */ + uint32_t num_of_sos_states; uint32_t beacon_filtered; uint32_t missed_beacons; - int8_t beacon_filter_average_energy; - int8_t beacon_filter_reason; - int8_t beacon_filter_current_energy; - int8_t beacon_filter_reserved; + uint8_t beacon_filter_average_energy; + uint8_t beacon_filter_reason; + uint8_t beacon_filter_current_energy; + uint8_t beacon_filter_reserved; uint32_t beacon_filter_delta_time; struct iwm_mvm_statistics_bt_activity bt_activity; -} __packed; /* IWM_STATISTICS_GENERAL_API_S_VER_5 */ + uint64_t rx_time; + uint64_t on_time_rf; + uint64_t on_time_scan; + uint64_t tx_time; + uint32_t beacon_counter[IWM_NUM_MAC_INDEX]; + uint8_t beacon_average_energy[IWM_NUM_MAC_INDEX]; + uint8_t reserved[4 - (IWM_NUM_MAC_INDEX % 4)]; +} __packed; /* IWM_STATISTICS_GENERAL_API_S_VER_8 */ struct iwm_mvm_statistics_rx { struct iwm_mvm_statistics_rx_phy ofdm; struct iwm_mvm_statistics_rx_phy cck; struct iwm_mvm_statistics_rx_non_phy general; struct iwm_mvm_statistics_rx_ht_phy ofdm_ht; } __packed; /* IWM_STATISTICS_RX_API_S_VER_3 */ /* * IWM_STATISTICS_NOTIFICATION = 0x9d (notification only, not a command) * * By default, uCode issues this notification after receiving a beacon * while associated. To disable this behavior, set DISABLE_NOTIF flag in the - * IWM_REPLY_STATISTICS_CMD 0x9c, above. - * - * Statistics counters continue to increment beacon after beacon, but are - * cleared when changing channels or when driver issues IWM_REPLY_STATISTICS_CMD - * 0x9c with CLEAR_STATS bit set (see above). - * - * uCode also issues this notification during scans. uCode clears statistics - * appropriately so that each notification contains statistics for only the - * one channel that has just been scanned. + * IWM_STATISTICS_CMD (0x9c), below. */ -struct iwm_notif_statistics { /* IWM_STATISTICS_NTFY_API_S_VER_8 */ +struct iwm_notif_statistics_v10 { uint32_t flag; struct iwm_mvm_statistics_rx rx; struct iwm_mvm_statistics_tx tx; - struct iwm_mvm_statistics_general general; -} __packed; + struct iwm_mvm_statistics_general_v8 general; +} __packed; /* IWM_STATISTICS_NTFY_API_S_VER_10 */ +#define IWM_STATISTICS_FLG_CLEAR 0x1 +#define IWM_STATISTICS_FLG_DISABLE_NOTIF 0x2 + +struct iwm_statistics_cmd { + uint32_t flags; +} __packed; /* IWM_STATISTICS_CMD_API_S_VER_1 */ + /*********************************** * Smart Fifo API ***********************************/ /* Smart Fifo state */ enum iwm_sf_state { IWM_SF_LONG_DELAY_ON = 0, /* should never be called by driver */ IWM_SF_FULL_ON, IWM_SF_UNINIT, IWM_SF_INIT_OFF, IWM_SF_HW_NUM_STATES }; /* Smart Fifo possible scenario */ enum iwm_sf_scenario { IWM_SF_SCENARIO_SINGLE_UNICAST, IWM_SF_SCENARIO_AGG_UNICAST, IWM_SF_SCENARIO_MULTICAST, IWM_SF_SCENARIO_BA_RESP, IWM_SF_SCENARIO_TX_RESP, IWM_SF_NUM_SCENARIO }; #define IWM_SF_TRANSIENT_STATES_NUMBER 2 /* IWM_SF_LONG_DELAY_ON and IWM_SF_FULL_ON */ #define IWM_SF_NUM_TIMEOUT_TYPES 2 /* Aging timer and Idle timer */ /* smart FIFO default values */ #define IWM_SF_W_MARK_SISO 4096 #define IWM_SF_W_MARK_MIMO2 8192 #define IWM_SF_W_MARK_MIMO3 6144 #define IWM_SF_W_MARK_LEGACY 4096 #define IWM_SF_W_MARK_SCAN 4096 /* SF Scenarios timers for default configuration (aligned to 32 uSec) */ #define IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_AGG_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_MCAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_MCAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_BA_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_BA_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_TX_RE_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_TX_RE_AGING_TIMER_DEF 400 /* 0.4 mSec */ /* SF Scenarios timers for FULL_ON state (aligned to 32 uSec) */ #define IWM_SF_SINGLE_UNICAST_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_SINGLE_UNICAST_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_AGG_UNICAST_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_AGG_UNICAST_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_MCAST_IDLE_TIMER 2016 /* 2 mSec */ #define IWM_SF_MCAST_AGING_TIMER 10016 /* 10 mSec */ #define IWM_SF_BA_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_BA_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_TX_RE_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_TX_RE_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_LONG_DELAY_AGING_TIMER 1000000 /* 1 Sec */ #define IWM_SF_CFG_DUMMY_NOTIF_OFF (1 << 16) /** * Smart Fifo configuration command. * @state: smart fifo state, types listed in iwm_sf_state. * @watermark: Minimum allowed available free space in RXF for transient state. * @long_delay_timeouts: aging and idle timer values for each scenario * in long delay state. * @full_on_timeouts: timer values for each scenario in full on state. */ struct iwm_sf_cfg_cmd { uint32_t state; uint32_t watermark[IWM_SF_TRANSIENT_STATES_NUMBER]; uint32_t long_delay_timeouts[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES]; uint32_t full_on_timeouts[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES]; } __packed; /* IWM_SF_CFG_API_S_VER_2 */ /* * END mvm/fw-api.h */ /* * BEGIN mvm/fw-api-mac.h */ -/* - * The first MAC indices (starting from 0) - * are available to the driver, AUX follows - */ -#define IWM_MAC_INDEX_AUX 4 -#define IWM_MAC_INDEX_MIN_DRIVER 0 -#define IWM_NUM_MAC_INDEX_DRIVER IWM_MAC_INDEX_AUX - enum iwm_ac { IWM_AC_BK, IWM_AC_BE, IWM_AC_VI, IWM_AC_VO, IWM_AC_NUM, }; /** * enum iwm_mac_protection_flags - MAC context flags * @IWM_MAC_PROT_FLG_TGG_PROTECT: 11g protection when transmitting OFDM frames, * this will require CCK RTS/CTS2self. * RTS/CTS will protect full burst time. * @IWM_MAC_PROT_FLG_HT_PROT: enable HT protection * @IWM_MAC_PROT_FLG_FAT_PROT: protect 40 MHz transmissions * @IWM_MAC_PROT_FLG_SELF_CTS_EN: allow CTS2self */ enum iwm_mac_protection_flags { IWM_MAC_PROT_FLG_TGG_PROTECT = (1 << 3), IWM_MAC_PROT_FLG_HT_PROT = (1 << 23), IWM_MAC_PROT_FLG_FAT_PROT = (1 << 24), IWM_MAC_PROT_FLG_SELF_CTS_EN = (1 << 30), }; #define IWM_MAC_FLG_SHORT_SLOT (1 << 4) #define IWM_MAC_FLG_SHORT_PREAMBLE (1 << 5) /** * enum iwm_mac_types - Supported MAC types * @IWM_FW_MAC_TYPE_FIRST: lowest supported MAC type * @IWM_FW_MAC_TYPE_AUX: Auxiliary MAC (internal) * @IWM_FW_MAC_TYPE_LISTENER: monitor MAC type (?) * @IWM_FW_MAC_TYPE_PIBSS: Pseudo-IBSS * @IWM_FW_MAC_TYPE_IBSS: IBSS * @IWM_FW_MAC_TYPE_BSS_STA: BSS (managed) station * @IWM_FW_MAC_TYPE_P2P_DEVICE: P2P Device * @IWM_FW_MAC_TYPE_P2P_STA: P2P client * @IWM_FW_MAC_TYPE_GO: P2P GO * @IWM_FW_MAC_TYPE_TEST: ? * @IWM_FW_MAC_TYPE_MAX: highest support MAC type */ enum iwm_mac_types { IWM_FW_MAC_TYPE_FIRST = 1, IWM_FW_MAC_TYPE_AUX = IWM_FW_MAC_TYPE_FIRST, IWM_FW_MAC_TYPE_LISTENER, IWM_FW_MAC_TYPE_PIBSS, IWM_FW_MAC_TYPE_IBSS, IWM_FW_MAC_TYPE_BSS_STA, IWM_FW_MAC_TYPE_P2P_DEVICE, IWM_FW_MAC_TYPE_P2P_STA, IWM_FW_MAC_TYPE_GO, IWM_FW_MAC_TYPE_TEST, IWM_FW_MAC_TYPE_MAX = IWM_FW_MAC_TYPE_TEST }; /* IWM_MAC_CONTEXT_TYPE_API_E_VER_1 */ /** * enum iwm_tsf_id - TSF hw timer ID * @IWM_TSF_ID_A: use TSF A * @IWM_TSF_ID_B: use TSF B * @IWM_TSF_ID_C: use TSF C * @IWM_TSF_ID_D: use TSF D * @IWM_NUM_TSF_IDS: number of TSF timers available */ enum iwm_tsf_id { IWM_TSF_ID_A = 0, IWM_TSF_ID_B = 1, IWM_TSF_ID_C = 2, IWM_TSF_ID_D = 3, IWM_NUM_TSF_IDS = 4, }; /* IWM_TSF_ID_API_E_VER_1 */ /** * struct iwm_mac_data_ap - configuration data for AP MAC context * @beacon_time: beacon transmit time in system time * @beacon_tsf: beacon transmit time in TSF * @bi: beacon interval in TU * @bi_reciprocal: 2^32 / bi * @dtim_interval: dtim transmit time in TU * @dtim_reciprocal: 2^32 / dtim_interval * @mcast_qid: queue ID for multicast traffic * @beacon_template: beacon template ID */ struct iwm_mac_data_ap { uint32_t beacon_time; uint64_t beacon_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t dtim_interval; uint32_t dtim_reciprocal; uint32_t mcast_qid; uint32_t beacon_template; } __packed; /* AP_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_ibss - configuration data for IBSS MAC context * @beacon_time: beacon transmit time in system time * @beacon_tsf: beacon transmit time in TSF * @bi: beacon interval in TU * @bi_reciprocal: 2^32 / bi * @beacon_template: beacon template ID */ struct iwm_mac_data_ibss { uint32_t beacon_time; uint64_t beacon_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t beacon_template; } __packed; /* IBSS_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_sta - configuration data for station MAC context * @is_assoc: 1 for associated state, 0 otherwise * @dtim_time: DTIM arrival time in system time * @dtim_tsf: DTIM arrival time in TSF * @bi: beacon interval in TU, applicable only when associated * @bi_reciprocal: 2^32 / bi , applicable only when associated * @dtim_interval: DTIM interval in TU, applicable only when associated * @dtim_reciprocal: 2^32 / dtim_interval , applicable only when associated * @listen_interval: in beacon intervals, applicable only when associated * @assoc_id: unique ID assigned by the AP during association */ struct iwm_mac_data_sta { uint32_t is_assoc; uint32_t dtim_time; uint64_t dtim_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t dtim_interval; uint32_t dtim_reciprocal; uint32_t listen_interval; uint32_t assoc_id; uint32_t assoc_beacon_arrive_time; } __packed; /* IWM_STA_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_go - configuration data for P2P GO MAC context * @ap: iwm_mac_data_ap struct with most config data * @ctwin: client traffic window in TU (period after TBTT when GO is present). * 0 indicates that there is no CT window. * @opp_ps_enabled: indicate that opportunistic PS allowed */ struct iwm_mac_data_go { struct iwm_mac_data_ap ap; uint32_t ctwin; uint32_t opp_ps_enabled; } __packed; /* GO_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_p2p_sta - configuration data for P2P client MAC context * @sta: iwm_mac_data_sta struct with most config data * @ctwin: client traffic window in TU (period after TBTT when GO is present). * 0 indicates that there is no CT window. */ struct iwm_mac_data_p2p_sta { struct iwm_mac_data_sta sta; uint32_t ctwin; } __packed; /* P2P_STA_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_pibss - Pseudo IBSS config data * @stats_interval: interval in TU between statistics notifications to host. */ struct iwm_mac_data_pibss { uint32_t stats_interval; } __packed; /* PIBSS_MAC_DATA_API_S_VER_1 */ /* * struct iwm_mac_data_p2p_dev - configuration data for the P2P Device MAC * context. * @is_disc_extended: if set to true, P2P Device discoverability is enabled on * other channels as well. This should be to true only in case that the * device is discoverable and there is an active GO. Note that setting this * field when not needed, will increase the number of interrupts and have * effect on the platform power, as this setting opens the Rx filters on * all macs. */ struct iwm_mac_data_p2p_dev { uint32_t is_disc_extended; } __packed; /* _P2P_DEV_MAC_DATA_API_S_VER_1 */ /** * enum iwm_mac_filter_flags - MAC context filter flags * @IWM_MAC_FILTER_IN_PROMISC: accept all data frames * @IWM_MAC_FILTER_IN_CONTROL_AND_MGMT: pass all mangement and * control frames to the host * @IWM_MAC_FILTER_ACCEPT_GRP: accept multicast frames * @IWM_MAC_FILTER_DIS_DECRYPT: don't decrypt unicast frames * @IWM_MAC_FILTER_DIS_GRP_DECRYPT: don't decrypt multicast frames * @IWM_MAC_FILTER_IN_BEACON: transfer foreign BSS's beacons to host * (in station mode when associated) * @IWM_MAC_FILTER_OUT_BCAST: filter out all broadcast frames * @IWM_MAC_FILTER_IN_CRC32: extract FCS and append it to frames * @IWM_MAC_FILTER_IN_PROBE_REQUEST: pass probe requests to host */ enum iwm_mac_filter_flags { IWM_MAC_FILTER_IN_PROMISC = (1 << 0), IWM_MAC_FILTER_IN_CONTROL_AND_MGMT = (1 << 1), IWM_MAC_FILTER_ACCEPT_GRP = (1 << 2), IWM_MAC_FILTER_DIS_DECRYPT = (1 << 3), IWM_MAC_FILTER_DIS_GRP_DECRYPT = (1 << 4), IWM_MAC_FILTER_IN_BEACON = (1 << 6), IWM_MAC_FILTER_OUT_BCAST = (1 << 8), IWM_MAC_FILTER_IN_CRC32 = (1 << 11), IWM_MAC_FILTER_IN_PROBE_REQUEST = (1 << 12), }; /** * enum iwm_mac_qos_flags - QoS flags * @IWM_MAC_QOS_FLG_UPDATE_EDCA: ? * @IWM_MAC_QOS_FLG_TGN: HT is enabled * @IWM_MAC_QOS_FLG_TXOP_TYPE: ? * */ enum iwm_mac_qos_flags { IWM_MAC_QOS_FLG_UPDATE_EDCA = (1 << 0), IWM_MAC_QOS_FLG_TGN = (1 << 1), IWM_MAC_QOS_FLG_TXOP_TYPE = (1 << 4), }; /** * struct iwm_ac_qos - QOS timing params for IWM_MAC_CONTEXT_CMD * @cw_min: Contention window, start value in numbers of slots. * Should be a power-of-2, minus 1. Device's default is 0x0f. * @cw_max: Contention window, max value in numbers of slots. * Should be a power-of-2, minus 1. Device's default is 0x3f. * @aifsn: Number of slots in Arbitration Interframe Space (before * performing random backoff timing prior to Tx). Device default 1. * @fifos_mask: FIFOs used by this MAC for this AC * @edca_txop: Length of Tx opportunity, in uSecs. Device default is 0. * * One instance of this config struct for each of 4 EDCA access categories * in struct iwm_qosparam_cmd. * * Device will automatically increase contention window by (2*CW) + 1 for each * transmission retry. Device uses cw_max as a bit mask, ANDed with new CW * value, to cap the CW value. */ struct iwm_ac_qos { uint16_t cw_min; uint16_t cw_max; uint8_t aifsn; uint8_t fifos_mask; uint16_t edca_txop; } __packed; /* IWM_AC_QOS_API_S_VER_2 */ /** * struct iwm_mac_ctx_cmd - command structure to configure MAC contexts * ( IWM_MAC_CONTEXT_CMD = 0x28 ) * @id_and_color: ID and color of the MAC * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @mac_type: one of IWM_FW_MAC_TYPE_* * @tsd_id: TSF HW timer, one of IWM_TSF_ID_* * @node_addr: MAC address * @bssid_addr: BSSID * @cck_rates: basic rates available for CCK * @ofdm_rates: basic rates available for OFDM * @protection_flags: combination of IWM_MAC_PROT_FLG_FLAG_* * @cck_short_preamble: 0x20 for enabling short preamble, 0 otherwise * @short_slot: 0x10 for enabling short slots, 0 otherwise * @filter_flags: combination of IWM_MAC_FILTER_* * @qos_flags: from IWM_MAC_QOS_FLG_* * @ac: one iwm_mac_qos configuration for each AC * @mac_specific: one of struct iwm_mac_data_*, according to mac_type */ struct iwm_mac_ctx_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_MAC_CONTEXT_COMMON_DATA_API_S_VER_1 */ uint32_t mac_type; uint32_t tsf_id; uint8_t node_addr[6]; uint16_t reserved_for_node_addr; uint8_t bssid_addr[6]; uint16_t reserved_for_bssid_addr; uint32_t cck_rates; uint32_t ofdm_rates; uint32_t protection_flags; uint32_t cck_short_preamble; uint32_t short_slot; uint32_t filter_flags; /* IWM_MAC_QOS_PARAM_API_S_VER_1 */ uint32_t qos_flags; struct iwm_ac_qos ac[IWM_AC_NUM+1]; /* IWM_MAC_CONTEXT_COMMON_DATA_API_S */ union { struct iwm_mac_data_ap ap; struct iwm_mac_data_go go; struct iwm_mac_data_sta sta; struct iwm_mac_data_p2p_sta p2p_sta; struct iwm_mac_data_p2p_dev p2p_dev; struct iwm_mac_data_pibss pibss; struct iwm_mac_data_ibss ibss; }; } __packed; /* IWM_MAC_CONTEXT_CMD_API_S_VER_1 */ static inline uint32_t iwm_mvm_reciprocal(uint32_t v) { if (!v) return 0; return 0xFFFFFFFF / v; } #define IWM_NONQOS_SEQ_GET 0x1 #define IWM_NONQOS_SEQ_SET 0x2 struct iwm_nonqos_seq_query_cmd { uint32_t get_set_flag; uint32_t mac_id_n_color; uint16_t value; uint16_t reserved; } __packed; /* IWM_NON_QOS_TX_COUNTER_GET_SET_API_S_VER_1 */ /* * END mvm/fw-api-mac.h */ /* * BEGIN mvm/fw-api-power.h */ /* Power Management Commands, Responses, Notifications */ /* Radio LP RX Energy Threshold measured in dBm */ #define IWM_POWER_LPRX_RSSI_THRESHOLD 75 #define IWM_POWER_LPRX_RSSI_THRESHOLD_MAX 94 #define IWM_POWER_LPRX_RSSI_THRESHOLD_MIN 30 /** * enum iwm_scan_flags - masks for power table command flags * @IWM_POWER_FLAGS_POWER_SAVE_ENA_MSK: '1' Allow to save power by turning off * receiver and transmitter. '0' - does not allow. * @IWM_POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK: '0' Driver disables power management, * '1' Driver enables PM (use rest of parameters) * @IWM_POWER_FLAGS_SKIP_OVER_DTIM_MSK: '0' PM have to walk up every DTIM, * '1' PM could sleep over DTIM till listen Interval. * @IWM_POWER_FLAGS_SNOOZE_ENA_MSK: Enable snoozing only if uAPSD is enabled and all * access categories are both delivery and trigger enabled. * @IWM_POWER_FLAGS_BT_SCO_ENA: Enable BT SCO coex only if uAPSD and * PBW Snoozing enabled * @IWM_POWER_FLAGS_ADVANCE_PM_ENA_MSK: Advanced PM (uAPSD) enable mask * @IWM_POWER_FLAGS_LPRX_ENA_MSK: Low Power RX enable. * @IWM_POWER_FLAGS_AP_UAPSD_MISBEHAVING_ENA_MSK: AP/GO's uAPSD misbehaving * detection enablement */ enum iwm_power_flags { IWM_POWER_FLAGS_POWER_SAVE_ENA_MSK = (1 << 0), IWM_POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK = (1 << 1), IWM_POWER_FLAGS_SKIP_OVER_DTIM_MSK = (1 << 2), IWM_POWER_FLAGS_SNOOZE_ENA_MSK = (1 << 5), IWM_POWER_FLAGS_BT_SCO_ENA = (1 << 8), IWM_POWER_FLAGS_ADVANCE_PM_ENA_MSK = (1 << 9), IWM_POWER_FLAGS_LPRX_ENA_MSK = (1 << 11), IWM_POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK = (1 << 12), }; #define IWM_POWER_VEC_SIZE 5 /** * struct iwm_powertable_cmd - legacy power command. Beside old API support this * is used also with a new power API for device wide power settings. * IWM_POWER_TABLE_CMD = 0x77 (command, has simple generic response) * * @flags: Power table command flags from IWM_POWER_FLAGS_* * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. * Minimum allowed:- 3 * DTIM. Keep alive period must be * set regardless of power scheme or current power state. * FW use this value also when PM is disabled. * @rx_data_timeout: Minimum time (usec) from last Rx packet for AM to * PSM transition - legacy PM * @tx_data_timeout: Minimum time (usec) from last Tx packet for AM to * PSM transition - legacy PM * @sleep_interval: not in use * @skip_dtim_periods: Number of DTIM periods to skip if Skip over DTIM flag * is set. For example, if it is required to skip over * one DTIM, this value need to be set to 2 (DTIM periods). * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled. * Default: 80dbm */ struct iwm_powertable_cmd { /* PM_POWER_TABLE_CMD_API_S_VER_6 */ uint16_t flags; uint8_t keep_alive_seconds; uint8_t debug_flags; uint32_t rx_data_timeout; uint32_t tx_data_timeout; uint32_t sleep_interval[IWM_POWER_VEC_SIZE]; uint32_t skip_dtim_periods; uint32_t lprx_rssi_threshold; } __packed; /** * enum iwm_device_power_flags - masks for device power command flags * @IWM_DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK: '1' Allow to save power by turning off * receiver and transmitter. '0' - does not allow. */ enum iwm_device_power_flags { IWM_DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK = (1 << 0), }; /** * struct iwm_device_power_cmd - device wide power command. * IWM_DEVICE_POWER_CMD = 0x77 (command, has simple generic response) * * @flags: Power table command flags from IWM_DEVICE_POWER_FLAGS_* */ struct iwm_device_power_cmd { /* PM_POWER_TABLE_CMD_API_S_VER_6 */ uint16_t flags; uint16_t reserved; } __packed; /** * struct iwm_mac_power_cmd - New power command containing uAPSD support * IWM_MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response) * @id_and_color: MAC contex identifier * @flags: Power table command flags from POWER_FLAGS_* * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. * Minimum allowed:- 3 * DTIM. Keep alive period must be * set regardless of power scheme or current power state. * FW use this value also when PM is disabled. * @rx_data_timeout: Minimum time (usec) from last Rx packet for AM to * PSM transition - legacy PM * @tx_data_timeout: Minimum time (usec) from last Tx packet for AM to * PSM transition - legacy PM * @sleep_interval: not in use * @skip_dtim_periods: Number of DTIM periods to skip if Skip over DTIM flag * is set. For example, if it is required to skip over * one DTIM, this value need to be set to 2 (DTIM periods). * @rx_data_timeout_uapsd: Minimum time (usec) from last Rx packet for AM to * PSM transition - uAPSD * @tx_data_timeout_uapsd: Minimum time (usec) from last Tx packet for AM to * PSM transition - uAPSD * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled. * Default: 80dbm * @num_skip_dtim: Number of DTIMs to skip if Skip over DTIM flag is set * @snooze_interval: Maximum time between attempts to retrieve buffered data * from the AP [msec] * @snooze_window: A window of time in which PBW snoozing insures that all * packets received. It is also the minimum time from last * received unicast RX packet, before client stops snoozing * for data. [msec] * @snooze_step: TBD * @qndp_tid: TID client shall use for uAPSD QNDP triggers * @uapsd_ac_flags: Set trigger-enabled and delivery-enabled indication for * each corresponding AC. * Use IEEE80211_WMM_IE_STA_QOSINFO_AC* for correct values. * @uapsd_max_sp: Use IEEE80211_WMM_IE_STA_QOSINFO_SP_* for correct * values. * @heavy_tx_thld_packets: TX threshold measured in number of packets * @heavy_rx_thld_packets: RX threshold measured in number of packets * @heavy_tx_thld_percentage: TX threshold measured in load's percentage * @heavy_rx_thld_percentage: RX threshold measured in load's percentage * @limited_ps_threshold: */ struct iwm_mac_power_cmd { /* CONTEXT_DESC_API_T_VER_1 */ uint32_t id_and_color; /* CLIENT_PM_POWER_TABLE_S_VER_1 */ uint16_t flags; uint16_t keep_alive_seconds; uint32_t rx_data_timeout; uint32_t tx_data_timeout; uint32_t rx_data_timeout_uapsd; uint32_t tx_data_timeout_uapsd; uint8_t lprx_rssi_threshold; uint8_t skip_dtim_periods; uint16_t snooze_interval; uint16_t snooze_window; uint8_t snooze_step; uint8_t qndp_tid; uint8_t uapsd_ac_flags; uint8_t uapsd_max_sp; uint8_t heavy_tx_thld_packets; uint8_t heavy_rx_thld_packets; uint8_t heavy_tx_thld_percentage; uint8_t heavy_rx_thld_percentage; uint8_t limited_ps_threshold; uint8_t reserved; } __packed; /* * struct iwm_uapsd_misbehaving_ap_notif - FW sends this notification when * associated AP is identified as improperly implementing uAPSD protocol. * IWM_PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78 * @sta_id: index of station in uCode's station table - associated AP ID in * this context. */ struct iwm_uapsd_misbehaving_ap_notif { uint32_t sta_id; uint8_t mac_id; uint8_t reserved[3]; } __packed; /** * struct iwm_beacon_filter_cmd * IWM_REPLY_BEACON_FILTERING_CMD = 0xd2 (command) * @id_and_color: MAC contex identifier * @bf_energy_delta: Used for RSSI filtering, if in 'normal' state. Send beacon * to driver if delta in Energy values calculated for this and last * passed beacon is greater than this threshold. Zero value means that * the Energy change is ignored for beacon filtering, and beacon will * not be forced to be sent to driver regardless of this delta. Typical * energy delta 5dB. * @bf_roaming_energy_delta: Used for RSSI filtering, if in 'roaming' state. * Send beacon to driver if delta in Energy values calculated for this * and last passed beacon is greater than this threshold. Zero value * means that the Energy change is ignored for beacon filtering while in * Roaming state, typical energy delta 1dB. * @bf_roaming_state: Used for RSSI filtering. If absolute Energy values * calculated for current beacon is less than the threshold, use * Roaming Energy Delta Threshold, otherwise use normal Energy Delta * Threshold. Typical energy threshold is -72dBm. * @bf_temp_threshold: This threshold determines the type of temperature * filtering (Slow or Fast) that is selected (Units are in Celsuis): * If the current temperature is above this threshold - Fast filter * will be used, If the current temperature is below this threshold - * Slow filter will be used. * @bf_temp_fast_filter: Send Beacon to driver if delta in temperature values * calculated for this and the last passed beacon is greater than this * threshold. Zero value means that the temperature change is ignored for * beacon filtering; beacons will not be forced to be sent to driver * regardless of whether its temperature has been changed. * @bf_temp_slow_filter: Send Beacon to driver if delta in temperature values * calculated for this and the last passed beacon is greater than this * threshold. Zero value means that the temperature change is ignored for * beacon filtering; beacons will not be forced to be sent to driver * regardless of whether its temperature has been changed. * @bf_enable_beacon_filter: 1, beacon filtering is enabled; 0, disabled. * @bf_filter_escape_timer: Send beacons to to driver if no beacons were passed * for a specific period of time. Units: Beacons. * @ba_escape_timer: Fully receive and parse beacon if no beacons were passed * for a longer period of time then this escape-timeout. Units: Beacons. * @ba_enable_beacon_abort: 1, beacon abort is enabled; 0, disabled. */ struct iwm_beacon_filter_cmd { uint32_t bf_energy_delta; uint32_t bf_roaming_energy_delta; uint32_t bf_roaming_state; uint32_t bf_temp_threshold; uint32_t bf_temp_fast_filter; uint32_t bf_temp_slow_filter; uint32_t bf_enable_beacon_filter; uint32_t bf_debug_flag; uint32_t bf_escape_timer; uint32_t ba_escape_timer; uint32_t ba_enable_beacon_abort; } __packed; /* Beacon filtering and beacon abort */ #define IWM_BF_ENERGY_DELTA_DEFAULT 5 #define IWM_BF_ENERGY_DELTA_MAX 255 #define IWM_BF_ENERGY_DELTA_MIN 0 #define IWM_BF_ROAMING_ENERGY_DELTA_DEFAULT 1 #define IWM_BF_ROAMING_ENERGY_DELTA_MAX 255 #define IWM_BF_ROAMING_ENERGY_DELTA_MIN 0 #define IWM_BF_ROAMING_STATE_DEFAULT 72 #define IWM_BF_ROAMING_STATE_MAX 255 #define IWM_BF_ROAMING_STATE_MIN 0 #define IWM_BF_TEMP_THRESHOLD_DEFAULT 112 #define IWM_BF_TEMP_THRESHOLD_MAX 255 #define IWM_BF_TEMP_THRESHOLD_MIN 0 #define IWM_BF_TEMP_FAST_FILTER_DEFAULT 1 #define IWM_BF_TEMP_FAST_FILTER_MAX 255 #define IWM_BF_TEMP_FAST_FILTER_MIN 0 #define IWM_BF_TEMP_SLOW_FILTER_DEFAULT 5 #define IWM_BF_TEMP_SLOW_FILTER_MAX 255 #define IWM_BF_TEMP_SLOW_FILTER_MIN 0 #define IWM_BF_ENABLE_BEACON_FILTER_DEFAULT 1 #define IWM_BF_DEBUG_FLAG_DEFAULT 0 #define IWM_BF_ESCAPE_TIMER_DEFAULT 50 #define IWM_BF_ESCAPE_TIMER_MAX 1024 #define IWM_BF_ESCAPE_TIMER_MIN 0 #define IWM_BA_ESCAPE_TIMER_DEFAULT 6 #define IWM_BA_ESCAPE_TIMER_D3 9 #define IWM_BA_ESCAPE_TIMER_MAX 1024 #define IWM_BA_ESCAPE_TIMER_MIN 0 #define IWM_BA_ENABLE_BEACON_ABORT_DEFAULT 1 #define IWM_BF_CMD_CONFIG_DEFAULTS \ .bf_energy_delta = htole32(IWM_BF_ENERGY_DELTA_DEFAULT), \ .bf_roaming_energy_delta = \ htole32(IWM_BF_ROAMING_ENERGY_DELTA_DEFAULT), \ .bf_roaming_state = htole32(IWM_BF_ROAMING_STATE_DEFAULT), \ .bf_temp_threshold = htole32(IWM_BF_TEMP_THRESHOLD_DEFAULT), \ .bf_temp_fast_filter = htole32(IWM_BF_TEMP_FAST_FILTER_DEFAULT), \ .bf_temp_slow_filter = htole32(IWM_BF_TEMP_SLOW_FILTER_DEFAULT), \ .bf_debug_flag = htole32(IWM_BF_DEBUG_FLAG_DEFAULT), \ .bf_escape_timer = htole32(IWM_BF_ESCAPE_TIMER_DEFAULT), \ .ba_escape_timer = htole32(IWM_BA_ESCAPE_TIMER_DEFAULT) /* * END mvm/fw-api-power.h */ /* * BEGIN mvm/fw-api-rs.h */ /* * These serve as indexes into * struct iwm_rate_info fw_rate_idx_to_plcp[IWM_RATE_COUNT]; * TODO: avoid overlap between legacy and HT rates */ enum { IWM_RATE_1M_INDEX = 0, IWM_FIRST_CCK_RATE = IWM_RATE_1M_INDEX, IWM_RATE_2M_INDEX, IWM_RATE_5M_INDEX, IWM_RATE_11M_INDEX, IWM_LAST_CCK_RATE = IWM_RATE_11M_INDEX, IWM_RATE_6M_INDEX, IWM_FIRST_OFDM_RATE = IWM_RATE_6M_INDEX, IWM_RATE_MCS_0_INDEX = IWM_RATE_6M_INDEX, IWM_FIRST_HT_RATE = IWM_RATE_MCS_0_INDEX, IWM_FIRST_VHT_RATE = IWM_RATE_MCS_0_INDEX, IWM_RATE_9M_INDEX, IWM_RATE_12M_INDEX, IWM_RATE_MCS_1_INDEX = IWM_RATE_12M_INDEX, IWM_RATE_18M_INDEX, IWM_RATE_MCS_2_INDEX = IWM_RATE_18M_INDEX, IWM_RATE_24M_INDEX, IWM_RATE_MCS_3_INDEX = IWM_RATE_24M_INDEX, IWM_RATE_36M_INDEX, IWM_RATE_MCS_4_INDEX = IWM_RATE_36M_INDEX, IWM_RATE_48M_INDEX, IWM_RATE_MCS_5_INDEX = IWM_RATE_48M_INDEX, IWM_RATE_54M_INDEX, IWM_RATE_MCS_6_INDEX = IWM_RATE_54M_INDEX, IWM_LAST_NON_HT_RATE = IWM_RATE_54M_INDEX, IWM_RATE_60M_INDEX, IWM_RATE_MCS_7_INDEX = IWM_RATE_60M_INDEX, IWM_LAST_HT_RATE = IWM_RATE_MCS_7_INDEX, IWM_RATE_MCS_8_INDEX, IWM_RATE_MCS_9_INDEX, IWM_LAST_VHT_RATE = IWM_RATE_MCS_9_INDEX, IWM_RATE_COUNT_LEGACY = IWM_LAST_NON_HT_RATE + 1, IWM_RATE_COUNT = IWM_LAST_VHT_RATE + 1, }; #define IWM_RATE_BIT_MSK(r) (1 << (IWM_RATE_##r##M_INDEX)) /* fw API values for legacy bit rates, both OFDM and CCK */ enum { IWM_RATE_6M_PLCP = 13, IWM_RATE_9M_PLCP = 15, IWM_RATE_12M_PLCP = 5, IWM_RATE_18M_PLCP = 7, IWM_RATE_24M_PLCP = 9, IWM_RATE_36M_PLCP = 11, IWM_RATE_48M_PLCP = 1, IWM_RATE_54M_PLCP = 3, IWM_RATE_1M_PLCP = 10, IWM_RATE_2M_PLCP = 20, IWM_RATE_5M_PLCP = 55, IWM_RATE_11M_PLCP = 110, IWM_RATE_INVM_PLCP = -1, }; /* * rate_n_flags bit fields * * The 32-bit value has different layouts in the low 8 bites depending on the * format. There are three formats, HT, VHT and legacy (11abg, with subformats * for CCK and OFDM). * * High-throughput (HT) rate format * bit 8 is 1, bit 26 is 0, bit 9 is 0 (OFDM) * Very High-throughput (VHT) rate format * bit 8 is 0, bit 26 is 1, bit 9 is 0 (OFDM) * Legacy OFDM rate format for bits 7:0 * bit 8 is 0, bit 26 is 0, bit 9 is 0 (OFDM) * Legacy CCK rate format for bits 7:0: * bit 8 is 0, bit 26 is 0, bit 9 is 1 (CCK) */ /* Bit 8: (1) HT format, (0) legacy or VHT format */ #define IWM_RATE_MCS_HT_POS 8 #define IWM_RATE_MCS_HT_MSK (1 << IWM_RATE_MCS_HT_POS) /* Bit 9: (1) CCK, (0) OFDM. HT (bit 8) must be "0" for this bit to be valid */ #define IWM_RATE_MCS_CCK_POS 9 #define IWM_RATE_MCS_CCK_MSK (1 << IWM_RATE_MCS_CCK_POS) /* Bit 26: (1) VHT format, (0) legacy format in bits 8:0 */ #define IWM_RATE_MCS_VHT_POS 26 #define IWM_RATE_MCS_VHT_MSK (1 << IWM_RATE_MCS_VHT_POS) /* * High-throughput (HT) rate format for bits 7:0 * * 2-0: MCS rate base * 0) 6 Mbps * 1) 12 Mbps * 2) 18 Mbps * 3) 24 Mbps * 4) 36 Mbps * 5) 48 Mbps * 6) 54 Mbps * 7) 60 Mbps * 4-3: 0) Single stream (SISO) * 1) Dual stream (MIMO) * 2) Triple stream (MIMO) * 5: Value of 0x20 in bits 7:0 indicates 6 Mbps HT40 duplicate data * (bits 7-6 are zero) * * Together the low 5 bits work out to the MCS index because we don't * support MCSes above 15/23, and 0-7 have one stream, 8-15 have two * streams and 16-23 have three streams. We could also support MCS 32 * which is the duplicate 20 MHz MCS (bit 5 set, all others zero.) */ #define IWM_RATE_HT_MCS_RATE_CODE_MSK 0x7 #define IWM_RATE_HT_MCS_NSS_POS 3 #define IWM_RATE_HT_MCS_NSS_MSK (3 << IWM_RATE_HT_MCS_NSS_POS) /* Bit 10: (1) Use Green Field preamble */ #define IWM_RATE_HT_MCS_GF_POS 10 #define IWM_RATE_HT_MCS_GF_MSK (1 << IWM_RATE_HT_MCS_GF_POS) #define IWM_RATE_HT_MCS_INDEX_MSK 0x3f /* * Very High-throughput (VHT) rate format for bits 7:0 * * 3-0: VHT MCS (0-9) * 5-4: number of streams - 1: * 0) Single stream (SISO) * 1) Dual stream (MIMO) * 2) Triple stream (MIMO) */ /* Bit 4-5: (0) SISO, (1) MIMO2 (2) MIMO3 */ #define IWM_RATE_VHT_MCS_RATE_CODE_MSK 0xf #define IWM_RATE_VHT_MCS_NSS_POS 4 #define IWM_RATE_VHT_MCS_NSS_MSK (3 << IWM_RATE_VHT_MCS_NSS_POS) /* * Legacy OFDM rate format for bits 7:0 * * 3-0: 0xD) 6 Mbps * 0xF) 9 Mbps * 0x5) 12 Mbps * 0x7) 18 Mbps * 0x9) 24 Mbps * 0xB) 36 Mbps * 0x1) 48 Mbps * 0x3) 54 Mbps * (bits 7-4 are 0) * * Legacy CCK rate format for bits 7:0: * bit 8 is 0, bit 26 is 0, bit 9 is 1 (CCK): * * 6-0: 10) 1 Mbps * 20) 2 Mbps * 55) 5.5 Mbps * 110) 11 Mbps * (bit 7 is 0) */ #define IWM_RATE_LEGACY_RATE_MSK 0xff /* * Bit 11-12: (0) 20MHz, (1) 40MHz, (2) 80MHz, (3) 160MHz * 0 and 1 are valid for HT and VHT, 2 and 3 only for VHT */ -#define IWM_RATE_MCS_CHAN_WIDTH_POS 11 -#define IWM_RATE_MCS_CHAN_WIDTH_MSK (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) -#define IWM_RATE_MCS_CHAN_WIDTH_20 (0 << IWM_RATE_MCS_CHAN_WIDTH_POS) -#define IWM_RATE_MCS_CHAN_WIDTH_40 (1 << IWM_RATE_MCS_CHAN_WIDTH_POS) -#define IWM_RATE_MCS_CHAN_WIDTH_80 (2 << IWM_RATE_MCS_CHAN_WIDTH_POS) -#define IWM_RATE_MCS_CHAN_WIDTH_160 (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) +#define IWM_RATE_MCS_CHAN_WIDTH_POS 11 +#define IWM_RATE_MCS_CHAN_WIDTH_MSK (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) +#define IWM_RATE_MCS_CHAN_WIDTH_20 (0 << IWM_RATE_MCS_CHAN_WIDTH_POS) +#define IWM_RATE_MCS_CHAN_WIDTH_40 (1 << IWM_RATE_MCS_CHAN_WIDTH_POS) +#define IWM_RATE_MCS_CHAN_WIDTH_80 (2 << IWM_RATE_MCS_CHAN_WIDTH_POS) +#define IWM_RATE_MCS_CHAN_WIDTH_160 (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) /* Bit 13: (1) Short guard interval (0.4 usec), (0) normal GI (0.8 usec) */ #define IWM_RATE_MCS_SGI_POS 13 #define IWM_RATE_MCS_SGI_MSK (1 << IWM_RATE_MCS_SGI_POS) /* Bit 14-16: Antenna selection (1) Ant A, (2) Ant B, (4) Ant C */ #define IWM_RATE_MCS_ANT_POS 14 #define IWM_RATE_MCS_ANT_A_MSK (1 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_B_MSK (2 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_C_MSK (4 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_AB_MSK (IWM_RATE_MCS_ANT_A_MSK | \ IWM_RATE_MCS_ANT_B_MSK) -#define IWM_RATE_MCS_ANT_ABC_MSK (IWM_RATE_MCS_ANT_AB_MSK | \ +#define IWM_RATE_MCS_ANT_ABC_MSK (IWM_RATE_MCS_ANT_AB_MSK | \ IWM_RATE_MCS_ANT_C_MSK) #define IWM_RATE_MCS_ANT_MSK IWM_RATE_MCS_ANT_ABC_MSK #define IWM_RATE_MCS_ANT_NUM 3 /* Bit 17-18: (0) SS, (1) SS*2 */ #define IWM_RATE_MCS_STBC_POS 17 #define IWM_RATE_MCS_STBC_MSK (1 << IWM_RATE_MCS_STBC_POS) /* Bit 19: (0) Beamforming is off, (1) Beamforming is on */ -#define IWM_RATE_MCS_BF_POS 19 -#define IWM_RATE_MCS_BF_MSK (1 << IWM_RATE_MCS_BF_POS) +#define IWM_RATE_MCS_BF_POS 19 +#define IWM_RATE_MCS_BF_MSK (1 << IWM_RATE_MCS_BF_POS) /* Bit 20: (0) ZLF is off, (1) ZLF is on */ #define IWM_RATE_MCS_ZLF_POS 20 #define IWM_RATE_MCS_ZLF_MSK (1 << IWM_RATE_MCS_ZLF_POS) /* Bit 24-25: (0) 20MHz (no dup), (1) 2x20MHz, (2) 4x20MHz, 3 8x20MHz */ #define IWM_RATE_MCS_DUP_POS 24 #define IWM_RATE_MCS_DUP_MSK (3 << IWM_RATE_MCS_DUP_POS) /* Bit 27: (1) LDPC enabled, (0) LDPC disabled */ #define IWM_RATE_MCS_LDPC_POS 27 #define IWM_RATE_MCS_LDPC_MSK (1 << IWM_RATE_MCS_LDPC_POS) /* Link Quality definitions */ /* # entries in rate scale table to support Tx retries */ #define IWM_LQ_MAX_RETRY_NUM 16 /* Link quality command flags bit fields */ /* Bit 0: (0) Don't use RTS (1) Use RTS */ -#define IWM_LQ_FLAG_USE_RTS_POS 0 -#define IWM_LQ_FLAG_USE_RTS_MSK (1 << IWM_LQ_FLAG_USE_RTS_POS) +#define IWM_LQ_FLAG_USE_RTS_POS 0 +#define IWM_LQ_FLAG_USE_RTS_MSK (1 << IWM_LQ_FLAG_USE_RTS_POS) /* Bit 1-3: LQ command color. Used to match responses to LQ commands */ -#define IWM_LQ_FLAG_COLOR_POS 1 -#define IWM_LQ_FLAG_COLOR_MSK (7 << IWM_LQ_FLAG_COLOR_POS) +#define IWM_LQ_FLAG_COLOR_POS 1 +#define IWM_LQ_FLAG_COLOR_MSK (7 << IWM_LQ_FLAG_COLOR_POS) /* Bit 4-5: Tx RTS BW Signalling * (0) No RTS BW signalling * (1) Static BW signalling * (2) Dynamic BW signalling */ -#define IWM_LQ_FLAG_RTS_BW_SIG_POS 4 -#define IWM_LQ_FLAG_RTS_BW_SIG_NONE (0 << IWM_LQ_FLAG_RTS_BW_SIG_POS) -#define IWM_LQ_FLAG_RTS_BW_SIG_STATIC (1 << IWM_LQ_FLAG_RTS_BW_SIG_POS) -#define IWM_LQ_FLAG_RTS_BW_SIG_DYNAMIC (2 << IWM_LQ_FLAG_RTS_BW_SIG_POS) +#define IWM_LQ_FLAG_RTS_BW_SIG_POS 4 +#define IWM_LQ_FLAG_RTS_BW_SIG_NONE (0 << IWM_LQ_FLAG_RTS_BW_SIG_POS) +#define IWM_LQ_FLAG_RTS_BW_SIG_STATIC (1 << IWM_LQ_FLAG_RTS_BW_SIG_POS) +#define IWM_LQ_FLAG_RTS_BW_SIG_DYNAMIC (2 << IWM_LQ_FLAG_RTS_BW_SIG_POS) /* Bit 6: (0) No dynamic BW selection (1) Allow dynamic BW selection * Dyanmic BW selection allows Tx with narrower BW then requested in rates */ -#define IWM_LQ_FLAG_DYNAMIC_BW_POS 6 -#define IWM_LQ_FLAG_DYNAMIC_BW_MSK (1 << IWM_LQ_FLAG_DYNAMIC_BW_POS) +#define IWM_LQ_FLAG_DYNAMIC_BW_POS 6 +#define IWM_LQ_FLAG_DYNAMIC_BW_MSK (1 << IWM_LQ_FLAG_DYNAMIC_BW_POS) +/* Single Stream Tx Parameters (lq_cmd->ss_params) + * Flags to control a smart FW decision about whether BFER/STBC/SISO will be + * used for single stream Tx. + */ + +/* Bit 0-1: Max STBC streams allowed. Can be 0-3. + * (0) - No STBC allowed + * (1) - 2x1 STBC allowed (HT/VHT) + * (2) - 4x2 STBC allowed (HT/VHT) + * (3) - 3x2 STBC allowed (HT only) + * All our chips are at most 2 antennas so only (1) is valid for now. + */ +#define IWM_LQ_SS_STBC_ALLOWED_POS 0 +#define IWM_LQ_SS_STBC_ALLOWED_MSK (3 << IWM_LQ_SS_STBC_ALLOWED_MSK) + +/* 2x1 STBC is allowed */ +#define IWM_LQ_SS_STBC_1SS_ALLOWED (1 << IWM_LQ_SS_STBC_ALLOWED_POS) + +/* Bit 2: Beamformer (VHT only) is allowed */ +#define IWM_LQ_SS_BFER_ALLOWED_POS 2 +#define IWM_LQ_SS_BFER_ALLOWED (1 << IWM_LQ_SS_BFER_ALLOWED_POS) + +/* Bit 3: Force BFER or STBC for testing + * If this is set: + * If BFER is allowed then force the ucode to choose BFER else + * If STBC is allowed then force the ucode to choose STBC over SISO + */ +#define IWM_LQ_SS_FORCE_POS 3 +#define IWM_LQ_SS_FORCE (1 << IWM_LQ_SS_FORCE_POS) + +/* Bit 31: ss_params field is valid. Used for FW backward compatibility + * with other drivers which don't support the ss_params API yet + */ +#define IWM_LQ_SS_PARAMS_VALID_POS 31 +#define IWM_LQ_SS_PARAMS_VALID (1 << IWM_LQ_SS_PARAMS_VALID_POS) + /** * struct iwm_lq_cmd - link quality command * @sta_id: station to update * @control: not used * @flags: combination of IWM_LQ_FLAG_* * @mimo_delim: the first SISO index in rs_table, which separates MIMO * and SISO rates * @single_stream_ant_msk: best antenna for SISO (can be dual in CDD). * Should be ANT_[ABC] * @dual_stream_ant_msk: best antennas for MIMO, combination of ANT_[ABC] * @initial_rate_index: first index from rs_table per AC category * @agg_time_limit: aggregation max time threshold in usec/100, meaning * value of 100 is one usec. Range is 100 to 8000 * @agg_disable_start_th: try-count threshold for starting aggregation. * If a frame has higher try-count, it should not be selected for * starting an aggregation sequence. * @agg_frame_cnt_limit: max frame count in an aggregation. * 0: no limit * 1: no aggregation (one frame per aggregation) * 2 - 0x3f: maximal number of frames (up to 3f == 63) * @rs_table: array of rates for each TX try, each is rate_n_flags, * meaning it is a combination of IWM_RATE_MCS_* and IWM_RATE_*_PLCP - * @bf_params: beam forming params, currently not used + * @ss_params: single stream features. declare whether STBC or BFER are allowed. */ struct iwm_lq_cmd { uint8_t sta_id; - uint8_t reserved1; + uint8_t reduced_tpc; uint16_t control; /* LINK_QUAL_GENERAL_PARAMS_API_S_VER_1 */ uint8_t flags; uint8_t mimo_delim; uint8_t single_stream_ant_msk; uint8_t dual_stream_ant_msk; uint8_t initial_rate_index[IWM_AC_NUM]; /* LINK_QUAL_AGG_PARAMS_API_S_VER_1 */ uint16_t agg_time_limit; uint8_t agg_disable_start_th; uint8_t agg_frame_cnt_limit; uint32_t reserved2; uint32_t rs_table[IWM_LQ_MAX_RETRY_NUM]; - uint32_t bf_params; + uint32_t ss_params; }; /* LINK_QUALITY_CMD_API_S_VER_1 */ /* * END mvm/fw-api-rs.h */ /* * BEGIN mvm/fw-api-tx.h */ /** * enum iwm_tx_flags - bitmasks for tx_flags in TX command * @IWM_TX_CMD_FLG_PROT_REQUIRE: use RTS or CTS-to-self to protect the frame * @IWM_TX_CMD_FLG_ACK: expect ACK from receiving station * @IWM_TX_CMD_FLG_STA_RATE: use RS table with initial index from the TX command. * Otherwise, use rate_n_flags from the TX command * @IWM_TX_CMD_FLG_BA: this frame is a block ack * @IWM_TX_CMD_FLG_BAR: this frame is a BA request, immediate BAR is expected * Must set IWM_TX_CMD_FLG_ACK with this flag. * @IWM_TX_CMD_FLG_TXOP_PROT: protect frame with full TXOP protection * @IWM_TX_CMD_FLG_VHT_NDPA: mark frame is NDPA for VHT beamformer sequence * @IWM_TX_CMD_FLG_HT_NDPA: mark frame is NDPA for HT beamformer sequence * @IWM_TX_CMD_FLG_CSI_FDBK2HOST: mark to send feedback to host (only if good CRC) * @IWM_TX_CMD_FLG_BT_DIS: disable BT priority for this frame * @IWM_TX_CMD_FLG_SEQ_CTL: set if FW should override the sequence control. * Should be set for mgmt, non-QOS data, mcast, bcast and in scan command * @IWM_TX_CMD_FLG_MORE_FRAG: this frame is non-last MPDU * @IWM_TX_CMD_FLG_NEXT_FRAME: this frame includes information of the next frame * @IWM_TX_CMD_FLG_TSF: FW should calculate and insert TSF in the frame * Should be set for beacons and probe responses * @IWM_TX_CMD_FLG_CALIB: activate PA TX power calibrations * @IWM_TX_CMD_FLG_KEEP_SEQ_CTL: if seq_ctl is set, don't increase inner seq count * @IWM_TX_CMD_FLG_AGG_START: allow this frame to start aggregation * @IWM_TX_CMD_FLG_MH_PAD: driver inserted 2 byte padding after MAC header. * Should be set for 26/30 length MAC headers * @IWM_TX_CMD_FLG_RESP_TO_DRV: zero this if the response should go only to FW * @IWM_TX_CMD_FLG_TKIP_MIC_DONE: FW already performed TKIP MIC calculation * @IWM_TX_CMD_FLG_DUR: disable duration overwriting used in PS-Poll Assoc-id * @IWM_TX_CMD_FLG_FW_DROP: FW should mark frame to be dropped * @IWM_TX_CMD_FLG_EXEC_PAPD: execute PAPD * @IWM_TX_CMD_FLG_PAPD_TYPE: 0 for reference power, 1 for nominal power * @IWM_TX_CMD_FLG_HCCA_CHUNK: mark start of TSPEC chunk */ enum iwm_tx_flags { IWM_TX_CMD_FLG_PROT_REQUIRE = (1 << 0), IWM_TX_CMD_FLG_ACK = (1 << 3), IWM_TX_CMD_FLG_STA_RATE = (1 << 4), IWM_TX_CMD_FLG_BA = (1 << 5), IWM_TX_CMD_FLG_BAR = (1 << 6), IWM_TX_CMD_FLG_TXOP_PROT = (1 << 7), IWM_TX_CMD_FLG_VHT_NDPA = (1 << 8), IWM_TX_CMD_FLG_HT_NDPA = (1 << 9), IWM_TX_CMD_FLG_CSI_FDBK2HOST = (1 << 10), IWM_TX_CMD_FLG_BT_DIS = (1 << 12), IWM_TX_CMD_FLG_SEQ_CTL = (1 << 13), IWM_TX_CMD_FLG_MORE_FRAG = (1 << 14), IWM_TX_CMD_FLG_NEXT_FRAME = (1 << 15), IWM_TX_CMD_FLG_TSF = (1 << 16), IWM_TX_CMD_FLG_CALIB = (1 << 17), IWM_TX_CMD_FLG_KEEP_SEQ_CTL = (1 << 18), IWM_TX_CMD_FLG_AGG_START = (1 << 19), IWM_TX_CMD_FLG_MH_PAD = (1 << 20), IWM_TX_CMD_FLG_RESP_TO_DRV = (1 << 21), IWM_TX_CMD_FLG_TKIP_MIC_DONE = (1 << 23), IWM_TX_CMD_FLG_DUR = (1 << 25), IWM_TX_CMD_FLG_FW_DROP = (1 << 26), IWM_TX_CMD_FLG_EXEC_PAPD = (1 << 27), IWM_TX_CMD_FLG_PAPD_TYPE = (1 << 28), IWM_TX_CMD_FLG_HCCA_CHUNK = (1 << 31) }; /* IWM_TX_FLAGS_BITS_API_S_VER_1 */ /** * enum iwm_tx_pm_timeouts - pm timeout values in TX command * @IWM_PM_FRAME_NONE: no need to suspend sleep mode * @IWM_PM_FRAME_MGMT: fw suspend sleep mode for 100TU * @IWM_PM_FRAME_ASSOC: fw suspend sleep mode for 10sec */ enum iwm_tx_pm_timeouts { IWM_PM_FRAME_NONE = 0, IWM_PM_FRAME_MGMT = 2, IWM_PM_FRAME_ASSOC = 3, }; /* * TX command security control */ #define IWM_TX_CMD_SEC_WEP 0x01 #define IWM_TX_CMD_SEC_CCM 0x02 #define IWM_TX_CMD_SEC_TKIP 0x03 #define IWM_TX_CMD_SEC_EXT 0x04 #define IWM_TX_CMD_SEC_MSK 0x07 #define IWM_TX_CMD_SEC_WEP_KEY_IDX_POS 6 #define IWM_TX_CMD_SEC_WEP_KEY_IDX_MSK 0xc0 #define IWM_TX_CMD_SEC_KEY128 0x08 /* TODO: how does these values are OK with only 16 bit variable??? */ /* * TX command next frame info * * bits 0:2 - security control (IWM_TX_CMD_SEC_*) * bit 3 - immediate ACK required * bit 4 - rate is taken from STA table * bit 5 - frame belongs to BA stream * bit 6 - immediate BA response expected * bit 7 - unused * bits 8:15 - Station ID * bits 16:31 - rate */ #define IWM_TX_CMD_NEXT_FRAME_ACK_MSK (0x8) #define IWM_TX_CMD_NEXT_FRAME_STA_RATE_MSK (0x10) #define IWM_TX_CMD_NEXT_FRAME_BA_MSK (0x20) #define IWM_TX_CMD_NEXT_FRAME_IMM_BA_RSP_MSK (0x40) #define IWM_TX_CMD_NEXT_FRAME_FLAGS_MSK (0xf8) #define IWM_TX_CMD_NEXT_FRAME_STA_ID_MSK (0xff00) #define IWM_TX_CMD_NEXT_FRAME_STA_ID_POS (8) #define IWM_TX_CMD_NEXT_FRAME_RATE_MSK (0xffff0000) #define IWM_TX_CMD_NEXT_FRAME_RATE_POS (16) /* * TX command Frame life time in us - to be written in pm_frame_timeout */ #define IWM_TX_CMD_LIFE_TIME_INFINITE 0xFFFFFFFF #define IWM_TX_CMD_LIFE_TIME_DEFAULT 2000000 /* 2000 ms*/ #define IWM_TX_CMD_LIFE_TIME_PROBE_RESP 40000 /* 40 ms */ #define IWM_TX_CMD_LIFE_TIME_EXPIRED_FRAME 0 /* * TID for non QoS frames - to be written in tid_tspec */ #define IWM_TID_NON_QOS IWM_MAX_TID_COUNT /* * Limits on the retransmissions - to be written in {data,rts}_retry_limit */ #define IWM_DEFAULT_TX_RETRY 15 #define IWM_MGMT_DFAULT_RETRY_LIMIT 3 #define IWM_RTS_DFAULT_RETRY_LIMIT 60 #define IWM_BAR_DFAULT_RETRY_LIMIT 60 #define IWM_LOW_RETRY_LIMIT 7 /* TODO: complete documentation for try_cnt and btkill_cnt */ /** * struct iwm_tx_cmd - TX command struct to FW * ( IWM_TX_CMD = 0x1c ) * @len: in bytes of the payload, see below for details * @next_frame_len: same as len, but for next frame (0 if not applicable) * Used for fragmentation and bursting, but not in 11n aggregation. * @tx_flags: combination of IWM_TX_CMD_FLG_* * @rate_n_flags: rate for *all* Tx attempts, if IWM_TX_CMD_FLG_STA_RATE_MSK is * cleared. Combination of IWM_RATE_MCS_* * @sta_id: index of destination station in FW station table * @sec_ctl: security control, IWM_TX_CMD_SEC_* * @initial_rate_index: index into the rate table for initial TX attempt. * Applied if IWM_TX_CMD_FLG_STA_RATE_MSK is set, normally 0 for data frames. * @key: security key * @next_frame_flags: IWM_TX_CMD_SEC_* and IWM_TX_CMD_NEXT_FRAME_* * @life_time: frame life time (usecs??) * @dram_lsb_ptr: Physical address of scratch area in the command (try_cnt + * btkill_cnd + reserved), first 32 bits. "0" disables usage. * @dram_msb_ptr: upper bits of the scratch physical address * @rts_retry_limit: max attempts for RTS * @data_retry_limit: max attempts to send the data packet * @tid_spec: TID/tspec * @pm_frame_timeout: PM TX frame timeout * @driver_txop: duration od EDCA TXOP, in 32-usec units. Set this if not * specified by HCCA protocol * * The byte count (both len and next_frame_len) includes MAC header * (24/26/30/32 bytes) * + 2 bytes pad if 26/30 header size * + 8 byte IV for CCM or TKIP (not used for WEP) * + Data payload * + 8-byte MIC (not used for CCM/WEP) * It does not include post-MAC padding, i.e., * MIC (CCM) 8 bytes, ICV (WEP/TKIP/CKIP) 4 bytes, CRC 4 bytes. * Range of len: 14-2342 bytes. * * After the struct fields the MAC header is placed, plus any padding, * and then the actial payload. */ struct iwm_tx_cmd { uint16_t len; uint16_t next_frame_len; uint32_t tx_flags; struct { uint8_t try_cnt; uint8_t btkill_cnt; uint16_t reserved; } scratch; /* DRAM_SCRATCH_API_U_VER_1 */ uint32_t rate_n_flags; uint8_t sta_id; uint8_t sec_ctl; uint8_t initial_rate_index; uint8_t reserved2; uint8_t key[16]; uint16_t next_frame_flags; uint16_t reserved3; uint32_t life_time; uint32_t dram_lsb_ptr; uint8_t dram_msb_ptr; uint8_t rts_retry_limit; uint8_t data_retry_limit; uint8_t tid_tspec; uint16_t pm_frame_timeout; uint16_t driver_txop; uint8_t payload[0]; struct ieee80211_frame hdr[0]; } __packed; /* IWM_TX_CMD_API_S_VER_3 */ /* * TX response related data */ /* * enum iwm_tx_status - status that is returned by the fw after attempts to Tx * @IWM_TX_STATUS_SUCCESS: * @IWM_TX_STATUS_DIRECT_DONE: * @IWM_TX_STATUS_POSTPONE_DELAY: * @IWM_TX_STATUS_POSTPONE_FEW_BYTES: * @IWM_TX_STATUS_POSTPONE_BT_PRIO: * @IWM_TX_STATUS_POSTPONE_QUIET_PERIOD: * @IWM_TX_STATUS_POSTPONE_CALC_TTAK: * @IWM_TX_STATUS_FAIL_INTERNAL_CROSSED_RETRY: * @IWM_TX_STATUS_FAIL_SHORT_LIMIT: * @IWM_TX_STATUS_FAIL_LONG_LIMIT: * @IWM_TX_STATUS_FAIL_UNDERRUN: * @IWM_TX_STATUS_FAIL_DRAIN_FLOW: * @IWM_TX_STATUS_FAIL_RFKILL_FLUSH: * @IWM_TX_STATUS_FAIL_LIFE_EXPIRE: * @IWM_TX_STATUS_FAIL_DEST_PS: * @IWM_TX_STATUS_FAIL_HOST_ABORTED: * @IWM_TX_STATUS_FAIL_BT_RETRY: * @IWM_TX_STATUS_FAIL_STA_INVALID: * @IWM_TX_TATUS_FAIL_FRAG_DROPPED: * @IWM_TX_STATUS_FAIL_TID_DISABLE: * @IWM_TX_STATUS_FAIL_FIFO_FLUSHED: * @IWM_TX_STATUS_FAIL_SMALL_CF_POLL: * @IWM_TX_STATUS_FAIL_FW_DROP: * @IWM_TX_STATUS_FAIL_STA_COLOR_MISMATCH: mismatch between color of Tx cmd and * STA table * @IWM_TX_FRAME_STATUS_INTERNAL_ABORT: * @IWM_TX_MODE_MSK: * @IWM_TX_MODE_NO_BURST: * @IWM_TX_MODE_IN_BURST_SEQ: * @IWM_TX_MODE_FIRST_IN_BURST: * @IWM_TX_QUEUE_NUM_MSK: * * Valid only if frame_count =1 * TODO: complete documentation */ enum iwm_tx_status { IWM_TX_STATUS_MSK = 0x000000ff, IWM_TX_STATUS_SUCCESS = 0x01, IWM_TX_STATUS_DIRECT_DONE = 0x02, /* postpone TX */ IWM_TX_STATUS_POSTPONE_DELAY = 0x40, IWM_TX_STATUS_POSTPONE_FEW_BYTES = 0x41, IWM_TX_STATUS_POSTPONE_BT_PRIO = 0x42, IWM_TX_STATUS_POSTPONE_QUIET_PERIOD = 0x43, IWM_TX_STATUS_POSTPONE_CALC_TTAK = 0x44, /* abort TX */ IWM_TX_STATUS_FAIL_INTERNAL_CROSSED_RETRY = 0x81, IWM_TX_STATUS_FAIL_SHORT_LIMIT = 0x82, IWM_TX_STATUS_FAIL_LONG_LIMIT = 0x83, IWM_TX_STATUS_FAIL_UNDERRUN = 0x84, IWM_TX_STATUS_FAIL_DRAIN_FLOW = 0x85, IWM_TX_STATUS_FAIL_RFKILL_FLUSH = 0x86, IWM_TX_STATUS_FAIL_LIFE_EXPIRE = 0x87, IWM_TX_STATUS_FAIL_DEST_PS = 0x88, IWM_TX_STATUS_FAIL_HOST_ABORTED = 0x89, IWM_TX_STATUS_FAIL_BT_RETRY = 0x8a, IWM_TX_STATUS_FAIL_STA_INVALID = 0x8b, IWM_TX_STATUS_FAIL_FRAG_DROPPED = 0x8c, IWM_TX_STATUS_FAIL_TID_DISABLE = 0x8d, IWM_TX_STATUS_FAIL_FIFO_FLUSHED = 0x8e, IWM_TX_STATUS_FAIL_SMALL_CF_POLL = 0x8f, IWM_TX_STATUS_FAIL_FW_DROP = 0x90, IWM_TX_STATUS_FAIL_STA_COLOR_MISMATCH = 0x91, IWM_TX_STATUS_INTERNAL_ABORT = 0x92, IWM_TX_MODE_MSK = 0x00000f00, IWM_TX_MODE_NO_BURST = 0x00000000, IWM_TX_MODE_IN_BURST_SEQ = 0x00000100, IWM_TX_MODE_FIRST_IN_BURST = 0x00000200, IWM_TX_QUEUE_NUM_MSK = 0x0001f000, IWM_TX_NARROW_BW_MSK = 0x00060000, IWM_TX_NARROW_BW_1DIV2 = 0x00020000, IWM_TX_NARROW_BW_1DIV4 = 0x00040000, IWM_TX_NARROW_BW_1DIV8 = 0x00060000, }; /* * enum iwm_tx_agg_status - TX aggregation status * @IWM_AGG_TX_STATE_STATUS_MSK: * @IWM_AGG_TX_STATE_TRANSMITTED: * @IWM_AGG_TX_STATE_UNDERRUN: * @IWM_AGG_TX_STATE_BT_PRIO: * @IWM_AGG_TX_STATE_FEW_BYTES: * @IWM_AGG_TX_STATE_ABORT: * @IWM_AGG_TX_STATE_LAST_SENT_TTL: * @IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT: * @IWM_AGG_TX_STATE_LAST_SENT_BT_KILL: * @IWM_AGG_TX_STATE_SCD_QUERY: * @IWM_AGG_TX_STATE_TEST_BAD_CRC32: * @IWM_AGG_TX_STATE_RESPONSE: * @IWM_AGG_TX_STATE_DUMP_TX: * @IWM_AGG_TX_STATE_DELAY_TX: * @IWM_AGG_TX_STATE_TRY_CNT_MSK: Retry count for 1st frame in aggregation (retries * occur if tx failed for this frame when it was a member of a previous * aggregation block). If rate scaling is used, retry count indicates the * rate table entry used for all frames in the new agg. *@ IWM_AGG_TX_STATE_SEQ_NUM_MSK: Command ID and sequence number of Tx command for * this frame * * TODO: complete documentation */ enum iwm_tx_agg_status { IWM_AGG_TX_STATE_STATUS_MSK = 0x00fff, IWM_AGG_TX_STATE_TRANSMITTED = 0x000, IWM_AGG_TX_STATE_UNDERRUN = 0x001, IWM_AGG_TX_STATE_BT_PRIO = 0x002, IWM_AGG_TX_STATE_FEW_BYTES = 0x004, IWM_AGG_TX_STATE_ABORT = 0x008, IWM_AGG_TX_STATE_LAST_SENT_TTL = 0x010, IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT = 0x020, IWM_AGG_TX_STATE_LAST_SENT_BT_KILL = 0x040, IWM_AGG_TX_STATE_SCD_QUERY = 0x080, IWM_AGG_TX_STATE_TEST_BAD_CRC32 = 0x0100, IWM_AGG_TX_STATE_RESPONSE = 0x1ff, IWM_AGG_TX_STATE_DUMP_TX = 0x200, IWM_AGG_TX_STATE_DELAY_TX = 0x400, IWM_AGG_TX_STATE_TRY_CNT_POS = 12, IWM_AGG_TX_STATE_TRY_CNT_MSK = 0xf << IWM_AGG_TX_STATE_TRY_CNT_POS, }; #define IWM_AGG_TX_STATE_LAST_SENT_MSK (IWM_AGG_TX_STATE_LAST_SENT_TTL| \ IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT| \ IWM_AGG_TX_STATE_LAST_SENT_BT_KILL) /* * The mask below describes a status where we are absolutely sure that the MPDU * wasn't sent. For BA/Underrun we cannot be that sure. All we know that we've * written the bytes to the TXE, but we know nothing about what the DSP did. */ #define IWM_AGG_TX_STAT_FRAME_NOT_SENT (IWM_AGG_TX_STATE_FEW_BYTES | \ IWM_AGG_TX_STATE_ABORT | \ IWM_AGG_TX_STATE_SCD_QUERY) /* * IWM_REPLY_TX = 0x1c (response) * * This response may be in one of two slightly different formats, indicated * by the frame_count field: * * 1) No aggregation (frame_count == 1). This reports Tx results for a single * frame. Multiple attempts, at various bit rates, may have been made for * this frame. * * 2) Aggregation (frame_count > 1). This reports Tx results for two or more * frames that used block-acknowledge. All frames were transmitted at * same rate. Rate scaling may have been used if first frame in this new * agg block failed in previous agg block(s). * * Note that, for aggregation, ACK (block-ack) status is not delivered * here; block-ack has not been received by the time the device records * this status. * This status relates to reasons the tx might have been blocked or aborted * within the device, rather than whether it was received successfully by * the destination station. */ /** * struct iwm_agg_tx_status - per packet TX aggregation status * @status: enum iwm_tx_agg_status * @sequence: Sequence # for this frame's Tx cmd (not SSN!) */ struct iwm_agg_tx_status { uint16_t status; uint16_t sequence; } __packed; /* * definitions for initial rate index field * bits [3:0] initial rate index * bits [6:4] rate table color, used for the initial rate * bit-7 invalid rate indication */ #define IWM_TX_RES_INIT_RATE_INDEX_MSK 0x0f #define IWM_TX_RES_RATE_TABLE_COLOR_MSK 0x70 #define IWM_TX_RES_INV_RATE_INDEX_MSK 0x80 #define IWM_MVM_TX_RES_GET_TID(_ra_tid) ((_ra_tid) & 0x0f) #define IWM_MVM_TX_RES_GET_RA(_ra_tid) ((_ra_tid) >> 4) /** * struct iwm_mvm_tx_resp - notifies that fw is TXing a packet * ( IWM_REPLY_TX = 0x1c ) * @frame_count: 1 no aggregation, >1 aggregation * @bt_kill_count: num of times blocked by bluetooth (unused for agg) * @failure_rts: num of failures due to unsuccessful RTS * @failure_frame: num failures due to no ACK (unused for agg) * @initial_rate: for non-agg: rate of the successful Tx. For agg: rate of the * Tx of all the batch. IWM_RATE_MCS_* * @wireless_media_time: for non-agg: RTS + CTS + frame tx attempts time + ACK. * for agg: RTS + CTS + aggregation tx time + block-ack time. * in usec. * @pa_status: tx power info * @pa_integ_res_a: tx power info * @pa_integ_res_b: tx power info * @pa_integ_res_c: tx power info * @measurement_req_id: tx power info * @tfd_info: TFD information set by the FH * @seq_ctl: sequence control from the Tx cmd * @byte_cnt: byte count from the Tx cmd * @tlc_info: TLC rate info * @ra_tid: bits [3:0] = ra, bits [7:4] = tid * @frame_ctrl: frame control * @status: for non-agg: frame status IWM_TX_STATUS_* * for agg: status of 1st frame, IWM_AGG_TX_STATE_*; other frame status fields * follow this one, up to frame_count. * * After the array of statuses comes the SSN of the SCD. Look at * %iwm_mvm_get_scd_ssn for more details. */ struct iwm_mvm_tx_resp { uint8_t frame_count; uint8_t bt_kill_count; uint8_t failure_rts; uint8_t failure_frame; uint32_t initial_rate; uint16_t wireless_media_time; uint8_t pa_status; uint8_t pa_integ_res_a[3]; uint8_t pa_integ_res_b[3]; uint8_t pa_integ_res_c[3]; uint16_t measurement_req_id; - uint16_t reserved; + uint8_t reduced_tpc; + uint8_t reserved; uint32_t tfd_info; uint16_t seq_ctl; uint16_t byte_cnt; uint8_t tlc_info; uint8_t ra_tid; uint16_t frame_ctrl; struct iwm_agg_tx_status status; } __packed; /* IWM_TX_RSP_API_S_VER_3 */ /** * struct iwm_mvm_ba_notif - notifies about reception of BA * ( IWM_BA_NOTIF = 0xc5 ) * @sta_addr_lo32: lower 32 bits of the MAC address * @sta_addr_hi16: upper 16 bits of the MAC address * @sta_id: Index of recipient (BA-sending) station in fw's station table * @tid: tid of the session * @seq_ctl: * @bitmap: the bitmap of the BA notification as seen in the air * @scd_flow: the tx queue this BA relates to * @scd_ssn: the index of the last contiguously sent packet * @txed: number of Txed frames in this batch * @txed_2_done: number of Acked frames in this batch */ struct iwm_mvm_ba_notif { uint32_t sta_addr_lo32; uint16_t sta_addr_hi16; uint16_t reserved; uint8_t sta_id; uint8_t tid; uint16_t seq_ctl; uint64_t bitmap; uint16_t scd_flow; uint16_t scd_ssn; uint8_t txed; uint8_t txed_2_done; uint16_t reserved1; } __packed; /* * struct iwm_mac_beacon_cmd - beacon template command * @tx: the tx commands associated with the beacon frame * @template_id: currently equal to the mac context id of the coresponding * mac. * @tim_idx: the offset of the tim IE in the beacon * @tim_size: the length of the tim IE * @frame: the template of the beacon frame */ struct iwm_mac_beacon_cmd { struct iwm_tx_cmd tx; uint32_t template_id; uint32_t tim_idx; uint32_t tim_size; struct ieee80211_frame frame[0]; } __packed; struct iwm_beacon_notif { struct iwm_mvm_tx_resp beacon_notify_hdr; uint64_t tsf; uint32_t ibss_mgr_status; } __packed; /** * enum iwm_dump_control - dump (flush) control flags * @IWM_DUMP_TX_FIFO_FLUSH: Dump MSDUs until the FIFO is empty * and the TFD queues are empty. */ enum iwm_dump_control { IWM_DUMP_TX_FIFO_FLUSH = (1 << 1), }; /** * struct iwm_tx_path_flush_cmd -- queue/FIFO flush command * @queues_ctl: bitmap of queues to flush * @flush_ctl: control flags * @reserved: reserved */ struct iwm_tx_path_flush_cmd { uint32_t queues_ctl; uint16_t flush_ctl; uint16_t reserved; } __packed; /* IWM_TX_PATH_FLUSH_CMD_API_S_VER_1 */ /** * iwm_mvm_get_scd_ssn - returns the SSN of the SCD * @tx_resp: the Tx response from the fw (agg or non-agg) * * When the fw sends an AMPDU, it fetches the MPDUs one after the other. Since * it can't know that everything will go well until the end of the AMPDU, it * can't know in advance the number of MPDUs that will be sent in the current * batch. This is why it writes the agg Tx response while it fetches the MPDUs. * Hence, it can't know in advance what the SSN of the SCD will be at the end * of the batch. This is why the SSN of the SCD is written at the end of the * whole struct at a variable offset. This function knows how to cope with the * variable offset and returns the SSN of the SCD. */ static inline uint32_t iwm_mvm_get_scd_ssn(struct iwm_mvm_tx_resp *tx_resp) { return le32_to_cpup((uint32_t *)&tx_resp->status + tx_resp->frame_count) & 0xfff; } /* * END mvm/fw-api-tx.h */ /* * BEGIN mvm/fw-api-scan.h */ /** * struct iwm_scd_txq_cfg_cmd - New txq hw scheduler config command * @token: * @sta_id: station id * @tid: * @scd_queue: scheduler queue to confiug * @enable: 1 queue enable, 0 queue disable * @aggregate: 1 aggregated queue, 0 otherwise * @tx_fifo: %enum iwm_mvm_tx_fifo * @window: BA window size * @ssn: SSN for the BA agreement */ struct iwm_scd_txq_cfg_cmd { uint8_t token; uint8_t sta_id; uint8_t tid; uint8_t scd_queue; uint8_t enable; uint8_t aggregate; uint8_t tx_fifo; uint8_t window; uint16_t ssn; uint16_t reserved; } __packed; /* SCD_QUEUE_CFG_CMD_API_S_VER_1 */ /** * struct iwm_scd_txq_cfg_rsp * @token: taken from the command * @sta_id: station id from the command * @tid: tid from the command * @scd_queue: scd_queue from the command */ struct iwm_scd_txq_cfg_rsp { uint8_t token; uint8_t sta_id; uint8_t tid; uint8_t scd_queue; } __packed; /* SCD_QUEUE_CFG_RSP_API_S_VER_1 */ /* Scan Commands, Responses, Notifications */ /* Masks for iwm_scan_channel.type flags */ #define IWM_SCAN_CHANNEL_TYPE_ACTIVE (1 << 0) #define IWM_SCAN_CHANNEL_NSSIDS(x) (((1 << (x)) - 1) << 1) /* Max number of IEs for direct SSID scans in a command */ #define IWM_PROBE_OPTION_MAX 20 /** * struct iwm_ssid_ie - directed scan network information element * * Up to 20 of these may appear in IWM_REPLY_SCAN_CMD, * selected by "type" bit field in struct iwm_scan_channel; * each channel may select different ssids from among the 20 entries. * SSID IEs get transmitted in reverse order of entry. */ struct iwm_ssid_ie { uint8_t id; uint8_t len; uint8_t ssid[IEEE80211_NWID_LEN]; } __packed; /* IWM_SCAN_DIRECT_SSID_IE_API_S_VER_1 */ /* scan offload */ #define IWM_SCAN_MAX_BLACKLIST_LEN 64 #define IWM_SCAN_SHORT_BLACKLIST_LEN 16 #define IWM_SCAN_MAX_PROFILES 11 #define IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE 512 /* Default watchdog (in MS) for scheduled scan iteration */ #define IWM_SCHED_SCAN_WATCHDOG cpu_to_le16(15000) #define IWM_GOOD_CRC_TH_DEFAULT cpu_to_le16(1) #define IWM_CAN_ABORT_STATUS 1 #define IWM_FULL_SCAN_MULTIPLIER 5 #define IWM_FAST_SCHED_SCAN_ITERATIONS 3 #define IWM_MAX_SCHED_SCAN_PLANS 2 /** * iwm_scan_schedule_lmac - schedule of scan offload * @delay: delay between iterations, in seconds. * @iterations: num of scan iterations * @full_scan_mul: number of partial scans before each full scan */ struct iwm_scan_schedule_lmac { uint16_t delay; uint8_t iterations; uint8_t full_scan_mul; } __packed; /* SCAN_SCHEDULE_API_S */ /** * iwm_scan_req_tx_cmd - SCAN_REQ_TX_CMD_API_S * @tx_flags: combination of TX_CMD_FLG_* * @rate_n_flags: rate for *all* Tx attempts, if TX_CMD_FLG_STA_RATE_MSK is * cleared. Combination of RATE_MCS_* * @sta_id: index of destination station in FW station table * @reserved: for alignment and future use */ struct iwm_scan_req_tx_cmd { uint32_t tx_flags; uint32_t rate_n_flags; uint8_t sta_id; uint8_t reserved[3]; } __packed; enum iwm_scan_channel_flags_lmac { IWM_UNIFIED_SCAN_CHANNEL_FULL = (1 << 27), IWM_UNIFIED_SCAN_CHANNEL_PARTIAL = (1 << 28), }; /** * iwm_scan_channel_cfg_lmac - SCAN_CHANNEL_CFG_S_VER2 * @flags: bits 1-20: directed scan to i'th ssid * other bits &enum iwm_scan_channel_flags_lmac * @channel_number: channel number 1-13 etc * @iter_count: scan iteration on this channel * @iter_interval: interval in seconds between iterations on one channel */ struct iwm_scan_channel_cfg_lmac { uint32_t flags; uint16_t channel_num; uint16_t iter_count; uint32_t iter_interval; } __packed; /* * iwm_scan_probe_segment - PROBE_SEGMENT_API_S_VER_1 * @offset: offset in the data block * @len: length of the segment */ struct iwm_scan_probe_segment { uint16_t offset; uint16_t len; } __packed; /* iwm_scan_probe_req - PROBE_REQUEST_FRAME_API_S_VER_2 * @mac_header: first (and common) part of the probe * @band_data: band specific data * @common_data: last (and common) part of the probe * @buf: raw data block */ struct iwm_scan_probe_req { struct iwm_scan_probe_segment mac_header; struct iwm_scan_probe_segment band_data[2]; struct iwm_scan_probe_segment common_data; uint8_t buf[IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE]; } __packed; enum iwm_scan_channel_flags { IWM_SCAN_CHANNEL_FLAG_EBS = (1 << 0), IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE = (1 << 1), IWM_SCAN_CHANNEL_FLAG_CACHE_ADD = (1 << 2), }; /* iwm_scan_channel_opt - CHANNEL_OPTIMIZATION_API_S * @flags: enum iwm_scan_channel_flags * @non_ebs_ratio: defines the ratio of number of scan iterations where EBS is * involved. * 1 - EBS is disabled. * 2 - every second scan will be full scan(and so on). */ struct iwm_scan_channel_opt { uint16_t flags; uint16_t non_ebs_ratio; } __packed; /** * iwm_mvm_lmac_scan_flags * @IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL: pass all beacons and probe responses * without filtering. * @IWM_MVM_LMAC_SCAN_FLAG_PASSIVE: force passive scan on all channels * @IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION: single channel scan * @IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE: send iteration complete notification * @IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS multiple SSID matching * @IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED: all passive scans will be fragmented * @IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED: insert WFA vendor-specific TPC report * and DS parameter set IEs into probe requests. * @IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL: use extended dwell time on channels * 1, 6 and 11. * @IWM_MVM_LMAC_SCAN_FLAG_MATCH: Send match found notification on matches */ enum iwm_mvm_lmac_scan_flags { IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL = (1 << 0), IWM_MVM_LMAC_SCAN_FLAG_PASSIVE = (1 << 1), IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION = (1 << 2), IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE = (1 << 3), IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS = (1 << 4), IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED = (1 << 5), IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED = (1 << 6), IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL = (1 << 7), IWM_MVM_LMAC_SCAN_FLAG_MATCH = (1 << 9), }; enum iwm_scan_priority { IWM_SCAN_PRIORITY_LOW, IWM_SCAN_PRIORITY_MEDIUM, IWM_SCAN_PRIORITY_HIGH, }; /** * iwm_scan_req_lmac - SCAN_REQUEST_CMD_API_S_VER_1 * @reserved1: for alignment and future use * @channel_num: num of channels to scan * @active-dwell: dwell time for active channels * @passive-dwell: dwell time for passive channels * @fragmented-dwell: dwell time for fragmented passive scan * @extended_dwell: dwell time for channels 1, 6 and 11 (in certain cases) * @reserved2: for alignment and future use * @rx_chain_selct: PHY_RX_CHAIN_* flags * @scan_flags: &enum iwm_mvm_lmac_scan_flags * @max_out_time: max time (in TU) to be out of associated channel * @suspend_time: pause scan this long (TUs) when returning to service channel * @flags: RXON flags * @filter_flags: RXON filter * @tx_cmd: tx command for active scan; for 2GHz and for 5GHz * @direct_scan: list of SSIDs for directed active scan * @scan_prio: enum iwm_scan_priority * @iter_num: number of scan iterations * @delay: delay in seconds before first iteration * @schedule: two scheduling plans. The first one is finite, the second one can * be infinite. * @channel_opt: channel optimization options, for full and partial scan * @data: channel configuration and probe request packet. */ struct iwm_scan_req_lmac { /* SCAN_REQUEST_FIXED_PART_API_S_VER_7 */ uint32_t reserved1; uint8_t n_channels; uint8_t active_dwell; uint8_t passive_dwell; uint8_t fragmented_dwell; uint8_t extended_dwell; uint8_t reserved2; uint16_t rx_chain_select; uint32_t scan_flags; uint32_t max_out_time; uint32_t suspend_time; /* RX_ON_FLAGS_API_S_VER_1 */ uint32_t flags; uint32_t filter_flags; struct iwm_scan_req_tx_cmd tx_cmd[2]; struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; uint32_t scan_prio; /* SCAN_REQ_PERIODIC_PARAMS_API_S */ uint32_t iter_num; uint32_t delay; struct iwm_scan_schedule_lmac schedule[IWM_MAX_SCHED_SCAN_PLANS]; struct iwm_scan_channel_opt channel_opt[2]; uint8_t data[]; } __packed; /** * iwm_scan_offload_complete - PERIODIC_SCAN_COMPLETE_NTF_API_S_VER_2 * @last_schedule_line: last schedule line executed (fast or regular) * @last_schedule_iteration: last scan iteration executed before scan abort * @status: enum iwm_scan_offload_complete_status * @ebs_status: EBS success status &enum iwm_scan_ebs_status * @time_after_last_iter; time in seconds elapsed after last iteration */ struct iwm_periodic_scan_complete { uint8_t last_schedule_line; uint8_t last_schedule_iteration; uint8_t status; uint8_t ebs_status; uint32_t time_after_last_iter; uint32_t reserved; } __packed; /* How many statistics are gathered for each channel */ #define IWM_SCAN_RESULTS_STATISTICS 1 /** * enum iwm_scan_complete_status - status codes for scan complete notifications * @IWM_SCAN_COMP_STATUS_OK: scan completed successfully * @IWM_SCAN_COMP_STATUS_ABORT: scan was aborted by user * @IWM_SCAN_COMP_STATUS_ERR_SLEEP: sending null sleep packet failed * @IWM_SCAN_COMP_STATUS_ERR_CHAN_TIMEOUT: timeout before channel is ready * @IWM_SCAN_COMP_STATUS_ERR_PROBE: sending probe request failed * @IWM_SCAN_COMP_STATUS_ERR_WAKEUP: sending null wakeup packet failed * @IWM_SCAN_COMP_STATUS_ERR_ANTENNAS: invalid antennas chosen at scan command * @IWM_SCAN_COMP_STATUS_ERR_INTERNAL: internal error caused scan abort * @IWM_SCAN_COMP_STATUS_ERR_COEX: medium was lost ot WiMax * @IWM_SCAN_COMP_STATUS_P2P_ACTION_OK: P2P public action frame TX was successful * (not an error!) * @IWM_SCAN_COMP_STATUS_ITERATION_END: indicates end of one repeatition the driver * asked for * @IWM_SCAN_COMP_STATUS_ERR_ALLOC_TE: scan could not allocate time events */ enum iwm_scan_complete_status { IWM_SCAN_COMP_STATUS_OK = 0x1, IWM_SCAN_COMP_STATUS_ABORT = 0x2, IWM_SCAN_COMP_STATUS_ERR_SLEEP = 0x3, IWM_SCAN_COMP_STATUS_ERR_CHAN_TIMEOUT = 0x4, IWM_SCAN_COMP_STATUS_ERR_PROBE = 0x5, IWM_SCAN_COMP_STATUS_ERR_WAKEUP = 0x6, IWM_SCAN_COMP_STATUS_ERR_ANTENNAS = 0x7, IWM_SCAN_COMP_STATUS_ERR_INTERNAL = 0x8, IWM_SCAN_COMP_STATUS_ERR_COEX = 0x9, IWM_SCAN_COMP_STATUS_P2P_ACTION_OK = 0xA, IWM_SCAN_COMP_STATUS_ITERATION_END = 0x0B, IWM_SCAN_COMP_STATUS_ERR_ALLOC_TE = 0x0C, }; /** * struct iwm_scan_results_notif - scan results for one channel * ( IWM_SCAN_RESULTS_NOTIFICATION = 0x83 ) * @channel: which channel the results are from * @band: 0 for 5.2 GHz, 1 for 2.4 GHz * @probe_status: IWM_SCAN_PROBE_STATUS_*, indicates success of probe request * @num_probe_not_sent: # of request that weren't sent due to not enough time * @duration: duration spent in channel, in usecs * @statistics: statistics gathered for this channel */ struct iwm_scan_results_notif { uint8_t channel; uint8_t band; uint8_t probe_status; uint8_t num_probe_not_sent; uint32_t duration; uint32_t statistics[IWM_SCAN_RESULTS_STATISTICS]; } __packed; /* IWM_SCAN_RESULT_NTF_API_S_VER_2 */ enum iwm_scan_framework_client { IWM_SCAN_CLIENT_SCHED_SCAN = (1 << 0), IWM_SCAN_CLIENT_NETDETECT = (1 << 1), IWM_SCAN_CLIENT_ASSET_TRACKING = (1 << 2), }; /** * iwm_scan_offload_blacklist - IWM_SCAN_OFFLOAD_BLACKLIST_S * @ssid: MAC address to filter out * @reported_rssi: AP rssi reported to the host * @client_bitmap: clients ignore this entry - enum scan_framework_client */ struct iwm_scan_offload_blacklist { uint8_t ssid[IEEE80211_ADDR_LEN]; uint8_t reported_rssi; uint8_t client_bitmap; } __packed; enum iwm_scan_offload_network_type { IWM_NETWORK_TYPE_BSS = 1, IWM_NETWORK_TYPE_IBSS = 2, IWM_NETWORK_TYPE_ANY = 3, }; enum iwm_scan_offload_band_selection { IWM_SCAN_OFFLOAD_SELECT_2_4 = 0x4, IWM_SCAN_OFFLOAD_SELECT_5_2 = 0x8, IWM_SCAN_OFFLOAD_SELECT_ANY = 0xc, }; /** * iwm_scan_offload_profile - IWM_SCAN_OFFLOAD_PROFILE_S * @ssid_index: index to ssid list in fixed part * @unicast_cipher: encryption olgorithm to match - bitmap * @aut_alg: authentication olgorithm to match - bitmap * @network_type: enum iwm_scan_offload_network_type * @band_selection: enum iwm_scan_offload_band_selection * @client_bitmap: clients waiting for match - enum scan_framework_client */ struct iwm_scan_offload_profile { uint8_t ssid_index; uint8_t unicast_cipher; uint8_t auth_alg; uint8_t network_type; uint8_t band_selection; uint8_t client_bitmap; uint8_t reserved[2]; } __packed; /** * iwm_scan_offload_profile_cfg - IWM_SCAN_OFFLOAD_PROFILES_CFG_API_S_VER_1 * @blaclist: AP list to filter off from scan results * @profiles: profiles to search for match * @blacklist_len: length of blacklist * @num_profiles: num of profiles in the list * @match_notify: clients waiting for match found notification * @pass_match: clients waiting for the results * @active_clients: active clients bitmap - enum scan_framework_client * @any_beacon_notify: clients waiting for match notification without match */ struct iwm_scan_offload_profile_cfg { struct iwm_scan_offload_profile profiles[IWM_SCAN_MAX_PROFILES]; uint8_t blacklist_len; uint8_t num_profiles; uint8_t match_notify; uint8_t pass_match; uint8_t active_clients; uint8_t any_beacon_notify; uint8_t reserved[2]; } __packed; enum iwm_scan_offload_complete_status { IWM_SCAN_OFFLOAD_COMPLETED = 1, IWM_SCAN_OFFLOAD_ABORTED = 2, }; enum iwm_scan_ebs_status { IWM_SCAN_EBS_SUCCESS, IWM_SCAN_EBS_FAILED, IWM_SCAN_EBS_CHAN_NOT_FOUND, IWM_SCAN_EBS_INACTIVE, }; /** * struct iwm_lmac_scan_complete_notif - notifies end of scanning (all channels) * SCAN_COMPLETE_NTF_API_S_VER_3 * @scanned_channels: number of channels scanned (and number of valid results) * @status: one of SCAN_COMP_STATUS_* * @bt_status: BT on/off status * @last_channel: last channel that was scanned * @tsf_low: TSF timer (lower half) in usecs * @tsf_high: TSF timer (higher half) in usecs * @results: an array of scan results, only "scanned_channels" of them are valid */ struct iwm_lmac_scan_complete_notif { uint8_t scanned_channels; uint8_t status; uint8_t bt_status; uint8_t last_channel; uint32_t tsf_low; uint32_t tsf_high; struct iwm_scan_results_notif results[]; } __packed; /* * END mvm/fw-api-scan.h */ /* * BEGIN mvm/fw-api-sta.h */ /* UMAC Scan API */ /* The maximum of either of these cannot exceed 8, because we use an * 8-bit mask (see IWM_MVM_SCAN_MASK). */ #define IWM_MVM_MAX_UMAC_SCANS 8 #define IWM_MVM_MAX_LMAC_SCANS 1 enum iwm_scan_config_flags { IWM_SCAN_CONFIG_FLAG_ACTIVATE = (1 << 0), IWM_SCAN_CONFIG_FLAG_DEACTIVATE = (1 << 1), IWM_SCAN_CONFIG_FLAG_FORBID_CHUB_REQS = (1 << 2), IWM_SCAN_CONFIG_FLAG_ALLOW_CHUB_REQS = (1 << 3), IWM_SCAN_CONFIG_FLAG_SET_TX_CHAINS = (1 << 8), IWM_SCAN_CONFIG_FLAG_SET_RX_CHAINS = (1 << 9), IWM_SCAN_CONFIG_FLAG_SET_AUX_STA_ID = (1 << 10), IWM_SCAN_CONFIG_FLAG_SET_ALL_TIMES = (1 << 11), IWM_SCAN_CONFIG_FLAG_SET_EFFECTIVE_TIMES = (1 << 12), IWM_SCAN_CONFIG_FLAG_SET_CHANNEL_FLAGS = (1 << 13), IWM_SCAN_CONFIG_FLAG_SET_LEGACY_RATES = (1 << 14), IWM_SCAN_CONFIG_FLAG_SET_MAC_ADDR = (1 << 15), IWM_SCAN_CONFIG_FLAG_SET_FRAGMENTED = (1 << 16), IWM_SCAN_CONFIG_FLAG_CLEAR_FRAGMENTED = (1 << 17), IWM_SCAN_CONFIG_FLAG_SET_CAM_MODE = (1 << 18), IWM_SCAN_CONFIG_FLAG_CLEAR_CAM_MODE = (1 << 19), IWM_SCAN_CONFIG_FLAG_SET_PROMISC_MODE = (1 << 20), IWM_SCAN_CONFIG_FLAG_CLEAR_PROMISC_MODE = (1 << 21), /* Bits 26-31 are for num of channels in channel_array */ #define IWM_SCAN_CONFIG_N_CHANNELS(n) ((n) << 26) }; enum iwm_scan_config_rates { /* OFDM basic rates */ IWM_SCAN_CONFIG_RATE_6M = (1 << 0), IWM_SCAN_CONFIG_RATE_9M = (1 << 1), IWM_SCAN_CONFIG_RATE_12M = (1 << 2), IWM_SCAN_CONFIG_RATE_18M = (1 << 3), IWM_SCAN_CONFIG_RATE_24M = (1 << 4), IWM_SCAN_CONFIG_RATE_36M = (1 << 5), IWM_SCAN_CONFIG_RATE_48M = (1 << 6), IWM_SCAN_CONFIG_RATE_54M = (1 << 7), /* CCK basic rates */ IWM_SCAN_CONFIG_RATE_1M = (1 << 8), IWM_SCAN_CONFIG_RATE_2M = (1 << 9), IWM_SCAN_CONFIG_RATE_5M = (1 << 10), IWM_SCAN_CONFIG_RATE_11M = (1 << 11), /* Bits 16-27 are for supported rates */ #define IWM_SCAN_CONFIG_SUPPORTED_RATE(rate) ((rate) << 16) }; enum iwm_channel_flags { IWM_CHANNEL_FLAG_EBS = (1 << 0), IWM_CHANNEL_FLAG_ACCURATE_EBS = (1 << 1), IWM_CHANNEL_FLAG_EBS_ADD = (1 << 2), IWM_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE = (1 << 3), }; /** * struct iwm_scan_config * @flags: enum scan_config_flags * @tx_chains: valid_tx antenna - ANT_* definitions * @rx_chains: valid_rx antenna - ANT_* definitions * @legacy_rates: default legacy rates - enum scan_config_rates * @out_of_channel_time: default max out of serving channel time * @suspend_time: default max suspend time * @dwell_active: default dwell time for active scan * @dwell_passive: default dwell time for passive scan * @dwell_fragmented: default dwell time for fragmented scan * @dwell_extended: default dwell time for channels 1, 6 and 11 * @mac_addr: default mac address to be used in probes * @bcast_sta_id: the index of the station in the fw * @channel_flags: default channel flags - enum iwm_channel_flags * scan_config_channel_flag * @channel_array: default supported channels */ struct iwm_scan_config { uint32_t flags; uint32_t tx_chains; uint32_t rx_chains; uint32_t legacy_rates; uint32_t out_of_channel_time; uint32_t suspend_time; uint8_t dwell_active; uint8_t dwell_passive; uint8_t dwell_fragmented; uint8_t dwell_extended; uint8_t mac_addr[IEEE80211_ADDR_LEN]; uint8_t bcast_sta_id; uint8_t channel_flags; uint8_t channel_array[]; } __packed; /* SCAN_CONFIG_DB_CMD_API_S */ /** * iwm_umac_scan_flags *@IWM_UMAC_SCAN_FLAG_PREEMPTIVE: scan process triggered by this scan request * can be preempted by other scan requests with higher priority. * The low priority scan will be resumed when the higher proirity scan is * completed. *@IWM_UMAC_SCAN_FLAG_START_NOTIF: notification will be sent to the driver * when scan starts. */ enum iwm_umac_scan_flags { IWM_UMAC_SCAN_FLAG_PREEMPTIVE = (1 << 0), IWM_UMAC_SCAN_FLAG_START_NOTIF = (1 << 1), }; enum iwm_umac_scan_uid_offsets { IWM_UMAC_SCAN_UID_TYPE_OFFSET = 0, IWM_UMAC_SCAN_UID_SEQ_OFFSET = 8, }; enum iwm_umac_scan_general_flags { IWM_UMAC_SCAN_GEN_FLAGS_PERIODIC = (1 << 0), IWM_UMAC_SCAN_GEN_FLAGS_OVER_BT = (1 << 1), IWM_UMAC_SCAN_GEN_FLAGS_PASS_ALL = (1 << 2), IWM_UMAC_SCAN_GEN_FLAGS_PASSIVE = (1 << 3), IWM_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT = (1 << 4), IWM_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE = (1 << 5), IWM_UMAC_SCAN_GEN_FLAGS_MULTIPLE_SSID = (1 << 6), IWM_UMAC_SCAN_GEN_FLAGS_FRAGMENTED = (1 << 7), IWM_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED = (1 << 8), IWM_UMAC_SCAN_GEN_FLAGS_MATCH = (1 << 9), IWM_UMAC_SCAN_GEN_FLAGS_EXTENDED_DWELL = (1 << 10), }; /** * struct iwm_scan_channel_cfg_umac * @flags: bitmap - 0-19: directed scan to i'th ssid. * @channel_num: channel number 1-13 etc. * @iter_count: repetition count for the channel. * @iter_interval: interval between two scan iterations on one channel. */ struct iwm_scan_channel_cfg_umac { uint32_t flags; #define IWM_SCAN_CHANNEL_UMAC_NSSIDS(x) ((1 << (x)) - 1) uint8_t channel_num; uint8_t iter_count; uint16_t iter_interval; } __packed; /* SCAN_CHANNEL_CFG_S_VER2 */ /** * struct iwm_scan_umac_schedule * @interval: interval in seconds between scan iterations * @iter_count: num of scan iterations for schedule plan, 0xff for infinite loop * @reserved: for alignment and future use */ struct iwm_scan_umac_schedule { uint16_t interval; uint8_t iter_count; uint8_t reserved; } __packed; /* SCAN_SCHED_PARAM_API_S_VER_1 */ /** * struct iwm_scan_req_umac_tail - the rest of the UMAC scan request command * parameters following channels configuration array. * @schedule: two scheduling plans. * @delay: delay in TUs before starting the first scan iteration * @reserved: for future use and alignment * @preq: probe request with IEs blocks * @direct_scan: list of SSIDs for directed active scan */ struct iwm_scan_req_umac_tail { /* SCAN_PERIODIC_PARAMS_API_S_VER_1 */ struct iwm_scan_umac_schedule schedule[IWM_MAX_SCHED_SCAN_PLANS]; uint16_t delay; uint16_t reserved; /* SCAN_PROBE_PARAMS_API_S_VER_1 */ struct iwm_scan_probe_req preq; struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; } __packed; /** * struct iwm_scan_req_umac * @flags: &enum iwm_umac_scan_flags * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @ooc_priority: out of channel priority - &enum iwm_scan_priority * @general_flags: &enum iwm_umac_scan_general_flags * @extended_dwell: dwell time for channels 1, 6 and 11 * @active_dwell: dwell time for active scan * @passive_dwell: dwell time for passive scan * @fragmented_dwell: dwell time for fragmented passive scan * @max_out_time: max out of serving channel time * @suspend_time: max suspend time * @scan_priority: scan internal prioritization &enum iwm_scan_priority * @channel_flags: &enum iwm_scan_channel_flags * @n_channels: num of channels in scan request * @reserved: for future use and alignment * @data: &struct iwm_scan_channel_cfg_umac and * &struct iwm_scan_req_umac_tail */ struct iwm_scan_req_umac { uint32_t flags; uint32_t uid; uint32_t ooc_priority; /* SCAN_GENERAL_PARAMS_API_S_VER_1 */ uint32_t general_flags; uint8_t extended_dwell; uint8_t active_dwell; uint8_t passive_dwell; uint8_t fragmented_dwell; uint32_t max_out_time; uint32_t suspend_time; uint32_t scan_priority; /* SCAN_CHANNEL_PARAMS_API_S_VER_1 */ uint8_t channel_flags; uint8_t n_channels; uint16_t reserved; uint8_t data[]; } __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_1 */ /** * struct iwm_umac_scan_abort * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @flags: reserved */ struct iwm_umac_scan_abort { uint32_t uid; uint32_t flags; } __packed; /* SCAN_ABORT_CMD_UMAC_API_S_VER_1 */ /** * struct iwm_umac_scan_complete * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @last_schedule: last scheduling line * @last_iter: last scan iteration number * @scan status: &enum iwm_scan_offload_complete_status * @ebs_status: &enum iwm_scan_ebs_status * @time_from_last_iter: time elapsed from last iteration * @reserved: for future use */ struct iwm_umac_scan_complete { uint32_t uid; uint8_t last_schedule; uint8_t last_iter; uint8_t status; uint8_t ebs_status; uint32_t time_from_last_iter; uint32_t reserved; } __packed; /* SCAN_COMPLETE_NTF_UMAC_API_S_VER_1 */ #define IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN 5 /** * struct iwm_scan_offload_profile_match - match information * @bssid: matched bssid * @channel: channel where the match occurred * @energy: * @matching_feature: * @matching_channels: bitmap of channels that matched, referencing * the channels passed in tue scan offload request */ struct iwm_scan_offload_profile_match { uint8_t bssid[IEEE80211_ADDR_LEN]; uint16_t reserved; uint8_t channel; uint8_t energy; uint8_t matching_feature; uint8_t matching_channels[IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN]; } __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_1 */ /** * struct iwm_scan_offload_profiles_query - match results query response * @matched_profiles: bitmap of matched profiles, referencing the * matches passed in the scan offload request * @last_scan_age: age of the last offloaded scan * @n_scans_done: number of offloaded scans done * @gp2_d0u: GP2 when D0U occurred * @gp2_invoked: GP2 when scan offload was invoked * @resume_while_scanning: not used * @self_recovery: obsolete * @reserved: reserved * @matches: array of match information, one for each match */ struct iwm_scan_offload_profiles_query { uint32_t matched_profiles; uint32_t last_scan_age; uint32_t n_scans_done; uint32_t gp2_d0u; uint32_t gp2_invoked; uint8_t resume_while_scanning; uint8_t self_recovery; uint16_t reserved; struct iwm_scan_offload_profile_match matches[IWM_SCAN_MAX_PROFILES]; } __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_2 */ /** * struct iwm_umac_scan_iter_complete_notif - notifies end of scanning iteration * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @scanned_channels: number of channels scanned and number of valid elements in * results array * @status: one of SCAN_COMP_STATUS_* * @bt_status: BT on/off status * @last_channel: last channel that was scanned * @tsf_low: TSF timer (lower half) in usecs * @tsf_high: TSF timer (higher half) in usecs * @results: array of scan results, only "scanned_channels" of them are valid */ struct iwm_umac_scan_iter_complete_notif { uint32_t uid; uint8_t scanned_channels; uint8_t status; uint8_t bt_status; uint8_t last_channel; uint32_t tsf_low; uint32_t tsf_high; struct iwm_scan_results_notif results[]; } __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_1 */ /* Please keep this enum *SORTED* by hex value. * Needed for binary search, otherwise a warning will be triggered. */ enum iwm_scan_subcmd_ids { IWM_GSCAN_START_CMD = 0x0, IWM_GSCAN_STOP_CMD = 0x1, IWM_GSCAN_SET_HOTLIST_CMD = 0x2, IWM_GSCAN_RESET_HOTLIST_CMD = 0x3, IWM_GSCAN_SET_SIGNIFICANT_CHANGE_CMD = 0x4, IWM_GSCAN_RESET_SIGNIFICANT_CHANGE_CMD = 0x5, IWM_GSCAN_SIGNIFICANT_CHANGE_EVENT = 0xFD, IWM_GSCAN_HOTLIST_CHANGE_EVENT = 0xFE, IWM_GSCAN_RESULTS_AVAILABLE_EVENT = 0xFF, }; /* STA API */ /** * enum iwm_sta_flags - flags for the ADD_STA host command * @IWM_STA_FLG_REDUCED_TX_PWR_CTRL: * @IWM_STA_FLG_REDUCED_TX_PWR_DATA: * @IWM_STA_FLG_DISABLE_TX: set if TX should be disabled * @IWM_STA_FLG_PS: set if STA is in Power Save * @IWM_STA_FLG_INVALID: set if STA is invalid * @IWM_STA_FLG_DLP_EN: Direct Link Protocol is enabled * @IWM_STA_FLG_SET_ALL_KEYS: the current key applies to all key IDs * @IWM_STA_FLG_DRAIN_FLOW: drain flow * @IWM_STA_FLG_PAN: STA is for PAN interface * @IWM_STA_FLG_CLASS_AUTH: * @IWM_STA_FLG_CLASS_ASSOC: * @IWM_STA_FLG_CLASS_MIMO_PROT: * @IWM_STA_FLG_MAX_AGG_SIZE_MSK: maximal size for A-MPDU * @IWM_STA_FLG_AGG_MPDU_DENS_MSK: maximal MPDU density for Tx aggregation * @IWM_STA_FLG_FAT_EN_MSK: support for channel width (for Tx). This flag is * initialised by driver and can be updated by fw upon reception of * action frames that can change the channel width. When cleared the fw * will send all the frames in 20MHz even when FAT channel is requested. * @IWM_STA_FLG_MIMO_EN_MSK: support for MIMO. This flag is initialised by the * driver and can be updated by fw upon reception of action frames. * @IWM_STA_FLG_MFP_EN: Management Frame Protection */ enum iwm_sta_flags { IWM_STA_FLG_REDUCED_TX_PWR_CTRL = (1 << 3), IWM_STA_FLG_REDUCED_TX_PWR_DATA = (1 << 6), IWM_STA_FLG_DISABLE_TX = (1 << 4), IWM_STA_FLG_PS = (1 << 8), IWM_STA_FLG_DRAIN_FLOW = (1 << 12), IWM_STA_FLG_PAN = (1 << 13), IWM_STA_FLG_CLASS_AUTH = (1 << 14), IWM_STA_FLG_CLASS_ASSOC = (1 << 15), IWM_STA_FLG_RTS_MIMO_PROT = (1 << 17), IWM_STA_FLG_MAX_AGG_SIZE_SHIFT = 19, IWM_STA_FLG_MAX_AGG_SIZE_8K = (0 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_16K = (1 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_32K = (2 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_64K = (3 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_128K = (4 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_256K = (5 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_512K = (6 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_1024K = (7 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_MSK = (7 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_SHIFT = 23, IWM_STA_FLG_AGG_MPDU_DENS_2US = (4 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_4US = (5 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_8US = (6 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_16US = (7 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_MSK = (7 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_FAT_EN_20MHZ = (0 << 26), IWM_STA_FLG_FAT_EN_40MHZ = (1 << 26), IWM_STA_FLG_FAT_EN_80MHZ = (2 << 26), IWM_STA_FLG_FAT_EN_160MHZ = (3 << 26), IWM_STA_FLG_FAT_EN_MSK = (3 << 26), IWM_STA_FLG_MIMO_EN_SISO = (0 << 28), IWM_STA_FLG_MIMO_EN_MIMO2 = (1 << 28), IWM_STA_FLG_MIMO_EN_MIMO3 = (2 << 28), IWM_STA_FLG_MIMO_EN_MSK = (3 << 28), }; /** * enum iwm_sta_key_flag - key flags for the ADD_STA host command * @IWM_STA_KEY_FLG_NO_ENC: no encryption * @IWM_STA_KEY_FLG_WEP: WEP encryption algorithm * @IWM_STA_KEY_FLG_CCM: CCMP encryption algorithm * @IWM_STA_KEY_FLG_TKIP: TKIP encryption algorithm * @IWM_STA_KEY_FLG_EXT: extended cipher algorithm (depends on the FW support) * @IWM_STA_KEY_FLG_CMAC: CMAC encryption algorithm * @IWM_STA_KEY_FLG_ENC_UNKNOWN: unknown encryption algorithm * @IWM_STA_KEY_FLG_EN_MSK: mask for encryption algorithmi value * @IWM_STA_KEY_FLG_WEP_KEY_MAP: wep is either a group key (0 - legacy WEP) or from * station info array (1 - n 1X mode) * @IWM_STA_KEY_FLG_KEYID_MSK: the index of the key * @IWM_STA_KEY_NOT_VALID: key is invalid * @IWM_STA_KEY_FLG_WEP_13BYTES: set for 13 bytes WEP key * @IWM_STA_KEY_MULTICAST: set for multical key * @IWM_STA_KEY_MFP: key is used for Management Frame Protection */ enum iwm_sta_key_flag { IWM_STA_KEY_FLG_NO_ENC = (0 << 0), IWM_STA_KEY_FLG_WEP = (1 << 0), IWM_STA_KEY_FLG_CCM = (2 << 0), IWM_STA_KEY_FLG_TKIP = (3 << 0), IWM_STA_KEY_FLG_EXT = (4 << 0), IWM_STA_KEY_FLG_CMAC = (6 << 0), IWM_STA_KEY_FLG_ENC_UNKNOWN = (7 << 0), IWM_STA_KEY_FLG_EN_MSK = (7 << 0), IWM_STA_KEY_FLG_WEP_KEY_MAP = (1 << 3), IWM_STA_KEY_FLG_KEYID_POS = 8, IWM_STA_KEY_FLG_KEYID_MSK = (3 << IWM_STA_KEY_FLG_KEYID_POS), IWM_STA_KEY_NOT_VALID = (1 << 11), IWM_STA_KEY_FLG_WEP_13BYTES = (1 << 12), IWM_STA_KEY_MULTICAST = (1 << 14), IWM_STA_KEY_MFP = (1 << 15), }; /** * enum iwm_sta_modify_flag - indicate to the fw what flag are being changed * @IWM_STA_MODIFY_QUEUE_REMOVAL: this command removes a queue * @IWM_STA_MODIFY_TID_DISABLE_TX: this command modifies %tid_disable_tx * @IWM_STA_MODIFY_TX_RATE: unused * @IWM_STA_MODIFY_ADD_BA_TID: this command modifies %add_immediate_ba_tid * @IWM_STA_MODIFY_REMOVE_BA_TID: this command modifies %remove_immediate_ba_tid * @IWM_STA_MODIFY_SLEEPING_STA_TX_COUNT: this command modifies %sleep_tx_count * @IWM_STA_MODIFY_PROT_TH: * @IWM_STA_MODIFY_QUEUES: modify the queues used by this station */ enum iwm_sta_modify_flag { IWM_STA_MODIFY_QUEUE_REMOVAL = (1 << 0), IWM_STA_MODIFY_TID_DISABLE_TX = (1 << 1), IWM_STA_MODIFY_TX_RATE = (1 << 2), IWM_STA_MODIFY_ADD_BA_TID = (1 << 3), IWM_STA_MODIFY_REMOVE_BA_TID = (1 << 4), IWM_STA_MODIFY_SLEEPING_STA_TX_COUNT = (1 << 5), IWM_STA_MODIFY_PROT_TH = (1 << 6), IWM_STA_MODIFY_QUEUES = (1 << 7), }; #define IWM_STA_MODE_MODIFY 1 /** * enum iwm_sta_sleep_flag - type of sleep of the station * @IWM_STA_SLEEP_STATE_AWAKE: * @IWM_STA_SLEEP_STATE_PS_POLL: * @IWM_STA_SLEEP_STATE_UAPSD: * @IWM_STA_SLEEP_STATE_MOREDATA: set more-data bit on * (last) released frame */ enum iwm_sta_sleep_flag { IWM_STA_SLEEP_STATE_AWAKE = 0, IWM_STA_SLEEP_STATE_PS_POLL = (1 << 0), IWM_STA_SLEEP_STATE_UAPSD = (1 << 1), IWM_STA_SLEEP_STATE_MOREDATA = (1 << 2), }; /* STA ID and color bits definitions */ #define IWM_STA_ID_SEED (0x0f) #define IWM_STA_ID_POS (0) #define IWM_STA_ID_MSK (IWM_STA_ID_SEED << IWM_STA_ID_POS) #define IWM_STA_COLOR_SEED (0x7) #define IWM_STA_COLOR_POS (4) #define IWM_STA_COLOR_MSK (IWM_STA_COLOR_SEED << IWM_STA_COLOR_POS) #define IWM_STA_ID_N_COLOR_GET_COLOR(id_n_color) \ (((id_n_color) & IWM_STA_COLOR_MSK) >> IWM_STA_COLOR_POS) #define IWM_STA_ID_N_COLOR_GET_ID(id_n_color) \ (((id_n_color) & IWM_STA_ID_MSK) >> IWM_STA_ID_POS) #define IWM_STA_KEY_MAX_NUM (16) #define IWM_STA_KEY_IDX_INVALID (0xff) #define IWM_STA_KEY_MAX_DATA_KEY_NUM (4) #define IWM_MAX_GLOBAL_KEYS (4) #define IWM_STA_KEY_LEN_WEP40 (5) #define IWM_STA_KEY_LEN_WEP104 (13) /** * struct iwm_mvm_keyinfo - key information * @key_flags: type %iwm_sta_key_flag * @tkip_rx_tsc_byte2: TSC[2] for key mix ph1 detection * @tkip_rx_ttak: 10-byte unicast TKIP TTAK for Rx * @key_offset: key offset in the fw's key table * @key: 16-byte unicast decryption key * @tx_secur_seq_cnt: initial RSC / PN needed for replay check * @hw_tkip_mic_rx_key: byte: MIC Rx Key - used for TKIP only * @hw_tkip_mic_tx_key: byte: MIC Tx Key - used for TKIP only */ struct iwm_mvm_keyinfo { uint16_t key_flags; uint8_t tkip_rx_tsc_byte2; uint8_t reserved1; uint16_t tkip_rx_ttak[5]; uint8_t key_offset; uint8_t reserved2; uint8_t key[16]; uint64_t tx_secur_seq_cnt; uint64_t hw_tkip_mic_rx_key; uint64_t hw_tkip_mic_tx_key; } __packed; #define IWM_ADD_STA_STATUS_MASK 0xFF #define IWM_ADD_STA_BAID_VALID_MASK 0x8000 #define IWM_ADD_STA_BAID_MASK 0x7F00 #define IWM_ADD_STA_BAID_SHIFT 8 /** * struct iwm_mvm_add_sta_cmd - Add/modify a station in the fw's sta table. * ( REPLY_ADD_STA = 0x18 ) * @add_modify: 1: modify existing, 0: add new station * @awake_acs: * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable * AMPDU for tid x. Set %IWM_STA_MODIFY_TID_DISABLE_TX to change this field. * @mac_id_n_color: the Mac context this station belongs to * @addr[IEEE80211_ADDR_LEN]: station's MAC address * @sta_id: index of station in uCode's station table * @modify_mask: IWM_STA_MODIFY_*, selects which parameters to modify vs. leave * alone. 1 - modify, 0 - don't change. * @station_flags: look at %iwm_sta_flags * @station_flags_msk: what of %station_flags have changed * @add_immediate_ba_tid: tid for which to add block-ack support (Rx) * Set %IWM_STA_MODIFY_ADD_BA_TID to use this field, and also set * add_immediate_ba_ssn. * @remove_immediate_ba_tid: tid for which to remove block-ack support (Rx) * Set %IWM_STA_MODIFY_REMOVE_BA_TID to use this field * @add_immediate_ba_ssn: ssn for the Rx block-ack session. Used together with * add_immediate_ba_tid. * @sleep_tx_count: number of packets to transmit to station even though it is * asleep. Used to synchronise PS-poll and u-APSD responses while ucode * keeps track of STA sleep state. * @sleep_state_flags: Look at %iwm_sta_sleep_flag. * @assoc_id: assoc_id to be sent in VHT PLCP (9-bit), for grp use 0, for AP * mac-addr. * @beamform_flags: beam forming controls * @tfd_queue_msk: tfd queues used by this station * * The device contains an internal table of per-station information, with info * on security keys, aggregation parameters, and Tx rates for initial Tx * attempt and any retries (set by IWM_REPLY_TX_LINK_QUALITY_CMD). * * ADD_STA sets up the table entry for one station, either creating a new * entry, or modifying a pre-existing one. */ struct iwm_mvm_add_sta_cmd { uint8_t add_modify; uint8_t awake_acs; uint16_t tid_disable_tx; uint32_t mac_id_n_color; uint8_t addr[IEEE80211_ADDR_LEN]; /* _STA_ID_MODIFY_INFO_API_S_VER_1 */ uint16_t reserved2; uint8_t sta_id; uint8_t modify_mask; uint16_t reserved3; uint32_t station_flags; uint32_t station_flags_msk; uint8_t add_immediate_ba_tid; uint8_t remove_immediate_ba_tid; uint16_t add_immediate_ba_ssn; uint16_t sleep_tx_count; uint16_t sleep_state_flags; uint16_t assoc_id; uint16_t beamform_flags; uint32_t tfd_queue_msk; } __packed; /* ADD_STA_CMD_API_S_VER_7 */ /** * struct iwm_mvm_add_sta_key_cmd - add/modify sta key * ( IWM_REPLY_ADD_STA_KEY = 0x17 ) * @sta_id: index of station in uCode's station table * @key_offset: key offset in key storage * @key_flags: type %iwm_sta_key_flag * @key: key material data * @key2: key material data * @rx_secur_seq_cnt: RX security sequence counter for the key * @tkip_rx_tsc_byte2: TSC[2] for key mix ph1 detection * @tkip_rx_ttak: 10-byte unicast TKIP TTAK for Rx */ struct iwm_mvm_add_sta_key_cmd { uint8_t sta_id; uint8_t key_offset; uint16_t key_flags; uint8_t key[16]; uint8_t key2[16]; uint8_t rx_secur_seq_cnt[16]; uint8_t tkip_rx_tsc_byte2; uint8_t reserved; uint16_t tkip_rx_ttak[5]; } __packed; /* IWM_ADD_MODIFY_STA_KEY_API_S_VER_1 */ /** * enum iwm_mvm_add_sta_rsp_status - status in the response to ADD_STA command * @IWM_ADD_STA_SUCCESS: operation was executed successfully * @IWM_ADD_STA_STATIONS_OVERLOAD: no room left in the fw's station table * @IWM_ADD_STA_IMMEDIATE_BA_FAILURE: can't add Rx block ack session * @IWM_ADD_STA_MODIFY_NON_EXISTING_STA: driver requested to modify a station * that doesn't exist. */ enum iwm_mvm_add_sta_rsp_status { IWM_ADD_STA_SUCCESS = 0x1, IWM_ADD_STA_STATIONS_OVERLOAD = 0x2, IWM_ADD_STA_IMMEDIATE_BA_FAILURE = 0x4, IWM_ADD_STA_MODIFY_NON_EXISTING_STA = 0x8, }; /** * struct iwm_mvm_rm_sta_cmd - Add / modify a station in the fw's station table * ( IWM_REMOVE_STA = 0x19 ) * @sta_id: the station id of the station to be removed */ struct iwm_mvm_rm_sta_cmd { uint8_t sta_id; uint8_t reserved[3]; } __packed; /* IWM_REMOVE_STA_CMD_API_S_VER_2 */ /** * struct iwm_mvm_mgmt_mcast_key_cmd * ( IWM_MGMT_MCAST_KEY = 0x1f ) * @ctrl_flags: %iwm_sta_key_flag * @IGTK: * @K1: IGTK master key * @K2: IGTK sub key * @sta_id: station ID that support IGTK * @key_id: * @receive_seq_cnt: initial RSC/PN needed for replay check */ struct iwm_mvm_mgmt_mcast_key_cmd { uint32_t ctrl_flags; uint8_t IGTK[16]; uint8_t K1[16]; uint8_t K2[16]; uint32_t key_id; uint32_t sta_id; uint64_t receive_seq_cnt; } __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_1 */ struct iwm_mvm_wep_key { uint8_t key_index; uint8_t key_offset; uint16_t reserved1; uint8_t key_size; uint8_t reserved2[3]; uint8_t key[16]; } __packed; struct iwm_mvm_wep_key_cmd { uint32_t mac_id_n_color; uint8_t num_keys; uint8_t decryption_type; uint8_t flags; uint8_t reserved; struct iwm_mvm_wep_key wep_key[0]; } __packed; /* SEC_CURR_WEP_KEY_CMD_API_S_VER_2 */ /* * END mvm/fw-api-sta.h */ /* * BT coex */ enum iwm_bt_coex_mode { IWM_BT_COEX_DISABLE = 0x0, IWM_BT_COEX_NW = 0x1, IWM_BT_COEX_BT = 0x2, IWM_BT_COEX_WIFI = 0x3, }; /* BT_COEX_MODES_E */ enum iwm_bt_coex_enabled_modules { IWM_BT_COEX_MPLUT_ENABLED = (1 << 0), IWM_BT_COEX_MPLUT_BOOST_ENABLED = (1 << 1), IWM_BT_COEX_SYNC2SCO_ENABLED = (1 << 2), IWM_BT_COEX_CORUN_ENABLED = (1 << 3), IWM_BT_COEX_HIGH_BAND_RET = (1 << 4), }; /* BT_COEX_MODULES_ENABLE_E_VER_1 */ /** * struct iwm_bt_coex_cmd - bt coex configuration command * @mode: enum %iwm_bt_coex_mode * @enabled_modules: enum %iwm_bt_coex_enabled_modules * * The structure is used for the BT_COEX command. */ struct iwm_bt_coex_cmd { uint32_t mode; uint32_t enabled_modules; } __packed; /* BT_COEX_CMD_API_S_VER_6 */ /* * Location Aware Regulatory (LAR) API - MCC updates */ /** * struct iwm_mcc_update_cmd_v1 - Request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: the source from where we got the MCC, see iwm_mcc_source * @reserved: reserved for alignment */ struct iwm_mcc_update_cmd_v1 { uint16_t mcc; uint8_t source_id; uint8_t reserved; } __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_1 */ /** * struct iwm_mcc_update_cmd - Request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: the source from where we got the MCC, see iwm_mcc_source * @reserved: reserved for alignment * @key: integrity key for MCC API OEM testing * @reserved2: reserved */ struct iwm_mcc_update_cmd { uint16_t mcc; uint8_t source_id; uint8_t reserved; uint32_t key; uint32_t reserved2[5]; } __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_2 */ /** * iwm_mcc_update_resp_v1 - response to MCC_UPDATE_CMD. * Contains the new channel control profile map, if changed, and the new MCC * (mobile country code). * The new MCC may be different than what was requested in MCC_UPDATE_CMD. * @status: see &enum iwm_mcc_update_status * @mcc: the new applied MCC * @cap: capabilities for all channels which matches the MCC * @source_id: the MCC source, see iwm_mcc_source * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 * channels, depending on platform) * @channels: channel control data map, DWORD for each channel. Only the first * 16bits are used. */ struct iwm_mcc_update_resp_v1 { uint32_t status; uint16_t mcc; uint8_t cap; uint8_t source_id; uint32_t n_channels; uint32_t channels[0]; } __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_1 */ /** * iwm_mcc_update_resp - response to MCC_UPDATE_CMD. * Contains the new channel control profile map, if changed, and the new MCC * (mobile country code). * The new MCC may be different than what was requested in MCC_UPDATE_CMD. * @status: see &enum iwm_mcc_update_status * @mcc: the new applied MCC * @cap: capabilities for all channels which matches the MCC * @source_id: the MCC source, see iwm_mcc_source * @time: time elapsed from the MCC test start (in 30 seconds TU) * @reserved: reserved. * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 * channels, depending on platform) * @channels: channel control data map, DWORD for each channel. Only the first * 16bits are used. */ struct iwm_mcc_update_resp { uint32_t status; uint16_t mcc; uint8_t cap; uint8_t source_id; uint16_t time; uint16_t reserved; uint32_t n_channels; uint32_t channels[0]; } __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_2 */ /** * struct iwm_mcc_chub_notif - chub notifies of mcc change * (MCC_CHUB_UPDATE_CMD = 0xc9) * The Chub (Communication Hub, CommsHUB) is a HW component that connects to * the cellular and connectivity cores that gets updates of the mcc, and * notifies the ucode directly of any mcc change. * The ucode requests the driver to request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: identity of the change originator, see iwm_mcc_source * @reserved1: reserved for alignment */ struct iwm_mcc_chub_notif { uint16_t mcc; uint8_t source_id; uint8_t reserved1; } __packed; /* LAR_MCC_NOTIFY_S */ enum iwm_mcc_update_status { IWM_MCC_RESP_NEW_CHAN_PROFILE, IWM_MCC_RESP_SAME_CHAN_PROFILE, IWM_MCC_RESP_INVALID, IWM_MCC_RESP_NVM_DISABLED, IWM_MCC_RESP_ILLEGAL, IWM_MCC_RESP_LOW_PRIORITY, IWM_MCC_RESP_TEST_MODE_ACTIVE, IWM_MCC_RESP_TEST_MODE_NOT_ACTIVE, IWM_MCC_RESP_TEST_MODE_DENIAL_OF_SERVICE, }; enum iwm_mcc_source { IWM_MCC_SOURCE_OLD_FW = 0, IWM_MCC_SOURCE_ME = 1, IWM_MCC_SOURCE_BIOS = 2, IWM_MCC_SOURCE_3G_LTE_HOST = 3, IWM_MCC_SOURCE_3G_LTE_DEVICE = 4, IWM_MCC_SOURCE_WIFI = 5, IWM_MCC_SOURCE_RESERVED = 6, IWM_MCC_SOURCE_DEFAULT = 7, IWM_MCC_SOURCE_UNINITIALIZED = 8, IWM_MCC_SOURCE_MCC_API = 9, IWM_MCC_SOURCE_GET_CURRENT = 0x10, IWM_MCC_SOURCE_GETTING_MCC_TEST_MODE = 0x11, }; /** * struct iwm_dts_measurement_notif_v1 - measurements notification * * @temp: the measured temperature * @voltage: the measured voltage */ struct iwm_dts_measurement_notif_v1 { int32_t temp; int32_t voltage; } __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_1*/ /** * struct iwm_dts_measurement_notif_v2 - measurements notification * * @temp: the measured temperature * @voltage: the measured voltage * @threshold_idx: the trip index that was crossed */ struct iwm_dts_measurement_notif_v2 { int32_t temp; int32_t voltage; int32_t threshold_idx; } __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_2 */ /* * Some cherry-picked definitions */ #define IWM_FRAME_LIMIT 64 /* * These functions retrieve specific information from the id field in * the iwm_host_cmd struct which contains the command id, the group id, * and the version of the command and vice versa. */ static inline uint8_t iwm_cmd_opcode(uint32_t cmdid) { return cmdid & 0xff; } static inline uint8_t iwm_cmd_groupid(uint32_t cmdid) { return ((cmdid & 0xff00) >> 8); } static inline uint8_t iwm_cmd_version(uint32_t cmdid) { return ((cmdid & 0xff0000) >> 16); } static inline uint32_t iwm_cmd_id(uint8_t opcode, uint8_t groupid, uint8_t version) { return opcode + (groupid << 8) + (version << 16); } /* make uint16_t wide id out of uint8_t group and opcode */ #define IWM_WIDE_ID(grp, opcode) ((grp << 8) | opcode) /* due to the conversion, this group is special */ #define IWM_ALWAYS_LONG_GROUP 1 struct iwm_cmd_header { uint8_t code; uint8_t flags; uint8_t idx; uint8_t qid; } __packed; struct iwm_cmd_header_wide { uint8_t opcode; uint8_t group_id; uint8_t idx; uint8_t qid; uint16_t length; uint8_t reserved; uint8_t version; } __packed; /** * enum iwm_power_scheme * @IWM_POWER_LEVEL_CAM - Continuously Active Mode * @IWM_POWER_LEVEL_BPS - Balanced Power Save (default) * @IWM_POWER_LEVEL_LP - Low Power */ enum iwm_power_scheme { IWM_POWER_SCHEME_CAM = 1, IWM_POWER_SCHEME_BPS, IWM_POWER_SCHEME_LP }; #define IWM_DEF_CMD_PAYLOAD_SIZE 320 #define IWM_MAX_CMD_PAYLOAD_SIZE ((4096 - 4) - sizeof(struct iwm_cmd_header)) #define IWM_CMD_FAILED_MSK 0x40 /** * struct iwm_device_cmd * * For allocation of the command and tx queues, this establishes the overall * size of the largest command we send to uCode, except for commands that * aren't fully copied and use other TFD space. */ struct iwm_device_cmd { union { struct { struct iwm_cmd_header hdr; uint8_t data[IWM_DEF_CMD_PAYLOAD_SIZE]; }; struct { struct iwm_cmd_header_wide hdr_wide; uint8_t data_wide[IWM_DEF_CMD_PAYLOAD_SIZE - sizeof(struct iwm_cmd_header_wide) + sizeof(struct iwm_cmd_header)]; }; }; } __packed; struct iwm_rx_packet { /* * The first 4 bytes of the RX frame header contain both the RX frame * size and some flags. * Bit fields: * 31: flag flush RB request * 30: flag ignore TC (terminal counter) request * 29: flag fast IRQ request * 28-14: Reserved * 13-00: RX frame size */ uint32_t len_n_flags; struct iwm_cmd_header hdr; uint8_t data[]; } __packed; #define IWM_FH_RSCSR_FRAME_SIZE_MSK 0x00003fff #define IWM_FH_RSCSR_FRAME_INVALID 0x55550000 #define IWM_FH_RSCSR_FRAME_ALIGN 0x40 static inline uint32_t iwm_rx_packet_len(const struct iwm_rx_packet *pkt) { return le32toh(pkt->len_n_flags) & IWM_FH_RSCSR_FRAME_SIZE_MSK; } static inline uint32_t iwm_rx_packet_payload_len(const struct iwm_rx_packet *pkt) { return iwm_rx_packet_len(pkt) - sizeof(pkt->hdr); } #define IWM_MIN_DBM -100 #define IWM_MAX_DBM -33 /* realistic guess */ #define IWM_READ(sc, reg) \ bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (reg)) #define IWM_WRITE(sc, reg, val) \ bus_space_write_4((sc)->sc_st, (sc)->sc_sh, (reg), (val)) #define IWM_WRITE_1(sc, reg, val) \ bus_space_write_1((sc)->sc_st, (sc)->sc_sh, (reg), (val)) #define IWM_SETBITS(sc, reg, mask) \ IWM_WRITE(sc, reg, IWM_READ(sc, reg) | (mask)) #define IWM_CLRBITS(sc, reg, mask) \ IWM_WRITE(sc, reg, IWM_READ(sc, reg) & ~(mask)) #define IWM_BARRIER_WRITE(sc) \ bus_space_barrier((sc)->sc_st, (sc)->sc_sh, 0, (sc)->sc_sz, \ BUS_SPACE_BARRIER_WRITE) #define IWM_BARRIER_READ_WRITE(sc) \ bus_space_barrier((sc)->sc_st, (sc)->sc_sh, 0, (sc)->sc_sz, \ BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE) #endif /* __IF_IWM_REG_H__ */ Index: projects/runtime-coverage/sys/dev/iwm/if_iwmvar.h =================================================================== --- projects/runtime-coverage/sys/dev/iwm/if_iwmvar.h (revision 321584) +++ projects/runtime-coverage/sys/dev/iwm/if_iwmvar.h (revision 321585) @@ -1,574 +1,574 @@ /* $OpenBSD: if_iwmvar.h,v 1.7 2015/03/02 13:51:10 jsg Exp $ */ /* $FreeBSD$ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ struct iwm_rx_radiotap_header { struct ieee80211_radiotap_header wr_ihdr; uint64_t wr_tsft; uint8_t wr_flags; uint8_t wr_rate; uint16_t wr_chan_freq; uint16_t wr_chan_flags; int8_t wr_dbm_antsignal; int8_t wr_dbm_antnoise; } __packed; #define IWM_RX_RADIOTAP_PRESENT \ ((1 << IEEE80211_RADIOTAP_TSFT) | \ (1 << IEEE80211_RADIOTAP_FLAGS) | \ (1 << IEEE80211_RADIOTAP_RATE) | \ (1 << IEEE80211_RADIOTAP_CHANNEL) | \ (1 << IEEE80211_RADIOTAP_DBM_ANTSIGNAL) | \ (1 << IEEE80211_RADIOTAP_DBM_ANTNOISE)) struct iwm_tx_radiotap_header { struct ieee80211_radiotap_header wt_ihdr; uint8_t wt_flags; uint8_t wt_rate; uint16_t wt_chan_freq; uint16_t wt_chan_flags; } __packed; #define IWM_TX_RADIOTAP_PRESENT \ ((1 << IEEE80211_RADIOTAP_FLAGS) | \ (1 << IEEE80211_RADIOTAP_RATE) | \ (1 << IEEE80211_RADIOTAP_CHANNEL)) #define IWM_UCODE_SECTION_MAX 16 /* * fw_status is used to determine if we've already parsed the firmware file * * In addition to the following, status < 0 ==> -error */ #define IWM_FW_STATUS_NONE 0 #define IWM_FW_STATUS_INPROGRESS 1 #define IWM_FW_STATUS_DONE 2 /** * enum iwm_ucode_type * * The type of ucode. * * @IWM_UCODE_REGULAR: Normal runtime ucode * @IWM_UCODE_INIT: Initial ucode * @IWM_UCODE_WOWLAN: Wake on Wireless enabled ucode * @IWM_UCODE_REGULAR_USNIFFER: Normal runtime ucode when using usniffer image */ enum iwm_ucode_type { IWM_UCODE_REGULAR, IWM_UCODE_INIT, IWM_UCODE_WOWLAN, IWM_UCODE_REGULAR_USNIFFER, IWM_UCODE_TYPE_MAX }; struct iwm_ucode_capabilities { uint32_t max_probe_length; uint32_t n_scan_channels; uint32_t flags; uint8_t enabled_api[howmany(IWM_NUM_UCODE_TLV_API, NBBY)]; uint8_t enabled_capa[howmany(IWM_NUM_UCODE_TLV_CAPA, NBBY)]; }; static inline int fw_has_api(const struct iwm_ucode_capabilities *capabilities, unsigned int api) { return isset(capabilities->enabled_api, api); } static inline int fw_has_capa(const struct iwm_ucode_capabilities *capabilities, unsigned int capa) { return isset(capabilities->enabled_capa, capa); } /* one for each uCode image (inst/data, init/runtime/wowlan) */ struct iwm_fw_desc { const void *data; /* vmalloc'ed data */ uint32_t len; /* size in bytes */ uint32_t offset; /* offset in the device */ }; struct iwm_fw_info { const struct firmware *fw_fp; int fw_status; struct iwm_fw_sects { struct iwm_fw_desc fw_sect[IWM_UCODE_SECTION_MAX]; int fw_count; int is_dual_cpus; uint32_t paging_mem_size; } fw_sects[IWM_UCODE_TYPE_MAX]; uint32_t phy_config; uint8_t valid_tx_ant; uint8_t valid_rx_ant; }; struct iwm_nvm_data { int n_hw_addrs; uint8_t hw_addr[IEEE80211_ADDR_LEN]; int sku_cap_band_24GHz_enable; int sku_cap_band_52GHz_enable; int sku_cap_11n_enable; int sku_cap_amt_enable; int sku_cap_ipan_enable; uint8_t radio_cfg_type; uint8_t radio_cfg_step; uint8_t radio_cfg_dash; uint8_t radio_cfg_pnum; uint8_t valid_tx_ant, valid_rx_ant; #define IWM_NUM_CHANNELS 39 #define IWM_NUM_CHANNELS_8000 51 uint16_t nvm_version; uint8_t max_tx_pwr_half_dbm; boolean_t lar_enabled; uint16_t nvm_ch_flags[]; }; /* max bufs per tfd the driver will use */ #define IWM_MAX_CMD_TBS_PER_TFD 2 struct iwm_rx_packet; struct iwm_host_cmd { const void *data[IWM_MAX_CMD_TBS_PER_TFD]; struct iwm_rx_packet *resp_pkt; unsigned long _rx_page_addr; uint32_t _rx_page_order; int handler_status; uint32_t flags; uint32_t id; uint16_t len[IWM_MAX_CMD_TBS_PER_TFD]; uint8_t dataflags[IWM_MAX_CMD_TBS_PER_TFD]; }; /* * DMA glue is from iwn */ typedef caddr_t iwm_caddr_t; typedef void *iwm_hookarg_t; struct iwm_dma_info { bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_addr_t paddr; void *vaddr; bus_size_t size; }; /** * struct iwm_fw_paging * @fw_paging_block: dma memory info * @fw_paging_size: page size */ struct iwm_fw_paging { struct iwm_dma_info fw_paging_block; uint32_t fw_paging_size; }; #define IWM_TX_RING_COUNT 256 #define IWM_TX_RING_LOMARK 192 #define IWM_TX_RING_HIMARK 224 struct iwm_tx_data { bus_dmamap_t map; bus_addr_t cmd_paddr; bus_addr_t scratch_paddr; struct mbuf *m; struct iwm_node *in; int done; }; struct iwm_tx_ring { struct iwm_dma_info desc_dma; struct iwm_dma_info cmd_dma; struct iwm_tfd *desc; struct iwm_device_cmd *cmd; bus_dma_tag_t data_dmat; struct iwm_tx_data data[IWM_TX_RING_COUNT]; int qid; int queued; int cur; }; #define IWM_RX_RING_COUNT 256 /* Linux driver optionally uses 8k buffer */ #define IWM_RBUF_SIZE 4096 #define IWM_MAX_SCATTER 20 struct iwm_rx_data { struct mbuf *m; bus_dmamap_t map; }; struct iwm_rx_ring { struct iwm_dma_info desc_dma; struct iwm_dma_info stat_dma; struct iwm_dma_info buf_dma; uint32_t *desc; struct iwm_rb_status *stat; struct iwm_rx_data data[IWM_RX_RING_COUNT]; bus_dmamap_t spare_map; /* for iwm_rx_addbuf() */ bus_dma_tag_t data_dmat; int cur; }; #define IWM_CMD_RESP_MAX PAGE_SIZE #define IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS 500 #define IWM_MVM_TE_SESSION_PROTECTION_MIN_TIME_MS 400 /* * Command headers are in iwl-trans.h, which is full of all * kinds of other junk, so we just replicate the structures here. * First the software bits: */ enum IWM_CMD_MODE { IWM_CMD_SYNC = 0, IWM_CMD_ASYNC = (1 << 0), IWM_CMD_WANT_SKB = (1 << 1), IWM_CMD_SEND_IN_RFKILL = (1 << 2), }; enum iwm_hcmd_dataflag { IWM_HCMD_DFL_NOCOPY = (1 << 0), IWM_HCMD_DFL_DUP = (1 << 1), }; struct iwm_int_sta { uint32_t sta_id; uint32_t tfd_queue_msk; }; struct iwm_mvm_phy_ctxt { uint16_t id; uint16_t color; uint32_t ref; struct ieee80211_channel *channel; }; struct iwm_bf_data { int bf_enabled; /* filtering */ int ba_enabled; /* abort */ int ave_beacon_signal; int last_cqm_event; }; struct iwm_vap { struct ieee80211vap iv_vap; int is_uploaded; int (*iv_newstate)(struct ieee80211vap *, enum ieee80211_state, int); struct iwm_mvm_phy_ctxt *phy_ctxt; uint16_t id; uint16_t color; boolean_t have_wme; /* * QoS data from net80211, need to store this here * as net80211 has a separate callback but we need * to have the data for the MAC context */ struct { uint16_t cw_min; uint16_t cw_max; uint16_t edca_txop; uint8_t aifsn; } queue_params[WME_NUM_AC]; /* indicates that this interface requires PS to be disabled */ boolean_t ps_disabled; }; #define IWM_VAP(_vap) ((struct iwm_vap *)(_vap)) struct iwm_node { struct ieee80211_node in_ni; /* status "bits" */ int in_assoc; struct iwm_lq_cmd in_lq; uint8_t in_ridx[IEEE80211_RATE_MAXSIZE]; }; #define IWM_NODE(_ni) ((struct iwm_node *)(_ni)) #define IWM_STATION_ID 0 #define IWM_AUX_STA_ID 1 #define IWM_DEFAULT_MACID 0 #define IWM_DEFAULT_COLOR 0 #define IWM_DEFAULT_TSFID 0 #define IWM_ICT_SIZE 4096 #define IWM_ICT_COUNT (IWM_ICT_SIZE / sizeof (uint32_t)) #define IWM_ICT_PADDR_SHIFT 12 struct iwm_cfg; struct iwm_softc { device_t sc_dev; uint32_t sc_debug; int sc_attached; struct mtx sc_mtx; struct mbufq sc_snd; struct ieee80211com sc_ic; struct ieee80211_ratectl_tx_status sc_txs; int sc_flags; #define IWM_FLAG_USE_ICT (1 << 0) #define IWM_FLAG_HW_INITED (1 << 1) #define IWM_FLAG_STOPPED (1 << 2) #define IWM_FLAG_RFKILL (1 << 3) #define IWM_FLAG_BUSY (1 << 4) #define IWM_FLAG_SCANNING (1 << 5) #define IWM_FLAG_SCAN_RUNNING (1 << 6) struct intr_config_hook sc_preinit_hook; struct callout sc_watchdog_to; struct callout sc_led_blink_to; struct task init_task; struct resource *sc_irq; struct resource *sc_mem; bus_space_tag_t sc_st; bus_space_handle_t sc_sh; bus_size_t sc_sz; bus_dma_tag_t sc_dmat; void *sc_ih; /* TX scheduler rings. */ struct iwm_dma_info sched_dma; uint32_t scd_base_addr; /* TX/RX rings. */ struct iwm_tx_ring txq[IWM_MVM_MAX_QUEUES]; struct iwm_rx_ring rxq; int qfullmsk; int sc_sf_state; /* ICT table. */ struct iwm_dma_info ict_dma; int ict_cur; int sc_hw_rev; int sc_hw_id; struct iwm_dma_info kw_dma; struct iwm_dma_info fw_dma; int sc_fw_chunk_done; enum iwm_ucode_type cur_ucode; int ucode_loaded; char sc_fwver[32]; struct iwm_ucode_capabilities ucode_capa; char sc_fw_mcc[3]; int sc_intmask; /* * So why do we need a separate stopped flag and a generation? * the former protects the device from issuing commands when it's * stopped (duh). The latter protects against race from a very * fast stop/unstop cycle where threads waiting for responses do * not have a chance to run in between. Notably: we want to stop * the device from interrupt context when it craps out, so we * don't have the luxury of waiting for quiescense. */ int sc_generation; struct iwm_fw_info sc_fw; struct iwm_tlv_calib_ctrl sc_default_calib[IWM_UCODE_TYPE_MAX]; const struct iwm_cfg *cfg; struct iwm_nvm_data *nvm_data; struct iwm_phy_db *sc_phy_db; struct iwm_bf_data sc_bf; int sc_tx_timer; int sc_scan_last_antenna; int sc_fixed_ridx; int sc_staid; int sc_nodecolor; uint8_t sc_cmd_resp[IWM_CMD_RESP_MAX]; int sc_wantresp; struct task sc_es_task; struct iwm_rx_phy_info sc_last_phy_info; int sc_ampdu_ref; struct iwm_int_sta sc_aux_sta; /* phy contexts. we only use the first one */ struct iwm_mvm_phy_ctxt sc_phyctxt[IWM_NUM_PHY_CTX]; - struct iwm_notif_statistics sc_stats; + struct iwm_notif_statistics_v10 sc_stats; int sc_noise; caddr_t sc_drvbpf; struct iwm_rx_radiotap_header sc_rxtap; struct iwm_tx_radiotap_header sc_txtap; int sc_max_rssi; struct iwm_notif_wait_data *sc_notif_wait; int cmd_hold_nic_awake; /* Firmware status */ uint32_t error_event_table; uint32_t log_event_table; uint32_t umac_error_event_table; int support_umac_log; /* * Paging parameters - All of the parameters should be set by the * opmode when paging is enabled */ struct iwm_fw_paging fw_paging_db[IWM_NUM_OF_FW_PAGING_BLOCKS]; uint16_t num_of_paging_blk; uint16_t num_of_pages_in_last_blk; boolean_t last_ebs_successful; /* last smart fifo state that was successfully sent to firmware */ enum iwm_sf_state sf_state; /* Indicate if device power save is allowed */ boolean_t sc_ps_disabled; }; #define IWM_LOCK_INIT(_sc) \ mtx_init(&(_sc)->sc_mtx, device_get_nameunit((_sc)->sc_dev), \ MTX_NETWORK_LOCK, MTX_DEF); #define IWM_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define IWM_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define IWM_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) Index: projects/runtime-coverage/sys/dev/ixgbe/ix_txrx.c =================================================================== --- projects/runtime-coverage/sys/dev/ixgbe/ix_txrx.c (revision 321584) +++ projects/runtime-coverage/sys/dev/ixgbe/ix_txrx.c (revision 321585) @@ -1,2199 +1,2199 @@ /****************************************************************************** Copyright (c) 2001-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ /*$FreeBSD$*/ #ifndef IXGBE_STANDALONE_BUILD #include "opt_inet.h" #include "opt_inet6.h" #include "opt_rss.h" #endif #include "ixgbe.h" /* * HW RSC control: * this feature only works with * IPv4, and only on 82599 and later. * Also this will cause IP forwarding to * fail and that can't be controlled by * the stack as LRO can. For all these * reasons I've deemed it best to leave * this off and not bother with a tuneable * interface, this would need to be compiled * to enable. */ static bool ixgbe_rsc_enable = FALSE; /* * For Flow Director: this is the * number of TX packets we sample * for the filter pool, this means * every 20th packet will be probed. * * This feature can be disabled by * setting this to 0. */ static int atr_sample_rate = 20; /************************************************************************ * Local Function prototypes ************************************************************************/ static void ixgbe_setup_transmit_ring(struct tx_ring *); static void ixgbe_free_transmit_buffers(struct tx_ring *); static int ixgbe_setup_receive_ring(struct rx_ring *); static void ixgbe_free_receive_buffers(struct rx_ring *); static void ixgbe_rx_checksum(u32, struct mbuf *, u32); static void ixgbe_refresh_mbufs(struct rx_ring *, int); static int ixgbe_xmit(struct tx_ring *, struct mbuf **); static int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); static int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); static __inline void ixgbe_rx_discard(struct rx_ring *, int); static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, struct mbuf *, u32); static int ixgbe_dma_malloc(struct adapter *, bus_size_t, struct ixgbe_dma_alloc *, int); static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); /************************************************************************ * ixgbe_legacy_start_locked - Transmit entry point * * Called by the stack to initiate a transmit. * The driver will remain in this routine as long as there are * packets to transmit and transmit resources are available. * In case resources are not available, the stack is notified * and the packet is requeued. ************************************************************************/ int ixgbe_legacy_start_locked(struct ifnet *ifp, struct tx_ring *txr) { struct mbuf *m_head; struct adapter *adapter = txr->adapter; IXGBE_TX_LOCK_ASSERT(txr); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return (ENETDOWN); if (!adapter->link_active) return (ENETDOWN); while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) break; IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; if (ixgbe_xmit(txr, &m_head)) { if (m_head != NULL) IFQ_DRV_PREPEND(&ifp->if_snd, m_head); break; } /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); } return IXGBE_SUCCESS; } /* ixgbe_legacy_start_locked */ /************************************************************************ * ixgbe_legacy_start * * Called by the stack, this always uses the first tx ring, * and should not be used with multiqueue tx enabled. ************************************************************************/ void ixgbe_legacy_start(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { IXGBE_TX_LOCK(txr); ixgbe_legacy_start_locked(ifp, txr); IXGBE_TX_UNLOCK(txr); } } /* ixgbe_legacy_start */ /************************************************************************ * ixgbe_mq_start - Multiqueue Transmit Entry Point * * (if_transmit function) ************************************************************************/ int ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; struct ix_queue *que; struct tx_ring *txr; int i, err = 0; uint32_t bucket_id; /* * When doing RSS, map it to the same outbound queue * as the incoming flow would be mapped to. * * If everything is setup correctly, it should be the * same bucket that the current CPU we're on is. */ if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { if ((adapter->feat_en & IXGBE_FEATURE_RSS) && (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), &bucket_id) == 0)) { i = bucket_id % adapter->num_queues; #ifdef IXGBE_DEBUG if (bucket_id > adapter->num_queues) if_printf(ifp, "bucket_id (%d) > num_queues (%d)\n", bucket_id, adapter->num_queues); #endif } else i = m->m_pkthdr.flowid % adapter->num_queues; } else i = curcpu % adapter->num_queues; /* Check for a hung queue and pick alternative */ if (((1 << i) & adapter->active_queues) == 0) i = ffsl(adapter->active_queues); txr = &adapter->tx_rings[i]; que = &adapter->queues[i]; err = drbr_enqueue(ifp, txr->br, m); if (err) return (err); if (IXGBE_TX_TRYLOCK(txr)) { ixgbe_mq_start_locked(ifp, txr); IXGBE_TX_UNLOCK(txr); } else taskqueue_enqueue(que->tq, &txr->txq_task); return (0); } /* ixgbe_mq_start */ /************************************************************************ * ixgbe_mq_start_locked ************************************************************************/ int ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) { struct mbuf *next; int enqueued = 0, err = 0; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return (ENETDOWN); if (txr->adapter->link_active == 0) return (ENETDOWN); /* Process the queue */ #if __FreeBSD_version < 901504 next = drbr_dequeue(ifp, txr->br); while (next != NULL) { if ((err = ixgbe_xmit(txr, &next)) != 0) { if (next != NULL) err = drbr_enqueue(ifp, txr->br, next); #else while ((next = drbr_peek(ifp, txr->br)) != NULL) { err = ixgbe_xmit(txr, &next); if (err != 0) { if (next == NULL) drbr_advance(ifp, txr->br); else drbr_putback(ifp, txr->br, next); #endif break; } #if __FreeBSD_version >= 901504 drbr_advance(ifp, txr->br); #endif enqueued++; #if __FreeBSD_version >= 1100036 /* * Since we're looking at the tx ring, we can check * to see if we're a VF by examing our tail register * address. */ if ((txr->adapter->feat_en & IXGBE_FEATURE_VF) && (next->m_flags & M_MCAST)) if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); #endif /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, next); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; #if __FreeBSD_version < 901504 next = drbr_dequeue(ifp, txr->br); #endif } if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD(txr->adapter)) ixgbe_txeof(txr); return (err); } /* ixgbe_mq_start_locked */ /************************************************************************ * ixgbe_deferred_mq_start * * Called from a taskqueue to drain queued transmit packets. ************************************************************************/ void ixgbe_deferred_mq_start(void *arg, int pending) { struct tx_ring *txr = arg; struct adapter *adapter = txr->adapter; struct ifnet *ifp = adapter->ifp; IXGBE_TX_LOCK(txr); if (!drbr_empty(ifp, txr->br)) ixgbe_mq_start_locked(ifp, txr); IXGBE_TX_UNLOCK(txr); } /* ixgbe_deferred_mq_start */ /************************************************************************ * ixgbe_qflush - Flush all ring buffers ************************************************************************/ void ixgbe_qflush(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; struct mbuf *m; for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_TX_LOCK(txr); while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) m_freem(m); IXGBE_TX_UNLOCK(txr); } if_qflush(ifp); } /* ixgbe_qflush */ /************************************************************************ * ixgbe_xmit * * Maps the mbufs to tx descriptors, allowing the * TX engine to transmit the packets. * * Return 0 on success, positive on failure ************************************************************************/ static int ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *txbuf; union ixgbe_adv_tx_desc *txd = NULL; struct mbuf *m_head; int i, j, error, nsegs; int first; u32 olinfo_status = 0, cmd_type_len; bool remap = TRUE; bus_dma_segment_t segs[adapter->num_segs]; bus_dmamap_t map; m_head = *m_headp; /* Basic descriptor defines */ cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); if (m_head->m_flags & M_VLANTAG) cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; /* * Important to capture the first descriptor * used because it will contain the index of * the one we tell the hardware to report back */ first = txr->next_avail_desc; txbuf = &txr->tx_buffers[first]; map = txbuf->map; /* * Map the packet for DMA. */ retry: error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (__predict_false(error)) { struct mbuf *m; switch (error) { case EFBIG: /* Try it again? - one try */ if (remap == TRUE) { remap = FALSE; /* * XXX: m_defrag will choke on * non-MCLBYTES-sized clusters */ m = m_defrag(*m_headp, M_NOWAIT); if (m == NULL) { adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); } *m_headp = m; goto retry; } else return (error); case ENOMEM: txr->no_tx_dma_setup++; return (error); default: txr->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } } /* Make certain there are enough descriptors */ if (txr->tx_avail < (nsegs + 2)) { txr->no_desc_avail++; bus_dmamap_unload(txr->txtag, map); return (ENOBUFS); } m_head = *m_headp; /* * Set up the appropriate offload context * this will consume the first descriptor */ error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); if (__predict_false(error)) { if (error == ENOBUFS) *m_headp = NULL; return (error); } /* Do the flow director magic */ if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && (txr->atr_sample) && (!adapter->fdir_reinit)) { ++txr->atr_count; if (txr->atr_count >= atr_sample_rate) { ixgbe_atr(txr, m_head); txr->atr_count = 0; } } olinfo_status |= IXGBE_ADVTXD_CC; i = txr->next_avail_desc; for (j = 0; j < nsegs; j++) { bus_size_t seglen; bus_addr_t segaddr; txbuf = &txr->tx_buffers[i]; txd = &txr->tx_base[i]; seglen = segs[j].ds_len; segaddr = htole64(segs[j].ds_addr); txd->read.buffer_addr = segaddr; txd->read.cmd_type_len = htole32(txr->txd_cmd | cmd_type_len | seglen); txd->read.olinfo_status = htole32(olinfo_status); if (++i == txr->num_desc) i = 0; } txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); txr->tx_avail -= nsegs; txr->next_avail_desc = i; txbuf->m_head = m_head; /* * Here we swap the map so the last descriptor, * which gets the completion interrupt has the * real map, and the first descriptor gets the * unused map from this descriptor. */ txr->tx_buffers[first].map = txbuf->map; txbuf->map = map; bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); /* Set the EOP descriptor that will be marked done */ txbuf = &txr->tx_buffers[first]; txbuf->eop = txd; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* * Advance the Transmit Descriptor Tail (Tdt), this tells the * hardware that this frame is available to transmit. */ ++txr->total_packets; IXGBE_WRITE_REG(&adapter->hw, txr->tail, i); /* Mark queue as having work */ if (txr->busy == 0) txr->busy = 1; return (0); } /* ixgbe_xmit */ /************************************************************************ * ixgbe_allocate_transmit_buffers * * Allocate memory for tx_buffer structures. The tx_buffer stores all * the information needed to transmit a packet on the wire. This is * called only once at attach, setup is done every reset. ************************************************************************/ static int ixgbe_allocate_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; device_t dev = adapter->dev; struct ixgbe_tx_buf *txbuf; int error, i; /* * Setup DMA descriptor areas. */ error = bus_dma_tag_create( /* parent */ bus_get_dma_tag(adapter->dev), /* alignment */ 1, /* bounds */ 0, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ IXGBE_TSO_SIZE, /* nsegments */ adapter->num_segs, /* maxsegsize */ PAGE_SIZE, /* flags */ 0, /* lockfunc */ NULL, /* lockfuncarg */ NULL, &txr->txtag); if (error != 0) { device_printf(dev, "Unable to allocate TX DMA tag\n"); goto fail; } txr->tx_buffers = (struct ixgbe_tx_buf *)malloc(sizeof(struct ixgbe_tx_buf) * adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); if (txr->tx_buffers == NULL) { device_printf(dev, "Unable to allocate tx_buffer memory\n"); error = ENOMEM; goto fail; } /* Create the descriptor buffer dma maps */ txbuf = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); if (error != 0) { device_printf(dev, "Unable to create TX DMA map\n"); goto fail; } } return 0; fail: /* We free all, it handles case where we are in the middle */ ixgbe_free_transmit_structures(adapter); return (error); } /* ixgbe_allocate_transmit_buffers */ /************************************************************************ * ixgbe_setup_transmit_ring - Initialize a transmit ring. ************************************************************************/ static void ixgbe_setup_transmit_ring(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *txbuf; #ifdef DEV_NETMAP struct netmap_adapter *na = NA(adapter->ifp); struct netmap_slot *slot; #endif /* DEV_NETMAP */ /* Clear the old ring contents */ IXGBE_TX_LOCK(txr); #ifdef DEV_NETMAP if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { /* * (under lock): if in netmap mode, do some consistency * checks and set slot to entry 0 of the netmap ring. */ slot = netmap_reset(na, NR_TX, txr->me, 0); } #endif /* DEV_NETMAP */ bzero((void *)txr->tx_base, (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); /* Reset indices */ txr->next_avail_desc = 0; txr->next_to_clean = 0; /* Free any existing tx buffers. */ txbuf = txr->tx_buffers; for (int i = 0; i < txr->num_desc; i++, txbuf++) { if (txbuf->m_head != NULL) { bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, txbuf->map); m_freem(txbuf->m_head); txbuf->m_head = NULL; } #ifdef DEV_NETMAP /* * In netmap mode, set the map for the packet buffer. * NOTE: Some drivers (not this one) also need to set * the physical buffer address in the NIC ring. * Slots in the netmap ring (indexed by "si") are * kring->nkr_hwofs positions "ahead" wrt the * corresponding slot in the NIC ring. In some drivers * (not here) nkr_hwofs can be negative. Function * netmap_idx_n2k() handles wraparounds properly. */ if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); } #endif /* DEV_NETMAP */ /* Clear the EOP descriptor pointer */ txbuf->eop = NULL; } /* Set the rate at which we sample packets */ if (adapter->feat_en & IXGBE_FEATURE_FDIR) txr->atr_sample = atr_sample_rate; /* Set number of descriptors available */ txr->tx_avail = adapter->num_tx_desc; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); IXGBE_TX_UNLOCK(txr); } /* ixgbe_setup_transmit_ring */ /************************************************************************ * ixgbe_setup_transmit_structures - Initialize all transmit rings. ************************************************************************/ int ixgbe_setup_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) ixgbe_setup_transmit_ring(txr); return (0); } /* ixgbe_setup_transmit_structures */ /************************************************************************ * ixgbe_free_transmit_structures - Free all transmit rings. ************************************************************************/ void ixgbe_free_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_TX_LOCK(txr); ixgbe_free_transmit_buffers(txr); ixgbe_dma_free(adapter, &txr->txdma); IXGBE_TX_UNLOCK(txr); IXGBE_TX_LOCK_DESTROY(txr); } free(adapter->tx_rings, M_DEVBUF); } /* ixgbe_free_transmit_structures */ /************************************************************************ * ixgbe_free_transmit_buffers * * Free transmit ring related data structures. ************************************************************************/ static void ixgbe_free_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *tx_buffer; int i; INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin"); if (txr->tx_buffers == NULL) return; tx_buffer = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { if (tx_buffer->m_head != NULL) { bus_dmamap_sync(txr->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; if (tx_buffer->map != NULL) { bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } else if (tx_buffer->map != NULL) { bus_dmamap_unload(txr->txtag, tx_buffer->map); bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } if (txr->br != NULL) buf_ring_free(txr->br, M_DEVBUF); if (txr->tx_buffers != NULL) { free(txr->tx_buffers, M_DEVBUF); txr->tx_buffers = NULL; } if (txr->txtag != NULL) { bus_dma_tag_destroy(txr->txtag); txr->txtag = NULL; } } /* ixgbe_free_transmit_buffers */ /************************************************************************ * ixgbe_tx_ctx_setup * * Advanced Context Descriptor setup for VLAN, CSUM or TSO ************************************************************************/ static int ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, u32 *cmd_type_len, u32 *olinfo_status) { struct ixgbe_adv_tx_context_desc *TXD; struct ether_vlan_header *eh; #ifdef INET struct ip *ip; #endif #ifdef INET6 struct ip6_hdr *ip6; #endif int ehdrlen, ip_hlen = 0; int offload = TRUE; int ctxd = txr->next_avail_desc; u32 vlan_macip_lens = 0; u32 type_tucmd_mlhl = 0; u16 vtag = 0; u16 etype; u8 ipproto = 0; caddr_t l3d; /* First check if TSO is to be used */ if (mp->m_pkthdr.csum_flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) offload = FALSE; /* Indicate the whole packet as payload when not doing TSO */ *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; /* Now ready a context descriptor */ TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; /* * In advanced descriptors the vlan tag must * be placed into the context descriptor. Hence * we need to make one even if not doing offloads. */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); } else if (!(txr->adapter->feat_en & IXGBE_FEATURE_NEEDS_CTXD) && (offload == FALSE)) return (0); /* * Determine where frame payload starts. * Jump over vlan headers if already present, * helpful for QinQ too. */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } /* Set the ether header length */ vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; if (offload == FALSE) goto no_offloads; /* * If the first mbuf only includes the ethernet header, * jump to the next one * XXX: This assumes the stack splits mbufs containing headers * on header boundaries * XXX: And assumes the entire IP header is contained in one mbuf */ if (mp->m_len == ehdrlen && mp->m_next) l3d = mtod(mp->m_next, caddr_t); else l3d = mtod(mp, caddr_t) + ehdrlen; switch (etype) { #ifdef INET case ETHERTYPE_IP: ip = (struct ip *)(l3d); ip_hlen = ip->ip_hl << 2; ipproto = ip->ip_p; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; /* Insert IPv4 checksum into data descriptors */ if (mp->m_pkthdr.csum_flags & CSUM_IP) { ip->ip_sum = 0; *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; } break; #endif #ifdef INET6 case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(l3d); ip_hlen = sizeof(struct ip6_hdr); ipproto = ip6->ip6_nxt; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; break; #endif default: offload = FALSE; break; } vlan_macip_lens |= ip_hlen; /* No support for offloads for non-L4 next headers */ switch (ipproto) { case IPPROTO_TCP: if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; else offload = false; break; case IPPROTO_UDP: if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; else offload = false; break; case IPPROTO_SCTP: if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; else offload = false; break; default: offload = false; break; } if (offload) /* Insert L4 checksum into data descriptors */ *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; no_offloads: type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; /* Now copy bits into descriptor */ TXD->vlan_macip_lens = htole32(vlan_macip_lens); TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); TXD->seqnum_seed = htole32(0); TXD->mss_l4len_idx = htole32(0); /* We've consumed the first desc, adjust counters */ if (++ctxd == txr->num_desc) ctxd = 0; txr->next_avail_desc = ctxd; --txr->tx_avail; return (0); } /* ixgbe_tx_ctx_setup */ /************************************************************************ * ixgbe_tso_setup * * Setup work for hardware segmentation offload (TSO) on * adapters using advanced tx descriptors ************************************************************************/ static int ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *cmd_type_len, u32 *olinfo_status) { struct ixgbe_adv_tx_context_desc *TXD; struct ether_vlan_header *eh; #ifdef INET6 struct ip6_hdr *ip6; #endif #ifdef INET struct ip *ip; #endif struct tcphdr *th; int ctxd, ehdrlen, ip_hlen, tcp_hlen; u32 vlan_macip_lens = 0; u32 type_tucmd_mlhl = 0; u32 mss_l4len_idx = 0, paylen; u16 vtag = 0, eh_type; /* * Determine where frame payload starts. * Jump over vlan headers if already present */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; eh_type = eh->evl_proto; } else { ehdrlen = ETHER_HDR_LEN; eh_type = eh->evl_encap_proto; } switch (ntohs(eh_type)) { #ifdef INET case ETHERTYPE_IP: ip = (struct ip *)(mp->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) return (ENXIO); ip->ip_sum = 0; ip_hlen = ip->ip_hl << 2; th = (struct tcphdr *)((caddr_t)ip + ip_hlen); th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; /* Tell transmit desc to also do IPv4 checksum. */ *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; break; #endif #ifdef INET6 case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); /* XXX-BZ For now we do not pretend to support ext. hdrs. */ if (ip6->ip6_nxt != IPPROTO_TCP) return (ENXIO); ip_hlen = sizeof(struct ip6_hdr); th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; break; #endif default: panic("%s: CSUM_TSO but no supported IP version (0x%04x)", __func__, ntohs(eh_type)); break; } ctxd = txr->next_avail_desc; TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; tcp_hlen = th->th_off << 2; /* This is used in the transmit desc in encap */ paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; /* VLAN MACLEN IPLEN */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); } vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; vlan_macip_lens |= ip_hlen; TXD->vlan_macip_lens = htole32(vlan_macip_lens); /* ADV DTYPE TUCMD */ type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); /* MSS L4LEN IDX */ mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); TXD->mss_l4len_idx = htole32(mss_l4len_idx); TXD->seqnum_seed = htole32(0); if (++ctxd == txr->num_desc) ctxd = 0; txr->tx_avail--; txr->next_avail_desc = ctxd; *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; ++txr->tso_tx; return (0); } /* ixgbe_tso_setup */ /************************************************************************ * ixgbe_txeof * * Examine each tx_buffer in the used queue. If the hardware is done * processing the packet then free associated resources. The * tx_buffer is put back on the free queue. ************************************************************************/ void ixgbe_txeof(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *buf; union ixgbe_adv_tx_desc *txd; u32 work, processed = 0; u32 limit = adapter->tx_process_limit; mtx_assert(&txr->tx_mtx, MA_OWNED); #ifdef DEV_NETMAP if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && (adapter->ifp->if_capenable & IFCAP_NETMAP)) { struct netmap_adapter *na = NA(adapter->ifp); struct netmap_kring *kring = &na->tx_rings[txr->me]; txd = txr->tx_base; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD); /* * In netmap mode, all the work is done in the context * of the client thread. Interrupt handlers only wake up * clients, which may be sleeping on individual rings * or on a global resource for all rings. * To implement tx interrupt mitigation, we wake up the client * thread roughly every half ring, even if the NIC interrupts * more frequently. This is implemented as follows: * - ixgbe_txsync() sets kring->nr_kflags with the index of * the slot that should wake up the thread (nkr_num_slots * means the user thread should not be woken up); * - the driver ignores tx interrupts unless netmap_mitigate=0 * or the slot has the DD bit set. */ if (!netmap_mitigate || (kring->nr_kflags < kring->nkr_num_slots && txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) { netmap_tx_irq(adapter->ifp, txr->me); } return; } #endif /* DEV_NETMAP */ if (txr->tx_avail == txr->num_desc) { txr->busy = 0; return; } /* Get work starting point */ work = txr->next_to_clean; buf = &txr->tx_buffers[work]; txd = &txr->tx_base[work]; work -= txr->num_desc; /* The distance to ring end */ bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD); do { union ixgbe_adv_tx_desc *eop = buf->eop; if (eop == NULL) /* No work */ break; if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0) break; /* I/O not complete */ if (buf->m_head) { txr->bytes += buf->m_head->m_pkthdr.len; bus_dmamap_sync(txr->txtag, buf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, buf->map); m_freem(buf->m_head); buf->m_head = NULL; } buf->eop = NULL; ++txr->tx_avail; /* We clean the range if multi segment */ while (txd != eop) { ++txd; ++buf; ++work; /* wrap the ring? */ if (__predict_false(!work)) { work -= txr->num_desc; buf = txr->tx_buffers; txd = txr->tx_base; } if (buf->m_head) { txr->bytes += buf->m_head->m_pkthdr.len; bus_dmamap_sync(txr->txtag, buf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, buf->map); m_freem(buf->m_head); buf->m_head = NULL; } ++txr->tx_avail; buf->eop = NULL; } ++txr->packets; ++processed; /* Try the next packet */ ++txd; ++buf; ++work; /* reset with a wrap */ if (__predict_false(!work)) { work -= txr->num_desc; buf = txr->tx_buffers; txd = txr->tx_base; } prefetch(txd); } while (__predict_true(--limit)); bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); work += txr->num_desc; txr->next_to_clean = work; /* * Queue Hang detection, we know there's * work outstanding or the first return * would have been taken, so increment busy * if nothing managed to get cleaned, then * in local_timer it will be checked and * marked as HUNG if it exceeds a MAX attempt. */ if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG)) ++txr->busy; /* * If anything gets cleaned we reset state to 1, * note this will turn off HUNG if its set. */ if (processed) txr->busy = 1; if (txr->tx_avail == txr->num_desc) txr->busy = 0; return; } /* ixgbe_txeof */ /************************************************************************ * ixgbe_rsc_count * * Used to detect a descriptor that has been merged by Hardware RSC. ************************************************************************/ static inline u32 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) { return (le32toh(rx->wb.lower.lo_dword.data) & IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; } /* ixgbe_rsc_count */ /************************************************************************ * ixgbe_setup_hw_rsc * * Initialize Hardware RSC (LRO) feature on 82599 * for an RX ring, this is toggled by the LRO capability * even though it is transparent to the stack. * * NOTE: Since this HW feature only works with IPv4 and * testing has shown soft LRO to be as effective, * this feature will be disabled by default. ************************************************************************/ static void ixgbe_setup_hw_rsc(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; struct ixgbe_hw *hw = &adapter->hw; u32 rscctrl, rdrxctl; /* If turning LRO/RSC off we need to disable it */ if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) { rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); rscctrl &= ~IXGBE_RSCCTL_RSCEN; return; } rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; #ifdef DEV_NETMAP /* Always strip CRC unless Netmap disabled it */ if (!(adapter->feat_en & IXGBE_FEATURE_NETMAP) || !(adapter->ifp->if_capenable & IFCAP_NETMAP) || ix_crcstrip) #endif /* DEV_NETMAP */ rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; rdrxctl |= IXGBE_RDRXCTL_RSCACKC; IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); rscctrl |= IXGBE_RSCCTL_RSCEN; /* * Limit the total number of descriptors that * can be combined, so it does not exceed 64K */ if (rxr->mbuf_sz == MCLBYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_16; else if (rxr->mbuf_sz == MJUMPAGESIZE) rscctrl |= IXGBE_RSCCTL_MAXDESC_8; else if (rxr->mbuf_sz == MJUM9BYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_4; else /* Using 16K cluster */ rscctrl |= IXGBE_RSCCTL_MAXDESC_1; IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); /* Enable TCP header recognition */ IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR)); /* Disable RSC for ACK packets */ IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); rxr->hw_rsc = TRUE; } /* ixgbe_setup_hw_rsc */ /************************************************************************ * ixgbe_refresh_mbufs * * Refresh mbuf buffers for RX descriptor rings * - now keeps its own state so discards due to resource * exhaustion are unnecessary, if an mbuf cannot be obtained * it just returns, keeping its placeholder, thus it can simply * be recalled to try again. ************************************************************************/ static void ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) { struct adapter *adapter = rxr->adapter; struct ixgbe_rx_buf *rxbuf; struct mbuf *mp; bus_dma_segment_t seg[1]; int i, j, nsegs, error; bool refreshed = FALSE; i = j = rxr->next_to_refresh; /* Control the loop with one beyond */ if (++j == rxr->num_desc) j = 0; while (j != limit) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->buf == NULL) { mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rxr->mbuf_sz); if (mp == NULL) goto update; if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) m_adj(mp, ETHER_ALIGN); } else mp = rxbuf->buf; mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* If we're dealing with an mbuf that was copied rather * than replaced, there's no need to go through busdma. */ if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { /* Get the memory mapping */ bus_dmamap_unload(rxr->ptag, rxbuf->pmap); error = bus_dmamap_load_mbuf_sg(rxr->ptag, rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { printf("Refresh mbufs: payload dmamap load failure - %d\n", error); m_free(mp); rxbuf->buf = NULL; goto update; } rxbuf->buf = mp; bus_dmamap_sync(rxr->ptag, rxbuf->pmap, BUS_DMASYNC_PREREAD); rxbuf->addr = rxr->rx_base[i].read.pkt_addr = htole64(seg[0].ds_addr); } else { rxr->rx_base[i].read.pkt_addr = rxbuf->addr; rxbuf->flags &= ~IXGBE_RX_COPY; } refreshed = TRUE; /* Next is precalculated */ i = j; rxr->next_to_refresh = i; if (++j == rxr->num_desc) j = 0; } update: if (refreshed) /* Update hardware tail index */ IXGBE_WRITE_REG(&adapter->hw, rxr->tail, rxr->next_to_refresh); return; } /* ixgbe_refresh_mbufs */ /************************************************************************ * ixgbe_allocate_receive_buffers * * Allocate memory for rx_buffer structures. Since we use one * rx_buffer per received packet, the maximum number of rx_buffer's * that we'll need is equal to the number of receive descriptors * that we've allocated. ************************************************************************/ static int ixgbe_allocate_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; device_t dev = adapter->dev; struct ixgbe_rx_buf *rxbuf; int bsize, error; bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; rxr->rx_buffers = (struct ixgbe_rx_buf *)malloc(bsize, M_DEVBUF, M_NOWAIT | M_ZERO); if (rxr->rx_buffers == NULL) { device_printf(dev, "Unable to allocate rx_buffer memory\n"); error = ENOMEM; goto fail; } error = bus_dma_tag_create( /* parent */ bus_get_dma_tag(dev), /* alignment */ 1, /* bounds */ 0, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ MJUM16BYTES, /* nsegments */ 1, /* maxsegsize */ MJUM16BYTES, /* flags */ 0, /* lockfunc */ NULL, /* lockfuncarg */ NULL, &rxr->ptag); if (error != 0) { device_printf(dev, "Unable to create RX DMA tag\n"); goto fail; } for (int i = 0; i < rxr->num_desc; i++, rxbuf++) { rxbuf = &rxr->rx_buffers[i]; error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); if (error) { device_printf(dev, "Unable to create RX dma map\n"); goto fail; } } return (0); fail: /* Frees all, but can handle partial completion */ ixgbe_free_receive_structures(adapter); return (error); } /* ixgbe_allocate_receive_buffers */ /************************************************************************ * ixgbe_free_receive_ring ************************************************************************/ static void ixgbe_free_receive_ring(struct rx_ring *rxr) { for (int i = 0; i < rxr->num_desc; i++) { ixgbe_rx_discard(rxr, i); } } /* ixgbe_free_receive_ring */ /************************************************************************ * ixgbe_setup_receive_ring * * Initialize a receive ring and its buffers. ************************************************************************/ static int ixgbe_setup_receive_ring(struct rx_ring *rxr) { struct adapter *adapter; struct ifnet *ifp; device_t dev; struct ixgbe_rx_buf *rxbuf; struct lro_ctrl *lro = &rxr->lro; #ifdef DEV_NETMAP struct netmap_adapter *na = NA(rxr->adapter->ifp); struct netmap_slot *slot; #endif /* DEV_NETMAP */ bus_dma_segment_t seg[1]; int rsize, nsegs, error = 0; adapter = rxr->adapter; ifp = adapter->ifp; dev = adapter->dev; /* Clear the ring contents */ IXGBE_RX_LOCK(rxr); #ifdef DEV_NETMAP if (adapter->feat_en & IXGBE_FEATURE_NETMAP) slot = netmap_reset(na, NR_RX, rxr->me, 0); #endif /* DEV_NETMAP */ rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); /* Cache the size */ rxr->mbuf_sz = adapter->rx_mbuf_sz; /* Free current RX buffer structs and their mbufs */ ixgbe_free_receive_ring(rxr); /* Now replenish the mbufs */ for (int j = 0; j != rxr->num_desc; ++j) { struct mbuf *mp; rxbuf = &rxr->rx_buffers[j]; #ifdef DEV_NETMAP /* * In netmap mode, fill the map and set the buffer * address in the NIC ring, considering the offset * between the netmap and NIC rings (see comment in * ixgbe_setup_transmit_ring() ). No need to allocate * an mbuf, so end the block with a continue; */ if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); uint64_t paddr; void *addr; addr = PNMB(na, slot + sj, &paddr); netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); /* Update descriptor and the cached value */ rxr->rx_base[j].read.pkt_addr = htole64(paddr); rxbuf->addr = htole64(paddr); continue; } #endif /* DEV_NETMAP */ rxbuf->flags = 0; rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); if (rxbuf->buf == NULL) { error = ENOBUFS; goto fail; } mp = rxbuf->buf; mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* Get the memory mapping */ error = bus_dmamap_load_mbuf_sg(rxr->ptag, rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) goto fail; bus_dmamap_sync(rxr->ptag, rxbuf->pmap, BUS_DMASYNC_PREREAD); /* Update the descriptor and the cached value */ rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr); rxbuf->addr = htole64(seg[0].ds_addr); } /* Setup our descriptor indices */ rxr->next_to_check = 0; rxr->next_to_refresh = 0; rxr->lro_enabled = FALSE; rxr->rx_copies = 0; rxr->rx_bytes = 0; rxr->vtag_strip = FALSE; bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* * Now set up the LRO interface */ if (ixgbe_rsc_enable) ixgbe_setup_hw_rsc(rxr); else if (ifp->if_capenable & IFCAP_LRO) { int err = tcp_lro_init(lro); if (err) { device_printf(dev, "LRO Initialization failed!\n"); goto fail; } INIT_DEBUGOUT("RX Soft LRO Initialized\n"); rxr->lro_enabled = TRUE; lro->ifp = adapter->ifp; } IXGBE_RX_UNLOCK(rxr); return (0); fail: ixgbe_free_receive_ring(rxr); IXGBE_RX_UNLOCK(rxr); return (error); } /* ixgbe_setup_receive_ring */ /************************************************************************ * ixgbe_setup_receive_structures - Initialize all receive rings. ************************************************************************/ int ixgbe_setup_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; int j; for (j = 0; j < adapter->num_queues; j++, rxr++) if (ixgbe_setup_receive_ring(rxr)) goto fail; return (0); fail: /* * Free RX buffers allocated so far, we will only handle * the rings that completed, the failing case will have * cleaned up for itself. 'j' failed, so its the terminus. */ for (int i = 0; i < j; ++i) { rxr = &adapter->rx_rings[i]; IXGBE_RX_LOCK(rxr); ixgbe_free_receive_ring(rxr); IXGBE_RX_UNLOCK(rxr); } return (ENOBUFS); } /* ixgbe_setup_receive_structures */ /************************************************************************ * ixgbe_free_receive_structures - Free all receive rings. ************************************************************************/ void ixgbe_free_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; INIT_DEBUGOUT("ixgbe_free_receive_structures: begin"); for (int i = 0; i < adapter->num_queues; i++, rxr++) { ixgbe_free_receive_buffers(rxr); /* Free LRO memory */ tcp_lro_free(&rxr->lro); /* Free the ring memory as well */ ixgbe_dma_free(adapter, &rxr->rxdma); } free(adapter->rx_rings, M_DEVBUF); } /* ixgbe_free_receive_structures */ /************************************************************************ * ixgbe_free_receive_buffers - Free receive ring data structures ************************************************************************/ static void ixgbe_free_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; struct ixgbe_rx_buf *rxbuf; INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin"); /* Cleanup any existing buffers */ if (rxr->rx_buffers != NULL) { for (int i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; ixgbe_rx_discard(rxr, i); if (rxbuf->pmap != NULL) { bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); rxbuf->pmap = NULL; } } if (rxr->rx_buffers != NULL) { free(rxr->rx_buffers, M_DEVBUF); rxr->rx_buffers = NULL; } } if (rxr->ptag != NULL) { bus_dma_tag_destroy(rxr->ptag); rxr->ptag = NULL; } return; } /* ixgbe_free_receive_buffers */ /************************************************************************ * ixgbe_rx_input ************************************************************************/ static __inline void ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) { /* * ATM LRO is only for IP/TCP packets and TCP checksum of the packet * should be computed by hardware. Also it should not have VLAN tag in * ethernet header. In case of IPv6 we do not yet support ext. hdrs. */ if (rxr->lro_enabled && (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { /* * Send to the stack if: * - LRO not enabled, or * - no LRO resources, or * - lro enqueue fails */ if (rxr->lro.lro_cnt != 0) if (tcp_lro_rx(&rxr->lro, m, 0) == 0) return; } - IXGBE_RX_UNLOCK(rxr); (*ifp->if_input)(ifp, m); - IXGBE_RX_LOCK(rxr); } /* ixgbe_rx_input */ /************************************************************************ * ixgbe_rx_discard ************************************************************************/ static __inline void ixgbe_rx_discard(struct rx_ring *rxr, int i) { struct ixgbe_rx_buf *rbuf; rbuf = &rxr->rx_buffers[i]; /* * With advanced descriptors the writeback * clobbers the buffer addrs, so its easier * to just free the existing mbufs and take * the normal refresh path to get new buffers * and mapping. */ if (rbuf->fmp != NULL) {/* Partial chain ? */ bus_dmamap_sync(rxr->ptag, rbuf->pmap, BUS_DMASYNC_POSTREAD); m_freem(rbuf->fmp); rbuf->fmp = NULL; rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */ } else if (rbuf->buf) { bus_dmamap_sync(rxr->ptag, rbuf->pmap, BUS_DMASYNC_POSTREAD); m_free(rbuf->buf); rbuf->buf = NULL; } bus_dmamap_unload(rxr->ptag, rbuf->pmap); rbuf->flags = 0; return; } /* ixgbe_rx_discard */ /************************************************************************ * ixgbe_rxeof * * Executes in interrupt context. It replenishes the * mbufs in the descriptor and sends data which has * been dma'ed into host memory to upper layer. * * Return TRUE for more work, FALSE for all clean. ************************************************************************/ bool ixgbe_rxeof(struct ix_queue *que) { struct adapter *adapter = que->adapter; struct rx_ring *rxr = que->rxr; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; union ixgbe_adv_rx_desc *cur; struct ixgbe_rx_buf *rbuf, *nbuf; int i, nextp, processed = 0; u32 staterr = 0; u32 count = adapter->rx_process_limit; u16 pkt_info; IXGBE_RX_LOCK(rxr); #ifdef DEV_NETMAP if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { /* Same as the txeof routine: wakeup clients on intr. */ if (netmap_rx_irq(ifp, rxr->me, &processed)) { IXGBE_RX_UNLOCK(rxr); return (FALSE); } } #endif /* DEV_NETMAP */ for (i = rxr->next_to_check; count != 0;) { struct mbuf *sendmp, *mp; u32 rsc, ptype; u16 len; u16 vtag = 0; bool eop; /* Sync the ring. */ bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cur = &rxr->rx_base[i]; staterr = le32toh(cur->wb.upper.status_error); pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); if ((staterr & IXGBE_RXD_STAT_DD) == 0) break; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; count--; sendmp = NULL; nbuf = NULL; rsc = 0; cur->wb.upper.status_error = 0; rbuf = &rxr->rx_buffers[i]; mp = rbuf->buf; len = le16toh(cur->wb.upper.length); ptype = le32toh(cur->wb.lower.lo_dword.data) & IXGBE_RXDADV_PKTTYPE_MASK; eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); /* Make sure bad packets are discarded */ if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { #if __FreeBSD_version >= 1100036 if (adapter->feat_en & IXGBE_FEATURE_VF) if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); #endif rxr->rx_discarded++; ixgbe_rx_discard(rxr, i); goto next_desc; } bus_dmamap_sync(rxr->ptag, rbuf->pmap, BUS_DMASYNC_POSTREAD); /* * On 82599 which supports a hardware * LRO (called HW RSC), packets need * not be fragmented across sequential * descriptors, rather the next descriptor * is indicated in bits of the descriptor. * This also means that we might proceses * more than one packet at a time, something * that has never been true before, it * required eliminating global chain pointers * in favor of what we are doing here. -jfv */ if (!eop) { /* * Figure out the next descriptor * of this frame. */ if (rxr->hw_rsc == TRUE) { rsc = ixgbe_rsc_count(cur); rxr->rsc_num += (rsc - 1); } if (rsc) { /* Get hardware index */ nextp = ((staterr & IXGBE_RXDADV_NEXTP_MASK) >> IXGBE_RXDADV_NEXTP_SHIFT); } else { /* Just sequential */ nextp = i + 1; if (nextp == adapter->num_rx_desc) nextp = 0; } nbuf = &rxr->rx_buffers[nextp]; prefetch(nbuf); } /* * Rather than using the fmp/lmp global pointers * we now keep the head of a packet chain in the * buffer struct and pass this along from one * descriptor to the next, until we get EOP. */ mp->m_len = len; /* * See if there is a stored head * that determines what we are */ sendmp = rbuf->fmp; if (sendmp != NULL) { /* secondary frag */ rbuf->buf = rbuf->fmp = NULL; mp->m_flags &= ~M_PKTHDR; sendmp->m_pkthdr.len += mp->m_len; } else { /* * Optimize. This might be a small packet, * maybe just a TCP ACK. Do a fast copy that * is cache aligned into a new mbuf, and * leave the old mbuf+cluster for re-use. */ if (eop && len <= IXGBE_RX_COPY_LEN) { sendmp = m_gethdr(M_NOWAIT, MT_DATA); if (sendmp != NULL) { sendmp->m_data += IXGBE_RX_COPY_ALIGN; ixgbe_bcopy(mp->m_data, sendmp->m_data, len); sendmp->m_len = len; rxr->rx_copies++; rbuf->flags |= IXGBE_RX_COPY; } } if (sendmp == NULL) { rbuf->buf = rbuf->fmp = NULL; sendmp = mp; } /* first desc of a non-ps chain */ sendmp->m_flags |= M_PKTHDR; sendmp->m_pkthdr.len = mp->m_len; } ++processed; /* Pass the head pointer on */ if (eop == 0) { nbuf->fmp = sendmp; sendmp = NULL; mp->m_next = nbuf->buf; } else { /* Sending this frame */ sendmp->m_pkthdr.rcvif = ifp; rxr->rx_packets++; /* capture data for AIM */ rxr->bytes += sendmp->m_pkthdr.len; rxr->rx_bytes += sendmp->m_pkthdr.len; /* Process vlan info */ if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) vtag = le16toh(cur->wb.upper.vlan); if (vtag) { sendmp->m_pkthdr.ether_vtag = vtag; sendmp->m_flags |= M_VLANTAG; } if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) ixgbe_rx_checksum(staterr, sendmp, ptype); /* * In case of multiqueue, we have RXCSUM.PCSD bit set * and never cleared. This means we have RSS hash * available to be used. */ if (adapter->num_queues > 1) { sendmp->m_pkthdr.flowid = le32toh(cur->wb.lower.hi_dword.rss); switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) { case IXGBE_RXDADV_RSSTYPE_IPV4: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV4); break; case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV4); break; case IXGBE_RXDADV_RSSTYPE_IPV6: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6); break; case IXGBE_RXDADV_RSSTYPE_IPV6_TCP: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6); break; case IXGBE_RXDADV_RSSTYPE_IPV6_EX: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6_EX); break; case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6_EX); break; #if __FreeBSD_version > 1100000 case IXGBE_RXDADV_RSSTYPE_IPV4_UDP: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV4); break; case IXGBE_RXDADV_RSSTYPE_IPV6_UDP: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6); break; case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6_EX); break; #endif default: M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE_HASH); } } else { sendmp->m_pkthdr.flowid = que->msix; M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); } } next_desc: bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* Advance our pointers to the next descriptor. */ if (++i == rxr->num_desc) i = 0; /* Now send to the stack or do LRO */ if (sendmp != NULL) { rxr->next_to_check = i; + IXGBE_RX_UNLOCK(rxr); ixgbe_rx_input(rxr, ifp, sendmp, ptype); + IXGBE_RX_LOCK(rxr); i = rxr->next_to_check; } /* Every 8 descriptors we go to refresh mbufs */ if (processed == 8) { ixgbe_refresh_mbufs(rxr, i); processed = 0; } } /* Refresh any remaining buf structs */ if (ixgbe_rx_unrefreshed(rxr)) ixgbe_refresh_mbufs(rxr, i); rxr->next_to_check = i; + IXGBE_RX_UNLOCK(rxr); + /* * Flush any outstanding LRO work */ tcp_lro_flush_all(lro); - - IXGBE_RX_UNLOCK(rxr); /* * Still have cleaning to do? */ if ((staterr & IXGBE_RXD_STAT_DD) != 0) return (TRUE); return (FALSE); } /* ixgbe_rxeof */ /************************************************************************ * ixgbe_rx_checksum * * Verify that the hardware indicated that the checksum is valid. * Inform the stack about the status of checksum so that stack * doesn't spend time verifying the checksum. ************************************************************************/ static void ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) { u16 status = (u16)staterr; u8 errors = (u8)(staterr >> 24); bool sctp = false; if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) sctp = true; /* IPv4 checksum */ if (status & IXGBE_RXD_STAT_IPCS) { mp->m_pkthdr.csum_flags |= CSUM_L3_CALC; /* IP Checksum Good */ if (!(errors & IXGBE_RXD_ERR_IPE)) mp->m_pkthdr.csum_flags |= CSUM_L3_VALID; } /* TCP/UDP/SCTP checksum */ if (status & IXGBE_RXD_STAT_L4CS) { mp->m_pkthdr.csum_flags |= CSUM_L4_CALC; if (!(errors & IXGBE_RXD_ERR_TCPE)) { mp->m_pkthdr.csum_flags |= CSUM_L4_VALID; if (!sctp) mp->m_pkthdr.csum_data = htons(0xffff); } } } /* ixgbe_rx_checksum */ /************************************************************************ * ixgbe_dmamap_cb - Manage DMA'able memory. ************************************************************************/ static void ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) { if (error) return; *(bus_addr_t *)arg = segs->ds_addr; return; } /* ixgbe_dmamap_cb */ /************************************************************************ * ixgbe_dma_malloc ************************************************************************/ static int ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, struct ixgbe_dma_alloc *dma, int mapflags) { device_t dev = adapter->dev; int r; r = bus_dma_tag_create( /* parent */ bus_get_dma_tag(adapter->dev), /* alignment */ DBA_ALIGN, /* bounds */ 0, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ size, /* nsegments */ 1, /* maxsegsize */ size, /* flags */ BUS_DMA_ALLOCNOW, /* lockfunc */ NULL, /* lockfuncarg */ NULL, &dma->dma_tag); if (r != 0) { device_printf(dev, "ixgbe_dma_malloc: bus_dma_tag_create failed; error %u\n", r); goto fail_0; } r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, BUS_DMA_NOWAIT, &dma->dma_map); if (r != 0) { device_printf(dev, "ixgbe_dma_malloc: bus_dmamem_alloc failed; error %u\n", r); goto fail_1; } r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, ixgbe_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); if (r != 0) { device_printf(dev, "ixgbe_dma_malloc: bus_dmamap_load failed; error %u\n", r); goto fail_2; } dma->dma_size = size; return (0); fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); fail_1: bus_dma_tag_destroy(dma->dma_tag); fail_0: dma->dma_tag = NULL; return (r); } /* ixgbe_dma_malloc */ /************************************************************************ * ixgbe_dma_free ************************************************************************/ static void ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); } /* ixgbe_dma_free */ /************************************************************************ * ixgbe_allocate_queues * * Allocate memory for the transmit and receive rings, and then * the descriptors associated with each, called only once at attach. ************************************************************************/ int ixgbe_allocate_queues(struct adapter *adapter) { device_t dev = adapter->dev; struct ix_queue *que; struct tx_ring *txr; struct rx_ring *rxr; int rsize, tsize, error = IXGBE_SUCCESS; int txconf = 0, rxconf = 0; /* First, allocate the top level queue structs */ adapter->queues = (struct ix_queue *)malloc(sizeof(struct ix_queue) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO); if (adapter->queues == NULL) { device_printf(dev, "Unable to allocate queue memory\n"); error = ENOMEM; goto fail; } /* Second, allocate the TX ring struct memory */ adapter->tx_rings = (struct tx_ring *)malloc(sizeof(struct tx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO); if (adapter->tx_rings == NULL) { device_printf(dev, "Unable to allocate TX ring memory\n"); error = ENOMEM; goto tx_fail; } /* Third, allocate the RX ring */ adapter->rx_rings = (struct rx_ring *)malloc(sizeof(struct rx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO); if (adapter->rx_rings == NULL) { device_printf(dev, "Unable to allocate RX ring memory\n"); error = ENOMEM; goto rx_fail; } /* For the ring itself */ tsize = roundup2(adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); /* * Now set up the TX queues, txconf is needed to handle the * possibility that things fail midcourse and we need to * undo memory gracefully */ for (int i = 0; i < adapter->num_queues; i++, txconf++) { /* Set up some basics */ txr = &adapter->tx_rings[i]; txr->adapter = adapter; txr->br = NULL; /* In case SR-IOV is enabled, align the index properly */ txr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, i); txr->num_desc = adapter->num_tx_desc; /* Initialize the TX side lock */ snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", device_get_nameunit(dev), txr->me); mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate TX Descriptor memory\n"); error = ENOMEM; goto err_tx_desc; } txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; bzero((void *)txr->tx_base, tsize); /* Now allocate transmit buffers for the ring */ if (ixgbe_allocate_transmit_buffers(txr)) { device_printf(dev, "Critical Failure setting up transmit buffers\n"); error = ENOMEM; goto err_tx_desc; } if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { /* Allocate a buf ring */ txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, M_WAITOK, &txr->tx_mtx); if (txr->br == NULL) { device_printf(dev, "Critical Failure setting up buf ring\n"); error = ENOMEM; goto err_tx_desc; } } } /* * Next the RX queues... */ rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); for (int i = 0; i < adapter->num_queues; i++, rxconf++) { rxr = &adapter->rx_rings[i]; /* Set up some basics */ rxr->adapter = adapter; /* In case SR-IOV is enabled, align the index properly */ rxr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, i); rxr->num_desc = adapter->num_rx_desc; /* Initialize the RX side lock */ snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", device_get_nameunit(dev), rxr->me); mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate RxDescriptor memory\n"); error = ENOMEM; goto err_rx_desc; } rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; bzero((void *)rxr->rx_base, rsize); /* Allocate receive buffers for the ring */ if (ixgbe_allocate_receive_buffers(rxr)) { device_printf(dev, "Critical Failure setting up receive buffers\n"); error = ENOMEM; goto err_rx_desc; } } /* * Finally set up the queue holding structs */ for (int i = 0; i < adapter->num_queues; i++) { que = &adapter->queues[i]; que->adapter = adapter; que->me = i; que->txr = &adapter->tx_rings[i]; que->rxr = &adapter->rx_rings[i]; } return (0); err_rx_desc: for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) ixgbe_dma_free(adapter, &rxr->rxdma); err_tx_desc: for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) ixgbe_dma_free(adapter, &txr->txdma); free(adapter->rx_rings, M_DEVBUF); rx_fail: free(adapter->tx_rings, M_DEVBUF); tx_fail: free(adapter->queues, M_DEVBUF); fail: return (error); } /* ixgbe_allocate_queues */ Index: projects/runtime-coverage/sys/dev/mmc/mmcsd.c =================================================================== --- projects/runtime-coverage/sys/dev/mmc/mmcsd.c (revision 321584) +++ projects/runtime-coverage/sys/dev/mmc/mmcsd.c (revision 321585) @@ -1,1414 +1,1414 @@ /*- * Copyright (c) 2006 Bernd Walter. All rights reserved. * Copyright (c) 2006 M. Warner Losh. All rights reserved. * Copyright (c) 2017 Marius Strobl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Portions of this software may have been developed with reference to * the SD Simplified Specification. The following disclaimer may apply: * * The following conditions apply to the release of the simplified * specification ("Simplified Specification") by the SD Card Association and * the SD Group. The Simplified Specification is a subset of the complete SD * Specification which is owned by the SD Card Association and the SD * Group. This Simplified Specification is provided on a non-confidential * basis subject to the disclaimers below. Any implementation of the * Simplified Specification may require a license from the SD Card * Association, SD Group, SD-3C LLC or other third parties. * * Disclaimers: * * The information contained in the Simplified Specification is presented only * as a standard specification for SD Cards and SD Host/Ancillary products and * is provided "AS-IS" without any representations or warranties of any * kind. No responsibility is assumed by the SD Group, SD-3C LLC or the SD * Card Association for any damages, any infringements of patents or other * right of the SD Group, SD-3C LLC, the SD Card Association or any third * parties, which may result from its use. No license is granted by * implication, estoppel or otherwise under any patent or other rights of the * SD Group, SD-3C LLC, the SD Card Association or any third party. Nothing * herein shall be construed as an obligation by the SD Group, the SD-3C LLC * or the SD Card Association to disclose or distribute any technical * information, know-how or other confidential information to any third party. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mmcbus_if.h" #if __FreeBSD_version < 800002 #define kproc_create kthread_create #define kproc_exit kthread_exit #endif #define MMCSD_CMD_RETRIES 5 #define MMCSD_FMT_BOOT "mmcsd%dboot" #define MMCSD_FMT_GP "mmcsd%dgp" #define MMCSD_FMT_RPMB "mmcsd%drpmb" #define MMCSD_LABEL_ENH "enh" #define MMCSD_PART_NAMELEN (16 + 1) struct mmcsd_softc; struct mmcsd_part { struct mtx disk_mtx; struct mtx ioctl_mtx; struct mmcsd_softc *sc; struct disk *disk; struct proc *p; struct bio_queue_head bio_queue; daddr_t eblock, eend; /* Range remaining after the last erase. */ u_int cnt; u_int type; int running; int suspend; int ioctl; bool ro; char name[MMCSD_PART_NAMELEN]; }; struct mmcsd_softc { device_t dev; device_t mmcbr; struct mmcsd_part *part[MMC_PART_MAX]; enum mmc_card_mode mode; u_int max_data; /* Maximum data size [blocks] */ u_int erase_sector; /* Device native erase sector size [blocks] */ uint8_t high_cap; /* High Capacity device (block addressed) */ uint8_t part_curr; /* Partition currently switched to */ uint8_t ext_csd[MMC_EXTCSD_SIZE]; uint16_t rca; uint32_t part_time; /* Partition switch timeout [us] */ off_t enh_base; /* Enhanced user data area slice base ... */ off_t enh_size; /* ... and size [bytes] */ int log_count; struct timeval log_time; struct cdev *rpmb_dev; }; static const char *errmsg[] = { "None", "Timeout", "Bad CRC", "Fifo", "Failed", "Invalid", "NO MEMORY" }; #define LOG_PPS 5 /* Log no more than 5 errors per second. */ /* bus entry points */ static int mmcsd_attach(device_t dev); static int mmcsd_detach(device_t dev); static int mmcsd_probe(device_t dev); /* disk routines */ static int mmcsd_close(struct disk *dp); static int mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length); static int mmcsd_getattr(struct bio *); static int mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, int fflag, struct thread *td); static int mmcsd_open(struct disk *dp); static void mmcsd_strategy(struct bio *bp); static void mmcsd_task(void *arg); /* RMPB cdev interface */ static int mmcsd_ioctl_rpmb(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td); static void mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, off_t media_size, off_t erase_size, bool ro); static int mmcsd_bus_bit_width(device_t dev); static daddr_t mmcsd_delete(struct mmcsd_part *part, struct bio *bp); static int mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag); static int mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag); static uintmax_t mmcsd_pretty_size(off_t size, char *unit); static daddr_t mmcsd_rw(struct mmcsd_part *part, struct bio *bp); static int mmcsd_set_blockcount(struct mmcsd_softc *sc, u_int count, bool rel); static int mmcsd_slicer(device_t dev, const char *provider, struct flash_slice *slices, int *nslices); static int mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part); #define MMCSD_DISK_LOCK(_part) mtx_lock(&(_part)->disk_mtx) #define MMCSD_DISK_UNLOCK(_part) mtx_unlock(&(_part)->disk_mtx) #define MMCSD_DISK_LOCK_INIT(_part) \ mtx_init(&(_part)->disk_mtx, (_part)->name, "mmcsd disk", MTX_DEF) #define MMCSD_DISK_LOCK_DESTROY(_part) mtx_destroy(&(_part)->disk_mtx); #define MMCSD_DISK_ASSERT_LOCKED(_part) \ mtx_assert(&(_part)->disk_mtx, MA_OWNED); #define MMCSD_DISK_ASSERT_UNLOCKED(_part) \ mtx_assert(&(_part)->disk_mtx, MA_NOTOWNED); #define MMCSD_IOCTL_LOCK(_part) mtx_lock(&(_part)->ioctl_mtx) #define MMCSD_IOCTL_UNLOCK(_part) mtx_unlock(&(_part)->ioctl_mtx) #define MMCSD_IOCTL_LOCK_INIT(_part) \ mtx_init(&(_part)->ioctl_mtx, (_part)->name, "mmcsd IOCTL", MTX_DEF) #define MMCSD_IOCTL_LOCK_DESTROY(_part) mtx_destroy(&(_part)->ioctl_mtx); #define MMCSD_IOCTL_ASSERT_LOCKED(_part) \ mtx_assert(&(_part)->ioctl_mtx, MA_OWNED); #define MMCSD_IOCLT_ASSERT_UNLOCKED(_part) \ mtx_assert(&(_part)->ioctl_mtx, MA_NOTOWNED); static int mmcsd_probe(device_t dev) { device_quiet(dev); device_set_desc(dev, "MMC/SD Memory Card"); return (0); } static int mmcsd_attach(device_t dev) { device_t mmcbr; struct mmcsd_softc *sc; const uint8_t *ext_csd; off_t erase_size, sector_size, size, wp_size; uintmax_t bytes; int err, i; uint8_t rev; bool comp, ro; char unit[2]; sc = device_get_softc(dev); sc->dev = dev; sc->mmcbr = mmcbr = device_get_parent(dev); sc->mode = mmcbr_get_mode(mmcbr); /* * Note that in principle with an SDHCI-like re-tuning implementation, * the maximum data size can change at runtime due to a device removal/ * insertion that results in switches to/from a transfer mode involving * re-tuning, iff there are multiple devices on a given bus. Until now * mmc(4) lacks support for rescanning already attached buses, however, - * and sdhci(4) has no support for embedded/shared buses in the first + * and sdhci(4) to date has no support for shared buses in the first * place either. */ sc->max_data = mmc_get_max_data(dev); sc->erase_sector = mmc_get_erase_sector(dev); sc->high_cap = mmc_get_high_cap(dev); sc->rca = mmc_get_rca(dev); /* Only MMC >= 4.x devices support EXT_CSD. */ if (mmc_get_spec_vers(dev) >= 4) { MMCBUS_ACQUIRE_BUS(mmcbr, dev); err = mmc_send_ext_csd(mmcbr, dev, sc->ext_csd); MMCBUS_RELEASE_BUS(mmcbr, dev); if (err != MMC_ERR_NONE) bzero(sc->ext_csd, sizeof(sc->ext_csd)); } ext_csd = sc->ext_csd; /* * Enhanced user data area and general purpose partitions are only * supported in revision 1.4 (EXT_CSD_REV == 4) and later, the RPMB * partition in revision 1.5 (MMC v4.41, EXT_CSD_REV == 5) and later. */ rev = ext_csd[EXT_CSD_REV]; /* * Ignore user-creatable enhanced user data area and general purpose * partitions partitions as long as partitioning hasn't been finished. */ comp = (ext_csd[EXT_CSD_PART_SET] & EXT_CSD_PART_SET_COMPLETED) != 0; /* * Add enhanced user data area slice, unless it spans the entirety of * the user data area. The enhanced area is of a multiple of high * capacity write protect groups ((ERASE_GRP_SIZE + HC_WP_GRP_SIZE) * * 512 KB) and its offset given in either sectors or bytes, depending * on whether it's a high capacity device or not. * NB: The slicer and its slices need to be registered before adding * the disk for the corresponding user data area as re-tasting is * racy. */ sector_size = mmc_get_sector_size(dev); size = ext_csd[EXT_CSD_ENH_SIZE_MULT] + (ext_csd[EXT_CSD_ENH_SIZE_MULT + 1] << 8) + (ext_csd[EXT_CSD_ENH_SIZE_MULT + 2] << 16); if (rev >= 4 && comp == TRUE && size > 0 && (ext_csd[EXT_CSD_PART_SUPPORT] & EXT_CSD_PART_SUPPORT_ENH_ATTR_EN) != 0 && (ext_csd[EXT_CSD_PART_ATTR] & (EXT_CSD_PART_ATTR_ENH_USR)) != 0) { erase_size = ext_csd[EXT_CSD_ERASE_GRP_SIZE] * 1024 * MMC_SECTOR_SIZE; wp_size = ext_csd[EXT_CSD_HC_WP_GRP_SIZE]; size *= erase_size * wp_size; if (size != mmc_get_media_size(dev) * sector_size) { sc->enh_size = size; sc->enh_base = (ext_csd[EXT_CSD_ENH_START_ADDR] + (ext_csd[EXT_CSD_ENH_START_ADDR + 1] << 8) + (ext_csd[EXT_CSD_ENH_START_ADDR + 2] << 16) + (ext_csd[EXT_CSD_ENH_START_ADDR + 3] << 24)) * (sc->high_cap != 0 ? MMC_SECTOR_SIZE : 1); } else if (bootverbose) device_printf(dev, "enhanced user data area spans entire device\n"); } /* * Add default partition. This may be the only one or the user * data area in case partitions are supported. */ ro = mmc_get_read_only(dev); mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_DEFAULT, "mmcsd", device_get_unit(dev), mmc_get_media_size(dev) * sector_size, sc->erase_sector * sector_size, ro); if (mmc_get_spec_vers(dev) < 3) return (0); /* Belatedly announce enhanced user data slice. */ if (sc->enh_size != 0) { bytes = mmcsd_pretty_size(size, unit); printf(FLASH_SLICES_FMT ": %ju%sB enhanced user data area " "slice offset 0x%jx at %s\n", device_get_nameunit(dev), MMCSD_LABEL_ENH, bytes, unit, (uintmax_t)sc->enh_base, device_get_nameunit(dev)); } /* * Determine partition switch timeout (provided in units of 10 ms) * and ensure it's at least 300 ms as some eMMC chips lie. */ sc->part_time = max(ext_csd[EXT_CSD_PART_SWITCH_TO] * 10 * 1000, 300 * 1000); /* Add boot partitions, which are of a fixed multiple of 128 KB. */ size = ext_csd[EXT_CSD_BOOT_SIZE_MULT] * MMC_BOOT_RPMB_BLOCK_SIZE; if (size > 0 && (mmcbr_get_caps(mmcbr) & MMC_CAP_BOOT_NOACC) == 0) { mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_BOOT0, MMCSD_FMT_BOOT, 0, size, MMC_BOOT_RPMB_BLOCK_SIZE, ro | ((ext_csd[EXT_CSD_BOOT_WP_STATUS] & EXT_CSD_BOOT_WP_STATUS_BOOT0_MASK) != 0)); mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_BOOT1, MMCSD_FMT_BOOT, 1, size, MMC_BOOT_RPMB_BLOCK_SIZE, ro | ((ext_csd[EXT_CSD_BOOT_WP_STATUS] & EXT_CSD_BOOT_WP_STATUS_BOOT1_MASK) != 0)); } /* Add RPMB partition, which also is of a fixed multiple of 128 KB. */ size = ext_csd[EXT_CSD_RPMB_MULT] * MMC_BOOT_RPMB_BLOCK_SIZE; if (rev >= 5 && size > 0) mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_RPMB, MMCSD_FMT_RPMB, 0, size, MMC_BOOT_RPMB_BLOCK_SIZE, ro); if (rev <= 3 || comp == FALSE) return (0); /* * Add general purpose partitions, which are of a multiple of high * capacity write protect groups, too. */ if ((ext_csd[EXT_CSD_PART_SUPPORT] & EXT_CSD_PART_SUPPORT_EN) != 0) { erase_size = ext_csd[EXT_CSD_ERASE_GRP_SIZE] * 1024 * MMC_SECTOR_SIZE; wp_size = ext_csd[EXT_CSD_HC_WP_GRP_SIZE]; for (i = 0; i < MMC_PART_GP_MAX; i++) { size = ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3] + (ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3 + 1] << 8) + (ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3 + 2] << 16); if (size == 0) continue; mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_GP0 + i, MMCSD_FMT_GP, i, size * erase_size * wp_size, erase_size, ro); } } return (0); } static uintmax_t mmcsd_pretty_size(off_t size, char *unit) { uintmax_t bytes; int i; /* * Display in most natural units. There's no card < 1MB. However, * RPMB partitions occasionally are smaller than that, though. The * SD standard goes to 2 GiB due to its reliance on FAT, but the data * format supports up to 4 GiB and some card makers push it up to this * limit. The SDHC standard only goes to 32 GiB due to FAT32, but the * data format supports up to 2 TiB however. 2048 GB isn't too ugly, * so we note it in passing here and don't add the code to print TB). * Since these cards are sold in terms of MB and GB not MiB and GiB, * report them like that. We also round to the nearest unit, since * many cards are a few percent short, even of the power of 10 size. */ bytes = size; unit[0] = unit[1] = '\0'; for (i = 0; i <= 2 && bytes >= 1000; i++) { bytes = (bytes + 1000 / 2 - 1) / 1000; switch (i) { case 0: unit[0] = 'k'; break; case 1: unit[0] = 'M'; break; case 2: unit[0] = 'G'; break; default: break; } } return (bytes); } static struct cdevsw mmcsd_rpmb_cdevsw = { .d_version = D_VERSION, .d_name = "mmcsdrpmb", .d_ioctl = mmcsd_ioctl_rpmb }; static void mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, off_t media_size, off_t erase_size, bool ro) { struct make_dev_args args; device_t dev, mmcbr; const char *ext; const uint8_t *ext_csd; struct mmcsd_part *part; struct disk *d; uintmax_t bytes; u_int gp; uint32_t speed; uint8_t extattr; bool enh; char unit[2]; dev = sc->dev; mmcbr = sc->mmcbr; part = sc->part[type] = malloc(sizeof(*part), M_DEVBUF, M_WAITOK | M_ZERO); part->sc = sc; part->cnt = cnt; part->type = type; part->ro = ro; snprintf(part->name, sizeof(part->name), name, device_get_unit(dev)); MMCSD_IOCTL_LOCK_INIT(part); /* * For the RPMB partition, allow IOCTL access only. * NB: If ever attaching RPMB partitions to disk(9), the re-tuning * implementation and especially its pausing need to be revisited, * because then re-tuning requests may be issued by the IOCTL half * of this driver while re-tuning is already paused by the disk(9) * one and vice versa. */ if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { make_dev_args_init(&args); args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; args.mda_devsw = &mmcsd_rpmb_cdevsw; args.mda_uid = UID_ROOT; args.mda_gid = GID_OPERATOR; args.mda_mode = 0640; args.mda_si_drv1 = part; if (make_dev_s(&args, &sc->rpmb_dev, "%s", part->name) != 0) { device_printf(dev, "Failed to make RPMB device\n"); free(part, M_DEVBUF); return; } } else { MMCSD_DISK_LOCK_INIT(part); d = part->disk = disk_alloc(); d->d_open = mmcsd_open; d->d_close = mmcsd_close; d->d_strategy = mmcsd_strategy; d->d_ioctl = mmcsd_ioctl_disk; d->d_dump = mmcsd_dump; d->d_getattr = mmcsd_getattr; d->d_name = part->name; d->d_drv1 = part; d->d_sectorsize = mmc_get_sector_size(dev); d->d_maxsize = sc->max_data * d->d_sectorsize; d->d_mediasize = media_size; d->d_stripesize = erase_size; d->d_unit = cnt; d->d_flags = DISKFLAG_CANDELETE; d->d_delmaxsize = erase_size; strlcpy(d->d_ident, mmc_get_card_sn_string(dev), sizeof(d->d_ident)); strlcpy(d->d_descr, mmc_get_card_id_string(dev), sizeof(d->d_descr)); d->d_rotation_rate = DISK_RR_NON_ROTATING; disk_create(d, DISK_VERSION); bioq_init(&part->bio_queue); part->running = 1; kproc_create(&mmcsd_task, part, &part->p, 0, 0, "%s%d: mmc/sd card", part->name, cnt); } bytes = mmcsd_pretty_size(media_size, unit); if (type == EXT_CSD_PART_CONFIG_ACC_DEFAULT) { speed = mmcbr_get_clock(mmcbr); printf("%s%d: %ju%sB <%s>%s at %s %d.%01dMHz/%dbit/%d-block\n", part->name, cnt, bytes, unit, mmc_get_card_id_string(dev), ro ? " (read-only)" : "", device_get_nameunit(mmcbr), speed / 1000000, (speed / 100000) % 10, mmcsd_bus_bit_width(dev), sc->max_data); } else if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { printf("%s: %ju%sB partion %d%s at %s\n", part->name, bytes, unit, type, ro ? " (read-only)" : "", device_get_nameunit(dev)); } else { enh = false; ext = NULL; extattr = 0; if (type >= EXT_CSD_PART_CONFIG_ACC_GP0 && type <= EXT_CSD_PART_CONFIG_ACC_GP3) { ext_csd = sc->ext_csd; gp = type - EXT_CSD_PART_CONFIG_ACC_GP0; if ((ext_csd[EXT_CSD_PART_SUPPORT] & EXT_CSD_PART_SUPPORT_ENH_ATTR_EN) != 0 && (ext_csd[EXT_CSD_PART_ATTR] & (EXT_CSD_PART_ATTR_ENH_GP0 << gp)) != 0) enh = true; else if ((ext_csd[EXT_CSD_PART_SUPPORT] & EXT_CSD_PART_SUPPORT_EXT_ATTR_EN) != 0) { extattr = (ext_csd[EXT_CSD_EXT_PART_ATTR + (gp / 2)] >> (4 * (gp % 2))) & 0xF; switch (extattr) { case EXT_CSD_EXT_PART_ATTR_DEFAULT: break; case EXT_CSD_EXT_PART_ATTR_SYSTEMCODE: ext = "system code"; break; case EXT_CSD_EXT_PART_ATTR_NPERSISTENT: ext = "non-persistent"; break; default: ext = "reserved"; break; } } } if (ext == NULL) printf("%s%d: %ju%sB partion %d%s%s at %s\n", part->name, cnt, bytes, unit, type, enh ? " enhanced" : "", ro ? " (read-only)" : "", device_get_nameunit(dev)); else printf("%s%d: %ju%sB partion %d extended 0x%x " "(%s)%s at %s\n", part->name, cnt, bytes, unit, type, extattr, ext, ro ? " (read-only)" : "", device_get_nameunit(dev)); } } static int mmcsd_slicer(device_t dev, const char *provider, struct flash_slice *slices, int *nslices) { char name[MMCSD_PART_NAMELEN]; struct mmcsd_softc *sc; struct mmcsd_part *part; *nslices = 0; if (slices == NULL) return (ENOMEM); sc = device_get_softc(dev); if (sc->enh_size == 0) return (ENXIO); part = sc->part[EXT_CSD_PART_CONFIG_ACC_DEFAULT]; snprintf(name, sizeof(name), "%s%d", part->disk->d_name, part->disk->d_unit); if (strcmp(name, provider) != 0) return (ENXIO); *nslices = 1; slices[0].base = sc->enh_base; slices[0].size = sc->enh_size; slices[0].label = MMCSD_LABEL_ENH; return (0); } static int mmcsd_detach(device_t dev) { struct mmcsd_softc *sc = device_get_softc(dev); struct mmcsd_part *part; int i; for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; if (part != NULL) { if (part->disk != NULL) { MMCSD_DISK_LOCK(part); part->suspend = 0; if (part->running > 0) { /* kill thread */ part->running = 0; wakeup(part); /* wait for thread to finish. */ while (part->running != -1) msleep(part, &part->disk_mtx, 0, "mmcsd disk detach", 0); } MMCSD_DISK_UNLOCK(part); } MMCSD_IOCTL_LOCK(part); while (part->ioctl > 0) msleep(part, &part->ioctl_mtx, 0, "mmcsd IOCTL detach", 0); part->ioctl = -1; MMCSD_IOCTL_UNLOCK(part); } } if (sc->rpmb_dev != NULL) destroy_dev(sc->rpmb_dev); for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; if (part != NULL) { if (part->disk != NULL) { /* Flush the request queue. */ bioq_flush(&part->bio_queue, NULL, ENXIO); /* kill disk */ disk_destroy(part->disk); MMCSD_DISK_LOCK_DESTROY(part); } MMCSD_IOCTL_LOCK_DESTROY(part); free(part, M_DEVBUF); } } return (0); } static int mmcsd_suspend(device_t dev) { struct mmcsd_softc *sc = device_get_softc(dev); struct mmcsd_part *part; int i; for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; if (part != NULL) { if (part->disk != NULL) { MMCSD_DISK_LOCK(part); part->suspend = 1; if (part->running > 0) { /* kill thread */ part->running = 0; wakeup(part); /* wait for thread to finish. */ while (part->running != -1) msleep(part, &part->disk_mtx, 0, "mmcsd disk suspension", 0); } MMCSD_DISK_UNLOCK(part); } MMCSD_IOCTL_LOCK(part); while (part->ioctl > 0) msleep(part, &part->ioctl_mtx, 0, "mmcsd IOCTL suspension", 0); part->ioctl = -1; MMCSD_IOCTL_UNLOCK(part); } } return (0); } static int mmcsd_resume(device_t dev) { struct mmcsd_softc *sc = device_get_softc(dev); struct mmcsd_part *part; int i; for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; if (part != NULL) { if (part->disk != NULL) { MMCSD_DISK_LOCK(part); part->suspend = 0; if (part->running <= 0) { part->running = 1; MMCSD_DISK_UNLOCK(part); kproc_create(&mmcsd_task, part, &part->p, 0, 0, "%s%d: mmc/sd card", part->name, part->cnt); } else MMCSD_DISK_UNLOCK(part); } MMCSD_IOCTL_LOCK(part); part->ioctl = 0; MMCSD_IOCTL_UNLOCK(part); } } return (0); } static int mmcsd_open(struct disk *dp __unused) { return (0); } static int mmcsd_close(struct disk *dp __unused) { return (0); } static void mmcsd_strategy(struct bio *bp) { struct mmcsd_softc *sc; struct mmcsd_part *part; part = bp->bio_disk->d_drv1; sc = part->sc; MMCSD_DISK_LOCK(part); if (part->running > 0 || part->suspend > 0) { bioq_disksort(&part->bio_queue, bp); MMCSD_DISK_UNLOCK(part); wakeup(part); } else { MMCSD_DISK_UNLOCK(part); biofinish(bp, NULL, ENXIO); } } static int mmcsd_ioctl_rpmb(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td __unused) { return (mmcsd_ioctl(dev->si_drv1, cmd, data, fflag)); } static int mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, int fflag, struct thread *td __unused) { return (mmcsd_ioctl(disk->d_drv1, cmd, data, fflag)); } static int mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) { struct mmc_ioc_cmd *mic; struct mmc_ioc_multi_cmd *mimc; int i, err; u_long cnt, size; if ((fflag & FREAD) == 0) return (EBADF); err = 0; switch (cmd) { case MMC_IOC_CMD: mic = data; err = mmcsd_ioctl_cmd(part, mic, fflag); break; case MMC_IOC_MULTI_CMD: mimc = data; if (mimc->num_of_cmds == 0) break; if (mimc->num_of_cmds > MMC_IOC_MAX_CMDS) return (EINVAL); cnt = mimc->num_of_cmds; size = sizeof(*mic) * cnt; mic = malloc(size, M_TEMP, M_WAITOK); err = copyin((const void *)mimc->cmds, mic, size); if (err == 0) { for (i = 0; i < cnt; i++) { err = mmcsd_ioctl_cmd(part, &mic[i], fflag); if (err != 0) break; } } free(mic, M_TEMP); break; default: return (ENOIOCTL); } return (err); } static int mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) { struct mmc_command cmd; struct mmc_data data; struct mmcsd_softc *sc; device_t dev, mmcbr; void *dp; u_long len; int err, retries; uint32_t status; uint16_t rca; if ((fflag & FWRITE) == 0 && mic->write_flag != 0) return (EBADF); if (part->ro == TRUE && mic->write_flag != 0) return (EROFS); /* * We don't need to explicitly lock against the disk(9) half of this * driver as MMCBUS_ACQUIRE_BUS() will serialize us. However, it's * necessary to protect against races with detachment and suspension, * especially since it's required to switch away from RPMB partitions * again after an access (see mmcsd_switch_part()). */ MMCSD_IOCTL_LOCK(part); while (part->ioctl != 0) { if (part->ioctl < 0) { MMCSD_IOCTL_UNLOCK(part); return (ENXIO); } msleep(part, &part->ioctl_mtx, 0, "mmcsd IOCTL", 0); } part->ioctl = 1; MMCSD_IOCTL_UNLOCK(part); err = 0; dp = NULL; len = mic->blksz * mic->blocks; if (len > MMC_IOC_MAX_BYTES) { err = EOVERFLOW; goto out; } if (len != 0) { dp = malloc(len, M_TEMP, M_WAITOK); err = copyin((void *)(uintptr_t)mic->data_ptr, dp, len); if (err != 0) goto out; } memset(&cmd, 0, sizeof(cmd)); memset(&data, 0, sizeof(data)); cmd.opcode = mic->opcode; cmd.arg = mic->arg; cmd.flags = mic->flags; if (len != 0) { data.len = len; data.data = dp; data.flags = mic->write_flag != 0 ? MMC_DATA_WRITE : MMC_DATA_READ; cmd.data = &data; } sc = part->sc; rca = sc->rca; if (mic->is_acmd == 0) { /* Enforce/patch/restrict RCA-based commands */ switch (cmd.opcode) { case MMC_SET_RELATIVE_ADDR: case MMC_SELECT_CARD: err = EPERM; goto out; case MMC_STOP_TRANSMISSION: if ((cmd.arg & 0x1) == 0) break; /* FALLTHROUGH */ case MMC_SLEEP_AWAKE: case MMC_SEND_CSD: case MMC_SEND_CID: case MMC_SEND_STATUS: case MMC_GO_INACTIVE_STATE: case MMC_FAST_IO: case MMC_APP_CMD: cmd.arg = (cmd.arg & 0x0000FFFF) | (rca << 16); break; default: break; } } dev = sc->dev; mmcbr = sc->mmcbr; MMCBUS_ACQUIRE_BUS(mmcbr, dev); err = mmcsd_switch_part(mmcbr, dev, rca, part->type); if (err != MMC_ERR_NONE) goto release; if (part->type == EXT_CSD_PART_CONFIG_ACC_RPMB) { err = mmcsd_set_blockcount(sc, mic->blocks, mic->write_flag & (1 << 31)); if (err != MMC_ERR_NONE) goto switch_back; } if (mic->is_acmd != 0) (void)mmc_wait_for_app_cmd(mmcbr, dev, rca, &cmd, 0); else (void)mmc_wait_for_cmd(mmcbr, dev, &cmd, 0); if (part->type == EXT_CSD_PART_CONFIG_ACC_RPMB) { /* * If the request went to the RPMB partition, try to ensure * that the command actually has completed ... */ retries = MMCSD_CMD_RETRIES; do { err = mmc_send_status(mmcbr, dev, rca, &status); if (err != MMC_ERR_NONE) break; if (R1_STATUS(status) == 0 && R1_CURRENT_STATE(status) != R1_STATE_PRG) break; DELAY(1000); } while (retries-- > 0); switch_back: /* ... and always switch back to the default partition. */ err = mmcsd_switch_part(mmcbr, dev, rca, EXT_CSD_PART_CONFIG_ACC_DEFAULT); if (err != MMC_ERR_NONE) goto release; } /* * If EXT_CSD was changed, our copy is outdated now. Specifically, * the upper bits of EXT_CSD_PART_CONFIG used in mmcsd_switch_part(), * so retrieve EXT_CSD again. */ if (cmd.opcode == MMC_SWITCH_FUNC) { err = mmc_send_ext_csd(mmcbr, dev, sc->ext_csd); if (err != MMC_ERR_NONE) goto release; } MMCBUS_RELEASE_BUS(mmcbr, dev); if (cmd.error != MMC_ERR_NONE) { switch (cmd.error) { case MMC_ERR_TIMEOUT: err = ETIMEDOUT; break; case MMC_ERR_BADCRC: err = EILSEQ; break; case MMC_ERR_INVALID: err = EINVAL; break; case MMC_ERR_NO_MEMORY: err = ENOMEM; break; default: err = EIO; break; } goto out; } memcpy(mic->response, cmd.resp, 4 * sizeof(uint32_t)); if (mic->write_flag == 0 && len != 0) { err = copyout(dp, (void *)(uintptr_t)mic->data_ptr, len); if (err != 0) goto out; } goto out; release: MMCBUS_RELEASE_BUS(mmcbr, dev); err = EIO; out: MMCSD_IOCTL_LOCK(part); part->ioctl = 0; MMCSD_IOCTL_UNLOCK(part); wakeup(part); if (dp != NULL) free(dp, M_TEMP); return (err); } static int mmcsd_getattr(struct bio *bp) { struct mmcsd_part *part; device_t dev; if (strcmp(bp->bio_attribute, "MMC::device") == 0) { if (bp->bio_length != sizeof(dev)) return (EFAULT); part = bp->bio_disk->d_drv1; dev = part->sc->dev; bcopy(&dev, bp->bio_data, sizeof(dev)); bp->bio_completed = bp->bio_length; return (0); } return (-1); } static int mmcsd_set_blockcount(struct mmcsd_softc *sc, u_int count, bool reliable) { struct mmc_command cmd; struct mmc_request req; memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); cmd.mrq = &req; req.cmd = &cmd; cmd.opcode = MMC_SET_BLOCK_COUNT; cmd.arg = count & 0x0000FFFF; if (reliable) cmd.arg |= 1 << 31; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(sc->mmcbr, sc->dev, &req); return (cmd.error); } static int mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part) { struct mmcsd_softc *sc; int err; uint8_t value; sc = device_get_softc(dev); if (sc->mode == mode_sd) return (MMC_ERR_NONE); /* * According to section "6.2.2 Command restrictions" of the eMMC * specification v5.1, CMD19/CMD21 aren't allowed to be used with * RPMB partitions. So we pause re-tuning along with triggering * it up-front to decrease the likelihood of re-tuning becoming * necessary while accessing an RPMB partition. Consequently, an * RPMB partition should immediately be switched away from again * after an access in order to allow for re-tuning to take place * anew. */ if (part == EXT_CSD_PART_CONFIG_ACC_RPMB) MMCBUS_RETUNE_PAUSE(sc->mmcbr, sc->dev, true); if (sc->part_curr == part) return (MMC_ERR_NONE); value = (sc->ext_csd[EXT_CSD_PART_CONFIG] & ~EXT_CSD_PART_CONFIG_ACC_MASK) | part; /* Jump! */ err = mmc_switch(bus, dev, rca, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_PART_CONFIG, value, sc->part_time, true); if (err != MMC_ERR_NONE) { if (part == EXT_CSD_PART_CONFIG_ACC_RPMB) MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev); return (err); } sc->ext_csd[EXT_CSD_PART_CONFIG] = value; if (sc->part_curr == EXT_CSD_PART_CONFIG_ACC_RPMB) MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev); sc->part_curr = part; return (MMC_ERR_NONE); } static const char * mmcsd_errmsg(int e) { if (e < 0 || e > MMC_ERR_MAX) return "Bad error code"; return (errmsg[e]); } static daddr_t mmcsd_rw(struct mmcsd_part *part, struct bio *bp) { daddr_t block, end; struct mmc_command cmd; struct mmc_command stop; struct mmc_request req; struct mmc_data data; struct mmcsd_softc *sc; device_t dev, mmcbr; u_int numblocks, sz; char *vaddr; sc = part->sc; dev = sc->dev; mmcbr = sc->mmcbr; block = bp->bio_pblkno; sz = part->disk->d_sectorsize; end = bp->bio_pblkno + (bp->bio_bcount / sz); while (block < end) { vaddr = bp->bio_data + (block - bp->bio_pblkno) * sz; numblocks = min(end - block, sc->max_data); memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); memset(&stop, 0, sizeof(stop)); memset(&data, 0, sizeof(data)); cmd.mrq = &req; req.cmd = &cmd; cmd.data = &data; if (bp->bio_cmd == BIO_READ) { if (numblocks > 1) cmd.opcode = MMC_READ_MULTIPLE_BLOCK; else cmd.opcode = MMC_READ_SINGLE_BLOCK; } else { if (numblocks > 1) cmd.opcode = MMC_WRITE_MULTIPLE_BLOCK; else cmd.opcode = MMC_WRITE_BLOCK; } cmd.arg = block; if (sc->high_cap == 0) cmd.arg <<= 9; cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; data.data = vaddr; data.mrq = &req; if (bp->bio_cmd == BIO_READ) data.flags = MMC_DATA_READ; else data.flags = MMC_DATA_WRITE; data.len = numblocks * sz; if (numblocks > 1) { data.flags |= MMC_DATA_MULTI; stop.opcode = MMC_STOP_TRANSMISSION; stop.arg = 0; stop.flags = MMC_RSP_R1B | MMC_CMD_AC; stop.mrq = &req; req.stop = &stop; } MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) device_printf(dev, "Error indicated: %d %s\n", req.cmd->error, mmcsd_errmsg(req.cmd->error)); break; } block += numblocks; } return (block); } static daddr_t mmcsd_delete(struct mmcsd_part *part, struct bio *bp) { daddr_t block, end, start, stop; struct mmc_command cmd; struct mmc_request req; struct mmcsd_softc *sc; device_t dev, mmcbr; u_int erase_sector, sz; sc = part->sc; dev = sc->dev; mmcbr = sc->mmcbr; block = bp->bio_pblkno; sz = part->disk->d_sectorsize; end = bp->bio_pblkno + (bp->bio_bcount / sz); /* Coalesce with part remaining from previous request. */ if (block > part->eblock && block <= part->eend) block = part->eblock; if (end >= part->eblock && end < part->eend) end = part->eend; /* Safe round to the erase sector boundaries. */ erase_sector = sc->erase_sector; start = block + erase_sector - 1; /* Round up. */ start -= start % erase_sector; stop = end; /* Round down. */ stop -= end % erase_sector; /* We can't erase an area smaller than a sector, store it for later. */ if (start >= stop) { part->eblock = block; part->eend = end; return (end); } /* * Pause re-tuning so it won't interfere with the order of erase * commands. Note that these latter don't use the data lines, so * re-tuning shouldn't actually become necessary during erase. */ MMCBUS_RETUNE_PAUSE(mmcbr, dev, false); /* Set erase start position. */ memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); cmd.mrq = &req; req.cmd = &cmd; if (mmc_get_card_type(dev) == mode_sd) cmd.opcode = SD_ERASE_WR_BLK_START; else cmd.opcode = MMC_ERASE_GROUP_START; cmd.arg = start; if (sc->high_cap == 0) cmd.arg <<= 9; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { device_printf(dev, "Setting erase start position failed %d\n", req.cmd->error); block = bp->bio_pblkno; goto unpause; } /* Set erase stop position. */ memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); req.cmd = &cmd; if (mmc_get_card_type(dev) == mode_sd) cmd.opcode = SD_ERASE_WR_BLK_END; else cmd.opcode = MMC_ERASE_GROUP_END; cmd.arg = stop; if (sc->high_cap == 0) cmd.arg <<= 9; cmd.arg--; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { device_printf(dev, "Setting erase stop position failed %d\n", req.cmd->error); block = bp->bio_pblkno; goto unpause; } /* Erase range. */ memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); req.cmd = &cmd; cmd.opcode = MMC_ERASE; cmd.arg = 0; cmd.flags = MMC_RSP_R1B | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { device_printf(dev, "erase err3: %d\n", req.cmd->error); device_printf(dev, "Issuing erase command failed %d\n", req.cmd->error); block = bp->bio_pblkno; goto unpause; } /* Store one of remaining parts for the next call. */ if (bp->bio_pblkno >= part->eblock || block == start) { part->eblock = stop; /* Predict next forward. */ part->eend = end; } else { part->eblock = block; /* Predict next backward. */ part->eend = start; } block = end; unpause: MMCBUS_RETUNE_UNPAUSE(mmcbr, dev); return (block); } static int mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length) { struct bio bp; daddr_t block, end; struct disk *disk; struct mmcsd_softc *sc; struct mmcsd_part *part; device_t dev, mmcbr; int err; /* length zero is special and really means flush buffers to media */ if (!length) return (0); disk = arg; part = disk->d_drv1; sc = part->sc; dev = sc->dev; mmcbr = sc->mmcbr; g_reset_bio(&bp); bp.bio_disk = disk; bp.bio_pblkno = offset / disk->d_sectorsize; bp.bio_bcount = length; bp.bio_data = virtual; bp.bio_cmd = BIO_WRITE; end = bp.bio_pblkno + bp.bio_bcount / disk->d_sectorsize; MMCBUS_ACQUIRE_BUS(mmcbr, dev); err = mmcsd_switch_part(mmcbr, dev, sc->rca, part->type); if (err != MMC_ERR_NONE) { if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) device_printf(dev, "Partition switch error\n"); MMCBUS_RELEASE_BUS(mmcbr, dev); return (EIO); } block = mmcsd_rw(part, &bp); MMCBUS_RELEASE_BUS(mmcbr, dev); return ((end < block) ? EIO : 0); } static void mmcsd_task(void *arg) { daddr_t block, end; struct mmcsd_part *part; struct mmcsd_softc *sc; struct bio *bp; device_t dev, mmcbr; int err, sz; part = arg; sc = part->sc; dev = sc->dev; mmcbr = sc->mmcbr; while (1) { MMCSD_DISK_LOCK(part); do { if (part->running == 0) goto out; bp = bioq_takefirst(&part->bio_queue); if (bp == NULL) msleep(part, &part->disk_mtx, PRIBIO, "mmcsd disk jobqueue", 0); } while (bp == NULL); MMCSD_DISK_UNLOCK(part); if (bp->bio_cmd != BIO_READ && part->ro) { bp->bio_error = EROFS; bp->bio_resid = bp->bio_bcount; bp->bio_flags |= BIO_ERROR; biodone(bp); continue; } MMCBUS_ACQUIRE_BUS(mmcbr, dev); sz = part->disk->d_sectorsize; block = bp->bio_pblkno; end = bp->bio_pblkno + (bp->bio_bcount / sz); err = mmcsd_switch_part(mmcbr, dev, sc->rca, part->type); if (err != MMC_ERR_NONE) { if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) device_printf(dev, "Partition switch error\n"); goto release; } if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { /* Access to the remaining erase block obsoletes it. */ if (block < part->eend && end > part->eblock) part->eblock = part->eend = 0; block = mmcsd_rw(part, bp); } else if (bp->bio_cmd == BIO_DELETE) { block = mmcsd_delete(part, bp); } release: MMCBUS_RELEASE_BUS(mmcbr, dev); if (block < end) { bp->bio_error = EIO; bp->bio_resid = (end - block) * sz; bp->bio_flags |= BIO_ERROR; } else { bp->bio_resid = 0; } biodone(bp); } out: /* tell parent we're done */ part->running = -1; MMCSD_DISK_UNLOCK(part); wakeup(part); kproc_exit(0); } static int mmcsd_bus_bit_width(device_t dev) { if (mmc_get_bus_width(dev) == bus_width_1) return (1); if (mmc_get_bus_width(dev) == bus_width_4) return (4); return (8); } static device_method_t mmcsd_methods[] = { DEVMETHOD(device_probe, mmcsd_probe), DEVMETHOD(device_attach, mmcsd_attach), DEVMETHOD(device_detach, mmcsd_detach), DEVMETHOD(device_suspend, mmcsd_suspend), DEVMETHOD(device_resume, mmcsd_resume), DEVMETHOD_END }; static driver_t mmcsd_driver = { "mmcsd", mmcsd_methods, sizeof(struct mmcsd_softc), }; static devclass_t mmcsd_devclass; static int mmcsd_handler(module_t mod __unused, int what, void *arg __unused) { switch (what) { case MOD_LOAD: flash_register_slicer(mmcsd_slicer, FLASH_SLICES_TYPE_MMC, TRUE); return (0); case MOD_UNLOAD: flash_register_slicer(NULL, FLASH_SLICES_TYPE_MMC, TRUE); return (0); } return (0); } DRIVER_MODULE(mmcsd, mmc, mmcsd_driver, mmcsd_devclass, mmcsd_handler, NULL); MODULE_DEPEND(mmcsd, g_flashmap, 0, 0, 0); MMC_DEPEND(mmcsd); Index: projects/runtime-coverage/sys/dev/mpr/mpr_sas.c =================================================================== --- projects/runtime-coverage/sys/dev/mpr/mpr_sas.c (revision 321584) +++ projects/runtime-coverage/sys/dev/mpr/mpr_sas.c (revision 321585) @@ -1,4009 +1,4009 @@ /*- * Copyright (c) 2009 Yahoo! Inc. * Copyright (c) 2011-2015 LSI Corp. * Copyright (c) 2013-2016 Avago Technologies * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD * */ #include __FBSDID("$FreeBSD$"); /* Communications core for Avago Technologies (LSI) MPT3 */ /* TODO Move headers to mprvar */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if __FreeBSD_version >= 900026 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #define MPRSAS_DISCOVERY_TIMEOUT 20 #define MPRSAS_MAX_DISCOVERY_TIMEOUTS 10 /* 200 seconds */ /* * static array to check SCSI OpCode for EEDP protection bits */ #define PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP #define PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP #define PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP static uint8_t op_code_prot[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V, 0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; MALLOC_DEFINE(M_MPRSAS, "MPRSAS", "MPR SAS memory"); static void mprsas_remove_device(struct mpr_softc *, struct mpr_command *); static void mprsas_remove_complete(struct mpr_softc *, struct mpr_command *); static void mprsas_action(struct cam_sim *sim, union ccb *ccb); static void mprsas_poll(struct cam_sim *sim); static void mprsas_scsiio_timeout(void *data); static void mprsas_abort_complete(struct mpr_softc *sc, struct mpr_command *cm); static void mprsas_action_scsiio(struct mprsas_softc *, union ccb *); static void mprsas_scsiio_complete(struct mpr_softc *, struct mpr_command *); static void mprsas_action_resetdev(struct mprsas_softc *, union ccb *); static void mprsas_resetdev_complete(struct mpr_softc *, struct mpr_command *); static int mprsas_send_abort(struct mpr_softc *sc, struct mpr_command *tm, struct mpr_command *cm); static void mprsas_async(void *callback_arg, uint32_t code, struct cam_path *path, void *arg); #if (__FreeBSD_version < 901503) || \ ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) static void mprsas_check_eedp(struct mpr_softc *sc, struct cam_path *path, struct ccb_getdev *cgd); static void mprsas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb); #endif static int mprsas_send_portenable(struct mpr_softc *sc); static void mprsas_portenable_complete(struct mpr_softc *sc, struct mpr_command *cm); #if __FreeBSD_version >= 900026 static void mprsas_smpio_complete(struct mpr_softc *sc, struct mpr_command *cm); static void mprsas_send_smpcmd(struct mprsas_softc *sassc, union ccb *ccb, uint64_t sasaddr); static void mprsas_action_smpio(struct mprsas_softc *sassc, union ccb *ccb); #endif //FreeBSD_version >= 900026 struct mprsas_target * mprsas_find_target_by_handle(struct mprsas_softc *sassc, int start, uint16_t handle) { struct mprsas_target *target; int i; for (i = start; i < sassc->maxtargets; i++) { target = &sassc->targets[i]; if (target->handle == handle) return (target); } return (NULL); } /* we need to freeze the simq during attach and diag reset, to avoid failing * commands before device handles have been found by discovery. Since * discovery involves reading config pages and possibly sending commands, * discovery actions may continue even after we receive the end of discovery * event, so refcount discovery actions instead of assuming we can unfreeze * the simq when we get the event. */ void mprsas_startup_increment(struct mprsas_softc *sassc) { MPR_FUNCTRACE(sassc->sc); if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) { if (sassc->startup_refcount++ == 0) { /* just starting, freeze the simq */ mpr_dprint(sassc->sc, MPR_INIT, "%s freezing simq\n", __func__); #if (__FreeBSD_version >= 1000039) || \ ((__FreeBSD_version < 1000000) && (__FreeBSD_version >= 902502)) xpt_hold_boot(); #endif xpt_freeze_simq(sassc->sim, 1); } mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__, sassc->startup_refcount); } } void mprsas_release_simq_reinit(struct mprsas_softc *sassc) { if (sassc->flags & MPRSAS_QUEUE_FROZEN) { sassc->flags &= ~MPRSAS_QUEUE_FROZEN; xpt_release_simq(sassc->sim, 1); mpr_dprint(sassc->sc, MPR_INFO, "Unfreezing SIM queue\n"); } } void mprsas_startup_decrement(struct mprsas_softc *sassc) { MPR_FUNCTRACE(sassc->sc); if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) { if (--sassc->startup_refcount == 0) { /* finished all discovery-related actions, release * the simq and rescan for the latest topology. */ mpr_dprint(sassc->sc, MPR_INIT, "%s releasing simq\n", __func__); sassc->flags &= ~MPRSAS_IN_STARTUP; xpt_release_simq(sassc->sim, 1); #if (__FreeBSD_version >= 1000039) || \ ((__FreeBSD_version < 1000000) && (__FreeBSD_version >= 902502)) xpt_release_boot(); #else mprsas_rescan_target(sassc->sc, NULL); #endif } mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__, sassc->startup_refcount); } } /* The firmware requires us to stop sending commands when we're doing task * management, so refcount the TMs and keep the simq frozen when any are in * use. */ struct mpr_command * mprsas_alloc_tm(struct mpr_softc *sc) { struct mpr_command *tm; MPR_FUNCTRACE(sc); tm = mpr_alloc_high_priority_command(sc); return tm; } void mprsas_free_tm(struct mpr_softc *sc, struct mpr_command *tm) { int target_id = 0xFFFFFFFF; MPR_FUNCTRACE(sc); if (tm == NULL) return; /* * For TM's the devq is frozen for the device. Unfreeze it here and * free the resources used for freezing the devq. Must clear the * INRESET flag as well or scsi I/O will not work. */ if (tm->cm_targ != NULL) { tm->cm_targ->flags &= ~MPRSAS_TARGET_INRESET; target_id = tm->cm_targ->tid; } if (tm->cm_ccb) { mpr_dprint(sc, MPR_INFO, "Unfreezing devq for target ID %d\n", target_id); xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE); xpt_free_path(tm->cm_ccb->ccb_h.path); xpt_free_ccb(tm->cm_ccb); } mpr_free_high_priority_command(sc, tm); } void mprsas_rescan_target(struct mpr_softc *sc, struct mprsas_target *targ) { struct mprsas_softc *sassc = sc->sassc; path_id_t pathid; target_id_t targetid; union ccb *ccb; MPR_FUNCTRACE(sc); pathid = cam_sim_path(sassc->sim); if (targ == NULL) targetid = CAM_TARGET_WILDCARD; else targetid = targ - sassc->targets; /* * Allocate a CCB and schedule a rescan. */ ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { mpr_dprint(sc, MPR_ERROR, "unable to alloc CCB for rescan\n"); return; } if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { mpr_dprint(sc, MPR_ERROR, "unable to create path for rescan\n"); xpt_free_ccb(ccb); return; } if (targetid == CAM_TARGET_WILDCARD) ccb->ccb_h.func_code = XPT_SCAN_BUS; else ccb->ccb_h.func_code = XPT_SCAN_TGT; mpr_dprint(sc, MPR_TRACE, "%s targetid %u\n", __func__, targetid); xpt_rescan(ccb); } static void mprsas_log_command(struct mpr_command *cm, u_int level, const char *fmt, ...) { struct sbuf sb; va_list ap; char str[192]; char path_str[64]; if (cm == NULL) return; /* No need to be in here if debugging isn't enabled */ if ((cm->cm_sc->mpr_debug & level) == 0) return; sbuf_new(&sb, str, sizeof(str), 0); va_start(ap, fmt); if (cm->cm_ccb != NULL) { xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str, sizeof(path_str)); sbuf_cat(&sb, path_str); if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) { scsi_command_string(&cm->cm_ccb->csio, &sb); sbuf_printf(&sb, "length %d ", cm->cm_ccb->csio.dxfer_len); } } else { sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ", cam_sim_name(cm->cm_sc->sassc->sim), cam_sim_unit(cm->cm_sc->sassc->sim), cam_sim_bus(cm->cm_sc->sassc->sim), cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF, cm->cm_lun); } sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID); sbuf_vprintf(&sb, fmt, ap); sbuf_finish(&sb); mpr_print_field(cm->cm_sc, "%s", sbuf_data(&sb)); va_end(ap); } static void mprsas_remove_volume(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SCSI_TASK_MANAGE_REPLY *reply; struct mprsas_target *targ; uint16_t handle; MPR_FUNCTRACE(sc); reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; handle = (uint16_t)(uintptr_t)tm->cm_complete_data; targ = tm->cm_targ; if (reply == NULL) { /* XXX retry the remove after the diag reset completes? */ mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device " "0x%04x\n", __func__, handle); mprsas_free_tm(sc, tm); return; } if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) { mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting " "device 0x%x\n", le16toh(reply->IOCStatus), handle); } mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n", le32toh(reply->TerminationCount)); mpr_free_reply(sc, tm->cm_reply_data); tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */ mpr_dprint(sc, MPR_XINFO, "clearing target %u handle 0x%04x\n", targ->tid, handle); /* * Don't clear target if remove fails because things will get confusing. * Leave the devname and sasaddr intact so that we know to avoid reusing * this target id if possible, and so we can assign the same target id * to this device if it comes back in the future. */ if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) == MPI2_IOCSTATUS_SUCCESS) { targ = tm->cm_targ; targ->handle = 0x0; targ->encl_handle = 0x0; targ->encl_level_valid = 0x0; targ->encl_level = 0x0; targ->connector_name[0] = ' '; targ->connector_name[1] = ' '; targ->connector_name[2] = ' '; targ->connector_name[3] = ' '; targ->encl_slot = 0x0; targ->exp_dev_handle = 0x0; targ->phy_num = 0x0; targ->linkrate = 0x0; targ->devinfo = 0x0; targ->flags = 0x0; targ->scsi_req_desc_type = 0; } mprsas_free_tm(sc, tm); } /* * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal. * Otherwise Volume Delete is same as Bare Drive Removal. */ void mprsas_prepare_volume_remove(struct mprsas_softc *sassc, uint16_t handle) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mpr_softc *sc; struct mpr_command *cm; struct mprsas_target *targ = NULL; MPR_FUNCTRACE(sassc->sc); sc = sassc->sc; targ = mprsas_find_target_by_handle(sassc, 0, handle); if (targ == NULL) { /* FIXME: what is the action? */ /* We don't know about this device? */ mpr_dprint(sc, MPR_ERROR, "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle); return; } targ->flags |= MPRSAS_TARGET_INREMOVAL; cm = mprsas_alloc_tm(sc); if (cm == NULL) { mpr_dprint(sc, MPR_ERROR, "%s: command alloc failure\n", __func__); return; } mprsas_rescan_target(sc, targ); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req; req->DevHandle = targ->handle; req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET; /* SAS Hard Link Reset / SATA Link Reset */ req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET; cm->cm_targ = targ; cm->cm_data = NULL; cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; cm->cm_complete = mprsas_remove_volume; cm->cm_complete_data = (void *)(uintptr_t)handle; mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n", __func__, targ->tid); mprsas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD); mpr_map_command(sc, cm); } /* * The firmware performs debounce on the link to avoid transient link errors * and false removals. When it does decide that link has been lost and a * device needs to go away, it expects that the host will perform a target reset * and then an op remove. The reset has the side-effect of aborting any * outstanding requests for the device, which is required for the op-remove to * succeed. It's not clear if the host should check for the device coming back * alive after the reset. */ void mprsas_prepare_remove(struct mprsas_softc *sassc, uint16_t handle) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mpr_softc *sc; struct mpr_command *cm; struct mprsas_target *targ = NULL; MPR_FUNCTRACE(sassc->sc); sc = sassc->sc; targ = mprsas_find_target_by_handle(sassc, 0, handle); if (targ == NULL) { /* FIXME: what is the action? */ /* We don't know about this device? */ mpr_dprint(sc, MPR_ERROR, "%s : invalid handle 0x%x \n", __func__, handle); return; } targ->flags |= MPRSAS_TARGET_INREMOVAL; cm = mprsas_alloc_tm(sc); if (cm == NULL) { mpr_dprint(sc, MPR_ERROR, "%s: command alloc failure\n", __func__); return; } mprsas_rescan_target(sc, targ); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req; memset(req, 0, sizeof(*req)); req->DevHandle = htole16(targ->handle); req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET; /* SAS Hard Link Reset / SATA Link Reset */ req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET; cm->cm_targ = targ; cm->cm_data = NULL; cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; cm->cm_complete = mprsas_remove_device; cm->cm_complete_data = (void *)(uintptr_t)handle; mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n", __func__, targ->tid); mprsas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD); mpr_map_command(sc, cm); } static void mprsas_remove_device(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SCSI_TASK_MANAGE_REPLY *reply; MPI2_SAS_IOUNIT_CONTROL_REQUEST *req; struct mprsas_target *targ; struct mpr_command *next_cm; uint16_t handle; MPR_FUNCTRACE(sc); reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; handle = (uint16_t)(uintptr_t)tm->cm_complete_data; targ = tm->cm_targ; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for remove of " "handle %#04x! This should not happen!\n", __func__, tm->cm_flags, handle); } if (reply == NULL) { /* XXX retry the remove after the diag reset completes? */ mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device " "0x%04x\n", __func__, handle); mprsas_free_tm(sc, tm); return; } if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) { mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting " "device 0x%x\n", le16toh(reply->IOCStatus), handle); } mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n", le32toh(reply->TerminationCount)); mpr_free_reply(sc, tm->cm_reply_data); tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */ /* Reuse the existing command */ req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req; memset(req, 0, sizeof(*req)); req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL; req->Operation = MPI2_SAS_OP_REMOVE_DEVICE; req->DevHandle = htole16(handle); tm->cm_data = NULL; tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; tm->cm_complete = mprsas_remove_complete; tm->cm_complete_data = (void *)(uintptr_t)handle; mpr_map_command(sc, tm); mpr_dprint(sc, MPR_INFO, "clearing target %u handle 0x%04x\n", targ->tid, handle); if (targ->encl_level_valid) { mpr_dprint(sc, MPR_INFO, "At enclosure level %d, slot %d, " "connector name (%4s)\n", targ->encl_level, targ->encl_slot, targ->connector_name); } TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) { union ccb *ccb; mpr_dprint(sc, MPR_XINFO, "Completing missed command %p\n", tm); ccb = tm->cm_complete_data; mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); mprsas_scsiio_complete(sc, tm); } } static void mprsas_remove_complete(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SAS_IOUNIT_CONTROL_REPLY *reply; uint16_t handle; struct mprsas_target *targ; struct mprsas_lun *lun; MPR_FUNCTRACE(sc); reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply; handle = (uint16_t)(uintptr_t)tm->cm_complete_data; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_XINFO, "%s: cm_flags = %#x for remove of " "handle %#04x! This should not happen!\n", __func__, tm->cm_flags, handle); mprsas_free_tm(sc, tm); return; } if (reply == NULL) { /* most likely a chip reset */ mpr_dprint(sc, MPR_FAULT, "%s NULL reply removing device " "0x%04x\n", __func__, handle); mprsas_free_tm(sc, tm); return; } mpr_dprint(sc, MPR_XINFO, "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__, handle, le16toh(reply->IOCStatus)); /* * Don't clear target if remove fails because things will get confusing. * Leave the devname and sasaddr intact so that we know to avoid reusing * this target id if possible, and so we can assign the same target id * to this device if it comes back in the future. */ if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) == MPI2_IOCSTATUS_SUCCESS) { targ = tm->cm_targ; targ->handle = 0x0; targ->encl_handle = 0x0; targ->encl_level_valid = 0x0; targ->encl_level = 0x0; targ->connector_name[0] = ' '; targ->connector_name[1] = ' '; targ->connector_name[2] = ' '; targ->connector_name[3] = ' '; targ->encl_slot = 0x0; targ->exp_dev_handle = 0x0; targ->phy_num = 0x0; targ->linkrate = 0x0; targ->devinfo = 0x0; targ->flags = 0x0; targ->scsi_req_desc_type = 0; while (!SLIST_EMPTY(&targ->luns)) { lun = SLIST_FIRST(&targ->luns); SLIST_REMOVE_HEAD(&targ->luns, lun_link); free(lun, M_MPR); } } mprsas_free_tm(sc, tm); } static int mprsas_register_events(struct mpr_softc *sc) { uint8_t events[16]; bzero(events, 16); setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE); setbit(events, MPI2_EVENT_SAS_DISCOVERY); setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE); setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE); setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW); setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST); setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE); setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST); setbit(events, MPI2_EVENT_IR_VOLUME); setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK); setbit(events, MPI2_EVENT_IR_OPERATION_STATUS); setbit(events, MPI2_EVENT_TEMP_THRESHOLD); if (sc->facts->MsgVersion >= MPI2_VERSION_02_06) { setbit(events, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION); if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) { setbit(events, MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE); setbit(events, MPI2_EVENT_PCIE_ENUMERATION); setbit(events, MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST); } } mpr_register_events(sc, events, mprsas_evt_handler, NULL, &sc->sassc->mprsas_eh); return (0); } int mpr_attach_sas(struct mpr_softc *sc) { struct mprsas_softc *sassc; cam_status status; int unit, error = 0; MPR_FUNCTRACE(sc); sassc = malloc(sizeof(struct mprsas_softc), M_MPR, M_WAITOK|M_ZERO); if (!sassc) { device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", __func__, __LINE__); return (ENOMEM); } /* * XXX MaxTargets could change during a reinit. Since we don't * resize the targets[] array during such an event, cache the value * of MaxTargets here so that we don't get into trouble later. This * should move into the reinit logic. */ sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes; sassc->targets = malloc(sizeof(struct mprsas_target) * sassc->maxtargets, M_MPR, M_WAITOK|M_ZERO); if (!sassc->targets) { device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", __func__, __LINE__); free(sassc, M_MPR); return (ENOMEM); } sc->sassc = sassc; sassc->sc = sc; if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate SIMQ\n"); error = ENOMEM; goto out; } unit = device_get_unit(sc->mpr_dev); sassc->sim = cam_sim_alloc(mprsas_action, mprsas_poll, "mpr", sassc, unit, &sc->mpr_mtx, sc->num_reqs, sc->num_reqs, sassc->devq); if (sassc->sim == NULL) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate SIM\n"); error = EINVAL; goto out; } TAILQ_INIT(&sassc->ev_queue); /* Initialize taskqueue for Event Handling */ TASK_INIT(&sassc->ev_task, 0, mprsas_firmware_event_work, sc); sassc->ev_tq = taskqueue_create("mpr_taskq", M_NOWAIT | M_ZERO, taskqueue_thread_enqueue, &sassc->ev_tq); taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq", device_get_nameunit(sc->mpr_dev)); mpr_lock(sc); /* * XXX There should be a bus for every port on the adapter, but since * we're just going to fake the topology for now, we'll pretend that * everything is just a target on a single bus. */ if ((error = xpt_bus_register(sassc->sim, sc->mpr_dev, 0)) != 0) { mpr_dprint(sc, MPR_ERROR, "Error %d registering SCSI bus\n", error); mpr_unlock(sc); goto out; } /* * Assume that discovery events will start right away. * * Hold off boot until discovery is complete. */ sassc->flags |= MPRSAS_IN_STARTUP | MPRSAS_IN_DISCOVERY; sc->sassc->startup_refcount = 0; mprsas_startup_increment(sassc); callout_init(&sassc->discovery_callout, 1 /*mpsafe*/); /* * Register for async events so we can determine the EEDP * capabilities of devices. */ status = xpt_create_path(&sassc->path, /*periph*/NULL, cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) { mpr_printf(sc, "Error %#x creating sim path\n", status); sassc->path = NULL; } else { int event; #if (__FreeBSD_version >= 1000006) || \ ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000)) event = AC_ADVINFO_CHANGED | AC_FOUND_DEVICE; #else event = AC_FOUND_DEVICE; #endif /* * Prior to the CAM locking improvements, we can't call * xpt_register_async() with a particular path specified. * * If a path isn't specified, xpt_register_async() will * generate a wildcard path and acquire the XPT lock while * it calls xpt_action() to execute the XPT_SASYNC_CB CCB. * It will then drop the XPT lock once that is done. * * If a path is specified for xpt_register_async(), it will * not acquire and drop the XPT lock around the call to * xpt_action(). xpt_action() asserts that the caller * holds the SIM lock, so the SIM lock has to be held when * calling xpt_register_async() when the path is specified. * * But xpt_register_async calls xpt_for_all_devices(), * which calls xptbustraverse(), which will acquire each * SIM lock. When it traverses our particular bus, it will * necessarily acquire the SIM lock, which will lead to a * recursive lock acquisition. * * The CAM locking changes fix this problem by acquiring * the XPT topology lock around bus traversal in * xptbustraverse(), so the caller can hold the SIM lock * and it does not cause a recursive lock acquisition. * * These __FreeBSD_version values are approximate, especially * for stable/10, which is two months later than the actual * change. */ #if (__FreeBSD_version < 1000703) || \ ((__FreeBSD_version >= 1100000) && (__FreeBSD_version < 1100002)) mpr_unlock(sc); status = xpt_register_async(event, mprsas_async, sc, NULL); mpr_lock(sc); #else status = xpt_register_async(event, mprsas_async, sc, sassc->path); #endif if (status != CAM_REQ_CMP) { mpr_dprint(sc, MPR_ERROR, "Error %#x registering async handler for " "AC_ADVINFO_CHANGED events\n", status); xpt_free_path(sassc->path); sassc->path = NULL; } } if (status != CAM_REQ_CMP) { /* * EEDP use is the exception, not the rule. * Warn the user, but do not fail to attach. */ mpr_printf(sc, "EEDP capabilities disabled.\n"); } mpr_unlock(sc); mprsas_register_events(sc); out: if (error) mpr_detach_sas(sc); return (error); } int mpr_detach_sas(struct mpr_softc *sc) { struct mprsas_softc *sassc; struct mprsas_lun *lun, *lun_tmp; struct mprsas_target *targ; int i; MPR_FUNCTRACE(sc); if (sc->sassc == NULL) return (0); sassc = sc->sassc; mpr_deregister_events(sc, sassc->mprsas_eh); /* * Drain and free the event handling taskqueue with the lock * unheld so that any parallel processing tasks drain properly * without deadlocking. */ if (sassc->ev_tq != NULL) taskqueue_free(sassc->ev_tq); /* Make sure CAM doesn't wedge if we had to bail out early. */ mpr_lock(sc); /* Deregister our async handler */ if (sassc->path != NULL) { xpt_register_async(0, mprsas_async, sc, sassc->path); xpt_free_path(sassc->path); sassc->path = NULL; } if (sassc->flags & MPRSAS_IN_STARTUP) xpt_release_simq(sassc->sim, 1); if (sassc->sim != NULL) { xpt_bus_deregister(cam_sim_path(sassc->sim)); cam_sim_free(sassc->sim, FALSE); } mpr_unlock(sc); if (sassc->devq != NULL) cam_simq_free(sassc->devq); for (i = 0; i < sassc->maxtargets; i++) { targ = &sassc->targets[i]; SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) { free(lun, M_MPR); } } free(sassc->targets, M_MPR); free(sassc, M_MPR); sc->sassc = NULL; return (0); } void mprsas_discovery_end(struct mprsas_softc *sassc) { struct mpr_softc *sc = sassc->sc; MPR_FUNCTRACE(sc); if (sassc->flags & MPRSAS_DISCOVERY_TIMEOUT_PENDING) callout_stop(&sassc->discovery_callout); /* * After discovery has completed, check the mapping table for any * missing devices and update their missing counts. Only do this once * whenever the driver is initialized so that missing counts aren't * updated unnecessarily. Note that just because discovery has * completed doesn't mean that events have been processed yet. The * check_devices function is a callout timer that checks if ALL devices * are missing. If so, it will wait a little longer for events to * complete and keep resetting itself until some device in the mapping * table is not missing, meaning that event processing has started. */ if (sc->track_mapping_events) { mpr_dprint(sc, MPR_XINFO | MPR_MAPPING, "Discovery has " "completed. Check for missing devices in the mapping " "table.\n"); callout_reset(&sc->device_check_callout, MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices, sc); } } static void mprsas_action(struct cam_sim *sim, union ccb *ccb) { struct mprsas_softc *sassc; sassc = cam_sim_softc(sim); MPR_FUNCTRACE(sassc->sc); mpr_dprint(sassc->sc, MPR_TRACE, "ccb func_code 0x%x\n", ccb->ccb_h.func_code); mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED); switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; struct mpr_softc *sc = sassc->sc; uint8_t sges_per_frame; cpi->version_num = 1; cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16; cpi->target_sprt = 0; #if (__FreeBSD_version >= 1000039) || \ ((__FreeBSD_version < 1000000) && (__FreeBSD_version >= 902502)) cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN; #else cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED; #endif cpi->hba_eng_cnt = 0; cpi->max_target = sassc->maxtargets - 1; cpi->max_lun = 255; /* * initiator_id is set here to an ID outside the set of valid * target IDs (including volumes). */ cpi->initiator_id = sassc->maxtargets; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); /* * XXXSLM-I think this needs to change based on config page or * something instead of hardcoded to 150000. */ cpi->base_transfer_speed = 150000; cpi->transport = XPORT_SAS; cpi->transport_version = 0; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC; /* * Max IO Size is Page Size * the following: * ((SGEs per frame - 1 for chain element) * * Max Chain Depth) + 1 for no chain needed in last frame * * If user suggests a Max IO size to use, use the smaller of the * user's value and the calculated value as long as the user's * value is larger than 0. The user's value is in pages. */ sges_per_frame = (sc->chain_frame_size / sizeof(MPI2_IEEE_SGE_SIMPLE64)) - 1; cpi->maxio = (sges_per_frame * sc->facts->MaxChainDepth) + 1; cpi->maxio *= PAGE_SIZE; if ((sc->max_io_pages > 0) && (sc->max_io_pages * PAGE_SIZE < cpi->maxio)) cpi->maxio = sc->max_io_pages * PAGE_SIZE; sc->maxio = cpi->maxio; mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts; struct ccb_trans_settings_sas *sas; struct ccb_trans_settings_scsi *scsi; struct mprsas_target *targ; cts = &ccb->cts; sas = &cts->xport_specific.sas; scsi = &cts->proto_specific.scsi; KASSERT(cts->ccb_h.target_id < sassc->maxtargets, ("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n", cts->ccb_h.target_id)); targ = &sassc->targets[cts->ccb_h.target_id]; if (targ->handle == 0x0) { mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); break; } cts->protocol_version = SCSI_REV_SPC2; cts->transport = XPORT_SAS; cts->transport_version = 0; sas->valid = CTS_SAS_VALID_SPEED; switch (targ->linkrate) { case 0x08: sas->bitrate = 150000; break; case 0x09: sas->bitrate = 300000; break; case 0x0a: sas->bitrate = 600000; break; case 0x0b: sas->bitrate = 1200000; break; default: sas->valid = 0; } cts->protocol = PROTO_SCSI; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); break; } case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, /*extended*/1); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); break; case XPT_RESET_DEV: mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action " "XPT_RESET_DEV\n"); mprsas_action_resetdev(sassc, ccb); return; case XPT_RESET_BUS: case XPT_ABORT: case XPT_TERM_IO: mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action faking success " "for abort or reset\n"); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); break; case XPT_SCSI_IO: mprsas_action_scsiio(sassc, ccb); return; #if __FreeBSD_version >= 900026 case XPT_SMP_IO: mprsas_action_smpio(sassc, ccb); return; #endif default: mprsas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL); break; } xpt_done(ccb); } static void mprsas_announce_reset(struct mpr_softc *sc, uint32_t ac_code, target_id_t target_id, lun_id_t lun_id) { path_id_t path_id = cam_sim_path(sc->sassc->sim); struct cam_path *path; mpr_dprint(sc, MPR_XINFO, "%s code %x target %d lun %jx\n", __func__, ac_code, target_id, (uintmax_t)lun_id); if (xpt_create_path(&path, NULL, path_id, target_id, lun_id) != CAM_REQ_CMP) { mpr_dprint(sc, MPR_ERROR, "unable to create path for reset " "notification\n"); return; } xpt_async(ac_code, path, NULL); xpt_free_path(path); } static void mprsas_complete_all_commands(struct mpr_softc *sc) { struct mpr_command *cm; int i; int completed; MPR_FUNCTRACE(sc); mtx_assert(&sc->mpr_mtx, MA_OWNED); /* complete all commands with a NULL reply */ for (i = 1; i < sc->num_reqs; i++) { cm = &sc->commands[i]; cm->cm_reply = NULL; completed = 0; if (cm->cm_flags & MPR_CM_FLAGS_POLLED) cm->cm_flags |= MPR_CM_FLAGS_COMPLETE; if (cm->cm_complete != NULL) { mprsas_log_command(cm, MPR_RECOVERY, "completing cm %p state %x ccb %p for diag reset\n", cm, cm->cm_state, cm->cm_ccb); cm->cm_complete(sc, cm); completed = 1; } if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) { mprsas_log_command(cm, MPR_RECOVERY, "waking up cm %p state %x ccb %p for diag reset\n", cm, cm->cm_state, cm->cm_ccb); wakeup(cm); completed = 1; } if (cm->cm_sc->io_cmds_active != 0) { cm->cm_sc->io_cmds_active--; } else { mpr_dprint(cm->cm_sc, MPR_INFO, "Warning: " "io_cmds_active is out of sync - resynching to " "0\n"); } if ((completed == 0) && (cm->cm_state != MPR_CM_STATE_FREE)) { /* this should never happen, but if it does, log */ mprsas_log_command(cm, MPR_RECOVERY, "cm %p state %x flags 0x%x ccb %p during diag " "reset\n", cm, cm->cm_state, cm->cm_flags, cm->cm_ccb); } } } void mprsas_handle_reinit(struct mpr_softc *sc) { int i; /* Go back into startup mode and freeze the simq, so that CAM * doesn't send any commands until after we've rediscovered all * targets and found the proper device handles for them. * * After the reset, portenable will trigger discovery, and after all * discovery-related activities have finished, the simq will be * released. */ mpr_dprint(sc, MPR_INIT, "%s startup\n", __func__); sc->sassc->flags |= MPRSAS_IN_STARTUP; sc->sassc->flags |= MPRSAS_IN_DISCOVERY; mprsas_startup_increment(sc->sassc); /* notify CAM of a bus reset */ mprsas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); /* complete and cleanup after all outstanding commands */ mprsas_complete_all_commands(sc); mpr_dprint(sc, MPR_INIT, "%s startup %u after command completion\n", __func__, sc->sassc->startup_refcount); /* zero all the target handles, since they may change after the * reset, and we have to rediscover all the targets and use the new * handles. */ for (i = 0; i < sc->sassc->maxtargets; i++) { if (sc->sassc->targets[i].outstanding != 0) mpr_dprint(sc, MPR_INIT, "target %u outstanding %u\n", i, sc->sassc->targets[i].outstanding); sc->sassc->targets[i].handle = 0x0; sc->sassc->targets[i].exp_dev_handle = 0x0; sc->sassc->targets[i].outstanding = 0; sc->sassc->targets[i].flags = MPRSAS_TARGET_INDIAGRESET; } } static void mprsas_tm_timeout(void *data) { struct mpr_command *tm = data; struct mpr_softc *sc = tm->cm_sc; mtx_assert(&sc->mpr_mtx, MA_OWNED); mprsas_log_command(tm, MPR_INFO|MPR_RECOVERY, "task mgmt %p timed " "out\n", tm); mpr_reinit(sc); } static void mprsas_logical_unit_reset_complete(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SCSI_TASK_MANAGE_REPLY *reply; MPI2_SCSI_TASK_MANAGE_REQUEST *req; unsigned int cm_count = 0; struct mpr_command *cm; struct mprsas_target *targ; callout_stop(&tm->cm_callout); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; targ = tm->cm_targ; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for LUN reset! " "This should not happen!\n", __func__, tm->cm_flags); mprsas_free_tm(sc, tm); return; } if (reply == NULL) { mprsas_log_command(tm, MPR_RECOVERY, "NULL reset reply for tm " "%p\n", tm); if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) { /* this completion was due to a reset, just cleanup */ targ->tm = NULL; mprsas_free_tm(sc, tm); } else { /* we should have gotten a reply. */ mpr_reinit(sc); } return; } mprsas_log_command(tm, MPR_RECOVERY, "logical unit reset status 0x%x code 0x%x count %u\n", le16toh(reply->IOCStatus), le32toh(reply->ResponseCode), le32toh(reply->TerminationCount)); /* See if there are any outstanding commands for this LUN. * This could be made more efficient by using a per-LU data * structure of some sort. */ TAILQ_FOREACH(cm, &targ->commands, cm_link) { if (cm->cm_lun == tm->cm_lun) cm_count++; } if (cm_count == 0) { mprsas_log_command(tm, MPR_RECOVERY|MPR_INFO, "logical unit %u finished recovery after reset\n", tm->cm_lun, tm); mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid, tm->cm_lun); /* we've finished recovery for this logical unit. check and * see if some other logical unit has a timedout command * that needs to be processed. */ cm = TAILQ_FIRST(&targ->timedout_commands); if (cm) { mprsas_send_abort(sc, tm, cm); } else { targ->tm = NULL; mprsas_free_tm(sc, tm); } } else { /* if we still have commands for this LUN, the reset * effectively failed, regardless of the status reported. * Escalate to a target reset. */ mprsas_log_command(tm, MPR_RECOVERY, "logical unit reset complete for tm %p, but still have %u " "command(s)\n", tm, cm_count); mprsas_send_reset(sc, tm, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET); } } static void mprsas_target_reset_complete(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SCSI_TASK_MANAGE_REPLY *reply; MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mprsas_target *targ; callout_stop(&tm->cm_callout); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; targ = tm->cm_targ; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for target " "reset! This should not happen!\n", __func__, tm->cm_flags); mprsas_free_tm(sc, tm); return; } if (reply == NULL) { mprsas_log_command(tm, MPR_RECOVERY, "NULL reset reply for tm " "%p\n", tm); if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) { /* this completion was due to a reset, just cleanup */ targ->tm = NULL; mprsas_free_tm(sc, tm); } else { /* we should have gotten a reply. */ mpr_reinit(sc); } return; } mprsas_log_command(tm, MPR_RECOVERY, "target reset status 0x%x code 0x%x count %u\n", le16toh(reply->IOCStatus), le32toh(reply->ResponseCode), le32toh(reply->TerminationCount)); if (targ->outstanding == 0) { /* we've finished recovery for this target and all * of its logical units. */ mprsas_log_command(tm, MPR_RECOVERY|MPR_INFO, "recovery finished after target reset\n"); mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid, CAM_LUN_WILDCARD); targ->tm = NULL; mprsas_free_tm(sc, tm); } else { /* after a target reset, if this target still has * outstanding commands, the reset effectively failed, * regardless of the status reported. escalate. */ mprsas_log_command(tm, MPR_RECOVERY, "target reset complete for tm %p, but still have %u " "command(s)\n", tm, targ->outstanding); mpr_reinit(sc); } } #define MPR_RESET_TIMEOUT 30 int mprsas_send_reset(struct mpr_softc *sc, struct mpr_command *tm, uint8_t type) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mprsas_target *target; int err; target = tm->cm_targ; if (target->handle == 0) { mpr_dprint(sc, MPR_ERROR, "%s null devhandle for target_id " "%d\n", __func__, target->tid); return -1; } req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; req->DevHandle = htole16(target->handle); req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; req->TaskType = type; if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) { /* XXX Need to handle invalid LUNs */ MPR_SET_LUN(req->LUN, tm->cm_lun); tm->cm_targ->logical_unit_resets++; mprsas_log_command(tm, MPR_RECOVERY|MPR_INFO, "sending logical unit reset\n"); tm->cm_complete = mprsas_logical_unit_reset_complete; mprsas_prepare_for_tm(sc, tm, target, tm->cm_lun); } else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) { /* * Target reset method = * SAS Hard Link Reset / SATA Link Reset */ req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET; tm->cm_targ->target_resets++; mprsas_log_command(tm, MPR_RECOVERY|MPR_INFO, "sending target reset\n"); tm->cm_complete = mprsas_target_reset_complete; mprsas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD); } else { mpr_dprint(sc, MPR_ERROR, "unexpected reset type 0x%x\n", type); return -1; } mpr_dprint(sc, MPR_INFO, "to target %u handle 0x%04x\n", target->tid, target->handle); if (target->encl_level_valid) { mpr_dprint(sc, MPR_INFO, "At enclosure level %d, slot %d, " "connector name (%4s)\n", target->encl_level, target->encl_slot, target->connector_name); } tm->cm_data = NULL; tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; tm->cm_complete_data = (void *)tm; callout_reset(&tm->cm_callout, MPR_RESET_TIMEOUT * hz, mprsas_tm_timeout, tm); err = mpr_map_command(sc, tm); if (err) mprsas_log_command(tm, MPR_RECOVERY, "error %d sending reset type %u\n", err, type); return err; } static void mprsas_abort_complete(struct mpr_softc *sc, struct mpr_command *tm) { struct mpr_command *cm; MPI2_SCSI_TASK_MANAGE_REPLY *reply; MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mprsas_target *targ; callout_stop(&tm->cm_callout); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; targ = tm->cm_targ; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mprsas_log_command(tm, MPR_RECOVERY, "cm_flags = %#x for abort %p TaskMID %u!\n", tm->cm_flags, tm, le16toh(req->TaskMID)); mprsas_free_tm(sc, tm); return; } if (reply == NULL) { mprsas_log_command(tm, MPR_RECOVERY, "NULL abort reply for tm %p TaskMID %u\n", tm, le16toh(req->TaskMID)); if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) { /* this completion was due to a reset, just cleanup */ targ->tm = NULL; mprsas_free_tm(sc, tm); } else { /* we should have gotten a reply. */ mpr_reinit(sc); } return; } mprsas_log_command(tm, MPR_RECOVERY, "abort TaskMID %u status 0x%x code 0x%x count %u\n", le16toh(req->TaskMID), le16toh(reply->IOCStatus), le32toh(reply->ResponseCode), le32toh(reply->TerminationCount)); cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands); if (cm == NULL) { /* if there are no more timedout commands, we're done with * error recovery for this target. */ mprsas_log_command(tm, MPR_RECOVERY, "finished recovery after aborting TaskMID %u\n", le16toh(req->TaskMID)); targ->tm = NULL; mprsas_free_tm(sc, tm); } else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) { /* abort success, but we have more timedout commands to abort */ mprsas_log_command(tm, MPR_RECOVERY, "continuing recovery after aborting TaskMID %u\n", le16toh(req->TaskMID)); mprsas_send_abort(sc, tm, cm); } else { /* we didn't get a command completion, so the abort * failed as far as we're concerned. escalate. */ mprsas_log_command(tm, MPR_RECOVERY, "abort failed for TaskMID %u tm %p\n", le16toh(req->TaskMID), tm); mprsas_send_reset(sc, tm, MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET); } } #define MPR_ABORT_TIMEOUT 5 static int mprsas_send_abort(struct mpr_softc *sc, struct mpr_command *tm, struct mpr_command *cm) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mprsas_target *targ; int err; targ = cm->cm_targ; if (targ->handle == 0) { mpr_dprint(sc, MPR_ERROR,"%s null devhandle for target_id %d\n", __func__, cm->cm_ccb->ccb_h.target_id); return -1; } mprsas_log_command(cm, MPR_RECOVERY|MPR_INFO, "Aborting command %p\n", cm); req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; req->DevHandle = htole16(targ->handle); req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK; /* XXX Need to handle invalid LUNs */ MPR_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun); req->TaskMID = htole16(cm->cm_desc.Default.SMID); tm->cm_data = NULL; tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; tm->cm_complete = mprsas_abort_complete; tm->cm_complete_data = (void *)tm; tm->cm_targ = cm->cm_targ; tm->cm_lun = cm->cm_lun; callout_reset(&tm->cm_callout, MPR_ABORT_TIMEOUT * hz, mprsas_tm_timeout, tm); targ->aborts++; mpr_dprint(sc, MPR_INFO, "Sending reset from %s for target ID %d\n", __func__, targ->tid); mprsas_prepare_for_tm(sc, tm, targ, tm->cm_lun); err = mpr_map_command(sc, tm); if (err) mpr_dprint(sc, MPR_RECOVERY, "error %d sending abort for cm %p SMID %u\n", err, cm, req->TaskMID); return err; } static void mprsas_scsiio_timeout(void *data) { struct mpr_softc *sc; struct mpr_command *cm; struct mprsas_target *targ; cm = (struct mpr_command *)data; sc = cm->cm_sc; MPR_FUNCTRACE(sc); mtx_assert(&sc->mpr_mtx, MA_OWNED); mpr_dprint(sc, MPR_XINFO, "Timeout checking cm %p\n", cm); /* * Run the interrupt handler to make sure it's not pending. This * isn't perfect because the command could have already completed * and been re-used, though this is unlikely. */ mpr_intr_locked(sc); if (cm->cm_state == MPR_CM_STATE_FREE) { mprsas_log_command(cm, MPR_XINFO, "SCSI command %p almost timed out\n", cm); return; } if (cm->cm_ccb == NULL) { mpr_dprint(sc, MPR_ERROR, "command timeout with NULL ccb\n"); return; } targ = cm->cm_targ; targ->timeouts++; mprsas_log_command(cm, MPR_ERROR, "command timeout %d cm %p target " "%u, handle(0x%04x)\n", cm->cm_ccb->ccb_h.timeout, cm, targ->tid, targ->handle); if (targ->encl_level_valid) { mpr_dprint(sc, MPR_ERROR, "At enclosure level %d, slot %d, " "connector name (%4s)\n", targ->encl_level, targ->encl_slot, targ->connector_name); } /* XXX first, check the firmware state, to see if it's still * operational. if not, do a diag reset. */ mprsas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT); cm->cm_state = MPR_CM_STATE_TIMEDOUT; TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery); if (targ->tm != NULL) { /* target already in recovery, just queue up another * timedout command to be processed later. */ mpr_dprint(sc, MPR_RECOVERY, "queued timedout cm %p for " "processing by tm %p\n", cm, targ->tm); } else if ((targ->tm = mprsas_alloc_tm(sc)) != NULL) { mpr_dprint(sc, MPR_RECOVERY, "timedout cm %p allocated tm %p\n", cm, targ->tm); /* start recovery by aborting the first timedout command */ mprsas_send_abort(sc, targ->tm, cm); } else { /* XXX queue this target up for recovery once a TM becomes * available. The firmware only has a limited number of * HighPriority credits for the high priority requests used * for task management, and we ran out. * * Isilon: don't worry about this for now, since we have * more credits than disks in an enclosure, and limit * ourselves to one TM per target for recovery. */ mpr_dprint(sc, MPR_RECOVERY, "timedout cm %p failed to " "allocate a tm\n", cm); } } /** * mprsas_build_nvme_unmap - Build Native NVMe DSM command equivalent * to SCSI Unmap. * Return 0 - for success, * 1 - to immediately return back the command with success status to CAM * negative value - to fallback to firmware path i.e. issue scsi unmap * to FW without any translation. */ static int mprsas_build_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm, union ccb *ccb, struct mprsas_target *targ) { Mpi26NVMeEncapsulatedRequest_t *req = NULL; struct ccb_scsiio *csio; struct unmap_parm_list *plist; struct nvme_dsm_range *nvme_dsm_ranges = NULL; struct nvme_command *c; int i, res; uint16_t ndesc, list_len, data_length; struct mpr_prp_page *prp_page_info; uint64_t nvme_dsm_ranges_dma_handle; csio = &ccb->csio; #if __FreeBSD_version >= 1100103 list_len = (scsiio_cdb_ptr(csio)[7] << 8 | scsiio_cdb_ptr(csio)[8]); #else if (csio->ccb_h.flags & CAM_CDB_POINTER) { list_len = (ccb->csio.cdb_io.cdb_ptr[7] << 8 | ccb->csio.cdb_io.cdb_ptr[8]); } else { list_len = (ccb->csio.cdb_io.cdb_bytes[7] << 8 | ccb->csio.cdb_io.cdb_bytes[8]); } #endif if (!list_len) { mpr_dprint(sc, MPR_ERROR, "Parameter list length is Zero\n"); return -EINVAL; } plist = malloc(csio->dxfer_len, M_MPR, M_ZERO|M_NOWAIT); if (!plist) { mpr_dprint(sc, MPR_ERROR, "Unable to allocate memory to " "save UNMAP data\n"); return -ENOMEM; } /* Copy SCSI unmap data to a local buffer */ bcopy(csio->data_ptr, plist, csio->dxfer_len); /* return back the unmap command to CAM with success status, * if number of descripts is zero. */ ndesc = be16toh(plist->unmap_blk_desc_data_len) >> 4; if (!ndesc) { mpr_dprint(sc, MPR_XINFO, "Number of descriptors in " "UNMAP cmd is Zero\n"); res = 1; goto out; } data_length = ndesc * sizeof(struct nvme_dsm_range); if (data_length > targ->MDTS) { mpr_dprint(sc, MPR_ERROR, "data length: %d is greater than " "Device's MDTS: %d\n", data_length, targ->MDTS); res = -EINVAL; goto out; } prp_page_info = mpr_alloc_prp_page(sc); KASSERT(prp_page_info != NULL, ("%s: There is no PRP Page for " "UNMAP command.\n", __func__)); /* * Insert the allocated PRP page into the command's PRP page list. This * will be freed when the command is freed. */ TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link); nvme_dsm_ranges = (struct nvme_dsm_range *)prp_page_info->prp_page; nvme_dsm_ranges_dma_handle = prp_page_info->prp_page_busaddr; bzero(nvme_dsm_ranges, data_length); /* Convert SCSI unmap's descriptor data to NVMe DSM specific Range data * for each descriptors contained in SCSI UNMAP data. */ for (i = 0; i < ndesc; i++) { nvme_dsm_ranges[i].length = htole32(be32toh(plist->desc[i].nlb)); nvme_dsm_ranges[i].starting_lba = htole64(be64toh(plist->desc[i].slba)); nvme_dsm_ranges[i].attributes = 0; } /* Build MPI2.6's NVMe Encapsulated Request Message */ req = (Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req; bzero(req, sizeof(*req)); req->DevHandle = htole16(targ->handle); req->Function = MPI2_FUNCTION_NVME_ENCAPSULATED; req->Flags = MPI26_NVME_FLAGS_WRITE; req->ErrorResponseBaseAddress.High = htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32)); req->ErrorResponseBaseAddress.Low = htole32(cm->cm_sense_busaddr); req->ErrorResponseAllocationLength = htole16(sizeof(struct nvme_completion)); req->EncapsulatedCommandLength = htole16(sizeof(struct nvme_command)); req->DataLength = htole32(data_length); /* Build NVMe DSM command */ c = (struct nvme_command *) req->NVMe_Command; c->opc = NVME_OPC_DATASET_MANAGEMENT; c->nsid = htole32(csio->ccb_h.target_lun + 1); c->cdw10 = htole32(ndesc - 1); c->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE); cm->cm_length = data_length; cm->cm_data = NULL; cm->cm_complete = mprsas_scsiio_complete; cm->cm_complete_data = ccb; cm->cm_targ = targ; cm->cm_lun = csio->ccb_h.target_lun; cm->cm_ccb = ccb; cm->cm_desc.Default.RequestFlags = MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED; #if __FreeBSD_version >= 1000029 callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0, mprsas_scsiio_timeout, cm, 0); #else //__FreeBSD_version < 1000029 callout_reset(&cm->cm_callout, (ccb->ccb_h.timeout * hz) / 1000, mprsas_scsiio_timeout, cm); #endif //__FreeBSD_version >= 1000029 targ->issued++; targ->outstanding++; TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link); ccb->ccb_h.status |= CAM_SIM_QUEUED; mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n", __func__, cm, ccb, targ->outstanding); mpr_build_nvme_prp(sc, cm, req, (void *)(uintptr_t)nvme_dsm_ranges_dma_handle, 0, data_length); mpr_map_command(sc, cm); out: free(plist, M_MPR); return 0; } static void mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb) { MPI2_SCSI_IO_REQUEST *req; struct ccb_scsiio *csio; struct mpr_softc *sc; struct mprsas_target *targ; struct mprsas_lun *lun; struct mpr_command *cm; uint8_t i, lba_byte, *ref_tag_addr, scsi_opcode; uint16_t eedp_flags; uint32_t mpi_control; int rc; sc = sassc->sc; MPR_FUNCTRACE(sc); mtx_assert(&sc->mpr_mtx, MA_OWNED); csio = &ccb->csio; KASSERT(csio->ccb_h.target_id < sassc->maxtargets, ("Target %d out of bounds in XPT_SCSI_IO\n", csio->ccb_h.target_id)); targ = &sassc->targets[csio->ccb_h.target_id]; mpr_dprint(sc, MPR_TRACE, "ccb %p target flag %x\n", ccb, targ->flags); if (targ->handle == 0x0) { mpr_dprint(sc, MPR_ERROR, "%s NULL handle for target %u\n", __func__, csio->ccb_h.target_id); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (targ->flags & MPR_TARGET_FLAGS_RAID_COMPONENT) { mpr_dprint(sc, MPR_ERROR, "%s Raid component no SCSI IO " "supported %u\n", __func__, csio->ccb_h.target_id); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } /* * Sometimes, it is possible to get a command that is not "In * Progress" and was actually aborted by the upper layer. Check for * this here and complete the command without error. */ if (mprsas_get_ccbstatus(ccb) != CAM_REQ_INPROG) { mpr_dprint(sc, MPR_TRACE, "%s Command is not in progress for " "target %u\n", __func__, csio->ccb_h.target_id); xpt_done(ccb); return; } /* * If devinfo is 0 this will be a volume. In that case don't tell CAM * that the volume has timed out. We want volumes to be enumerated * until they are deleted/removed, not just failed. */ if (targ->flags & MPRSAS_TARGET_INREMOVAL) { if (targ->devinfo == 0) mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); else mprsas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); xpt_done(ccb); return; } if ((sc->mpr_flags & MPR_FLAGS_SHUTDOWN) != 0) { mpr_dprint(sc, MPR_INFO, "%s shutting down\n", __func__); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } /* * If target has a reset in progress, freeze the devq and return. The * devq will be released when the TM reset is finished. */ if (targ->flags & MPRSAS_TARGET_INRESET) { ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN; mpr_dprint(sc, MPR_INFO, "%s: Freezing devq for target ID %d\n", __func__, targ->tid); xpt_freeze_devq(ccb->ccb_h.path, 1); xpt_done(ccb); return; } cm = mpr_alloc_command(sc); if (cm == NULL || (sc->mpr_flags & MPR_FLAGS_DIAGRESET)) { if (cm != NULL) { mpr_free_command(sc, cm); } if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) { xpt_freeze_simq(sassc->sim, 1); sassc->flags |= MPRSAS_QUEUE_FROZEN; } ccb->ccb_h.status &= ~CAM_SIM_QUEUED; ccb->ccb_h.status |= CAM_REQUEUE_REQ; xpt_done(ccb); return; } /* For NVME device's issue UNMAP command directly to NVME drives by * constructing equivalent native NVMe DataSetManagement command. */ #if __FreeBSD_version >= 1100103 scsi_opcode = scsiio_cdb_ptr(csio)[0]; #else if (csio->ccb_h.flags & CAM_CDB_POINTER) scsi_opcode = csio->cdb_io.cdb_ptr[0]; else scsi_opcode = csio->cdb_io.cdb_bytes[0]; #endif if (scsi_opcode == UNMAP && targ->is_nvme && (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) { rc = mprsas_build_nvme_unmap(sc, cm, ccb, targ); if (rc == 1) { /* return command to CAM with success status */ mpr_free_command(sc, cm); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); xpt_done(ccb); return; } else if (!rc) /* Issued NVMe Encapsulated Request Message */ return; } req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req; bzero(req, sizeof(*req)); req->DevHandle = htole16(targ->handle); req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST; req->MsgFlags = 0; req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr); req->SenseBufferLength = MPR_SENSE_LEN; req->SGLFlags = 0; req->ChainOffset = 0; req->SGLOffset0 = 24; /* 32bit word offset to the SGL */ req->SGLOffset1= 0; req->SGLOffset2= 0; req->SGLOffset3= 0; req->SkipCount = 0; req->DataLength = htole32(csio->dxfer_len); req->BidirectionalDataLength = 0; req->IoFlags = htole16(csio->cdb_len); req->EEDPFlags = 0; /* Note: BiDirectional transfers are not supported */ switch (csio->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: mpi_control = MPI2_SCSIIO_CONTROL_READ; cm->cm_flags |= MPR_CM_FLAGS_DATAIN; break; case CAM_DIR_OUT: mpi_control = MPI2_SCSIIO_CONTROL_WRITE; cm->cm_flags |= MPR_CM_FLAGS_DATAOUT; break; case CAM_DIR_NONE: default: mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER; break; } if (csio->cdb_len == 32) mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT; /* * It looks like the hardware doesn't require an explicit tag * number for each transaction. SAM Task Management not supported * at the moment. */ switch (csio->tag_action) { case MSG_HEAD_OF_Q_TAG: mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ; break; case MSG_ORDERED_Q_TAG: mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ; break; case MSG_ACA_TASK: mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ; break; case CAM_TAG_ACTION_NONE: case MSG_SIMPLE_Q_TAG: default: mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ; break; } mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits; req->Control = htole32(mpi_control); if (MPR_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) { mpr_free_command(sc, cm); mprsas_set_ccbstatus(ccb, CAM_LUN_INVALID); xpt_done(ccb); return; } if (csio->ccb_h.flags & CAM_CDB_POINTER) bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len); else { KASSERT(csio->cdb_len <= IOCDBLEN, ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER " "is not set", csio->cdb_len)); bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len); } req->IoFlags = htole16(csio->cdb_len); /* * Check if EEDP is supported and enabled. If it is then check if the * SCSI opcode could be using EEDP. If so, make sure the LUN exists and * is formatted for EEDP support. If all of this is true, set CDB up * for EEDP transfer. */ eedp_flags = op_code_prot[req->CDB.CDB32[0]]; if (sc->eedp_enabled && eedp_flags) { SLIST_FOREACH(lun, &targ->luns, lun_link) { if (lun->lun_id == csio->ccb_h.target_lun) { break; } } if ((lun != NULL) && (lun->eedp_formatted)) { req->EEDPBlockSize = htole16(lun->eedp_block_size); eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG | MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG | MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD); if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) { eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE; } req->EEDPFlags = htole16(eedp_flags); /* * If CDB less than 32, fill in Primary Ref Tag with * low 4 bytes of LBA. If CDB is 32, tag stuff is * already there. Also, set protection bit. FreeBSD * currently does not support CDBs bigger than 16, but * the code doesn't hurt, and will be here for the * future. */ if (csio->cdb_len != 32) { lba_byte = (csio->cdb_len == 16) ? 6 : 2; ref_tag_addr = (uint8_t *)&req->CDB.EEDP32. PrimaryReferenceTag; for (i = 0; i < 4; i++) { *ref_tag_addr = req->CDB.CDB32[lba_byte + i]; ref_tag_addr++; } req->CDB.EEDP32.PrimaryReferenceTag = htole32(req-> CDB.EEDP32.PrimaryReferenceTag); req->CDB.EEDP32.PrimaryApplicationTagMask = 0xFFFF; req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) | 0x20; } else { eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG; req->EEDPFlags = htole16(eedp_flags); req->CDB.CDB32[10] = (req->CDB.CDB32[10] & 0x1F) | 0x20; } } } cm->cm_length = csio->dxfer_len; if (cm->cm_length != 0) { cm->cm_data = ccb; cm->cm_flags |= MPR_CM_FLAGS_USE_CCB; } else { cm->cm_data = NULL; } cm->cm_sge = &req->SGL; cm->cm_sglsize = (32 - 24) * 4; cm->cm_complete = mprsas_scsiio_complete; cm->cm_complete_data = ccb; cm->cm_targ = targ; cm->cm_lun = csio->ccb_h.target_lun; cm->cm_ccb = ccb; /* * If using FP desc type, need to set a bit in IoFlags (SCSI IO is 0) * and set descriptor type. */ if (targ->scsi_req_desc_type == MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO) { req->IoFlags |= MPI25_SCSIIO_IOFLAGS_FAST_PATH; cm->cm_desc.FastPathSCSIIO.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; if (!sc->atomic_desc_capable) { cm->cm_desc.FastPathSCSIIO.DevHandle = htole16(targ->handle); } } else { cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; if (!sc->atomic_desc_capable) cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle); } #if __FreeBSD_version >= 1000029 callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0, mprsas_scsiio_timeout, cm, 0); #else //__FreeBSD_version < 1000029 callout_reset(&cm->cm_callout, (ccb->ccb_h.timeout * hz) / 1000, mprsas_scsiio_timeout, cm); #endif //__FreeBSD_version >= 1000029 targ->issued++; targ->outstanding++; TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link); ccb->ccb_h.status |= CAM_SIM_QUEUED; mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n", __func__, cm, ccb, targ->outstanding); mpr_map_command(sc, cm); return; } static void mpr_response_code(struct mpr_softc *sc, u8 response_code) { char *desc; switch (response_code) { case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE: desc = "task management request completed"; break; case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME: desc = "invalid frame"; break; case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED: desc = "task management request not supported"; break; case MPI2_SCSITASKMGMT_RSP_TM_FAILED: desc = "task management request failed"; break; case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED: desc = "task management request succeeded"; break; case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN: desc = "invalid lun"; break; case 0xA: desc = "overlapped tag attempted"; break; case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC: desc = "task queued, however not sent to target"; break; default: desc = "unknown"; break; } mpr_dprint(sc, MPR_XINFO, "response_code(0x%01x): %s\n", response_code, desc); } /** * mpr_sc_failed_io_info - translated non-succesfull SCSI_IO request */ static void mpr_sc_failed_io_info(struct mpr_softc *sc, struct ccb_scsiio *csio, Mpi2SCSIIOReply_t *mpi_reply, struct mprsas_target *targ) { u32 response_info; u8 *response_bytes; u16 ioc_status = le16toh(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK; u8 scsi_state = mpi_reply->SCSIState; u8 scsi_status = mpi_reply->SCSIStatus; char *desc_ioc_state = NULL; char *desc_scsi_status = NULL; char *desc_scsi_state = sc->tmp_string; u32 log_info = le32toh(mpi_reply->IOCLogInfo); if (log_info == 0x31170000) return; switch (ioc_status) { case MPI2_IOCSTATUS_SUCCESS: desc_ioc_state = "success"; break; case MPI2_IOCSTATUS_INVALID_FUNCTION: desc_ioc_state = "invalid function"; break; case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR: desc_ioc_state = "scsi recovered error"; break; case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE: desc_ioc_state = "scsi invalid dev handle"; break; case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE: desc_ioc_state = "scsi device not there"; break; case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN: desc_ioc_state = "scsi data overrun"; break; case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN: desc_ioc_state = "scsi data underrun"; break; case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR: desc_ioc_state = "scsi io data error"; break; case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR: desc_ioc_state = "scsi protocol error"; break; case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED: desc_ioc_state = "scsi task terminated"; break; case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH: desc_ioc_state = "scsi residual mismatch"; break; case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED: desc_ioc_state = "scsi task mgmt failed"; break; case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED: desc_ioc_state = "scsi ioc terminated"; break; case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED: desc_ioc_state = "scsi ext terminated"; break; case MPI2_IOCSTATUS_EEDP_GUARD_ERROR: desc_ioc_state = "eedp guard error"; break; case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR: desc_ioc_state = "eedp ref tag error"; break; case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR: desc_ioc_state = "eedp app tag error"; break; case MPI2_IOCSTATUS_INSUFFICIENT_POWER: desc_ioc_state = "insufficient power"; break; default: desc_ioc_state = "unknown"; break; } switch (scsi_status) { case MPI2_SCSI_STATUS_GOOD: desc_scsi_status = "good"; break; case MPI2_SCSI_STATUS_CHECK_CONDITION: desc_scsi_status = "check condition"; break; case MPI2_SCSI_STATUS_CONDITION_MET: desc_scsi_status = "condition met"; break; case MPI2_SCSI_STATUS_BUSY: desc_scsi_status = "busy"; break; case MPI2_SCSI_STATUS_INTERMEDIATE: desc_scsi_status = "intermediate"; break; case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET: desc_scsi_status = "intermediate condmet"; break; case MPI2_SCSI_STATUS_RESERVATION_CONFLICT: desc_scsi_status = "reservation conflict"; break; case MPI2_SCSI_STATUS_COMMAND_TERMINATED: desc_scsi_status = "command terminated"; break; case MPI2_SCSI_STATUS_TASK_SET_FULL: desc_scsi_status = "task set full"; break; case MPI2_SCSI_STATUS_ACA_ACTIVE: desc_scsi_status = "aca active"; break; case MPI2_SCSI_STATUS_TASK_ABORTED: desc_scsi_status = "task aborted"; break; default: desc_scsi_status = "unknown"; break; } desc_scsi_state[0] = '\0'; if (!scsi_state) desc_scsi_state = " "; if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) strcat(desc_scsi_state, "response info "); if (scsi_state & MPI2_SCSI_STATE_TERMINATED) strcat(desc_scsi_state, "state terminated "); if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS) strcat(desc_scsi_state, "no status "); if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED) strcat(desc_scsi_state, "autosense failed "); if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) strcat(desc_scsi_state, "autosense valid "); mpr_dprint(sc, MPR_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n", le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status); if (targ->encl_level_valid) { mpr_dprint(sc, MPR_XINFO, "At enclosure level %d, slot %d, " "connector name (%4s)\n", targ->encl_level, targ->encl_slot, targ->connector_name); } /* We can add more detail about underflow data here * TO-DO * */ mpr_dprint(sc, MPR_XINFO, "\tscsi_status(%s)(0x%02x), " "scsi_state(%s)(0x%02x)\n", desc_scsi_status, scsi_status, desc_scsi_state, scsi_state); if (sc->mpr_debug & MPR_XINFO && scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) { mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : Start :\n"); scsi_sense_print(csio); mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : End :\n"); } if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) { response_info = le32toh(mpi_reply->ResponseInfo); response_bytes = (u8 *)&response_info; mpr_response_code(sc,response_bytes[0]); } } /** mprsas_nvme_trans_status_code * * Convert Native NVMe command error status to * equivalent SCSI error status. * * Returns appropriate scsi_status */ static u8 mprsas_nvme_trans_status_code(struct nvme_status nvme_status, struct mpr_command *cm) { u8 status = MPI2_SCSI_STATUS_GOOD; int skey, asc, ascq; union ccb *ccb = cm->cm_complete_data; int returned_sense_len; status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_NO_SENSE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; switch (nvme_status.sct) { case NVME_SCT_GENERIC: switch (nvme_status.sc) { case NVME_SC_SUCCESS: status = MPI2_SCSI_STATUS_GOOD; skey = SSD_KEY_NO_SENSE; asc = SCSI_ASC_NO_SENSE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_INVALID_OPCODE: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_ILLEGAL_COMMAND; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_INVALID_FIELD: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_INVALID_CDB; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_DATA_TRANSFER_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_NO_SENSE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_ABORTED_POWER_LOSS: status = MPI2_SCSI_STATUS_TASK_ABORTED; skey = SSD_KEY_ABORTED_COMMAND; asc = SCSI_ASC_WARNING; ascq = SCSI_ASCQ_POWER_LOSS_EXPECTED; break; case NVME_SC_INTERNAL_DEVICE_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_HARDWARE_ERROR; asc = SCSI_ASC_INTERNAL_TARGET_FAILURE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_ABORTED_BY_REQUEST: case NVME_SC_ABORTED_SQ_DELETION: case NVME_SC_ABORTED_FAILED_FUSED: case NVME_SC_ABORTED_MISSING_FUSED: status = MPI2_SCSI_STATUS_TASK_ABORTED; skey = SSD_KEY_ABORTED_COMMAND; asc = SCSI_ASC_NO_SENSE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID; ascq = SCSI_ASCQ_INVALID_LUN_ID; break; case NVME_SC_LBA_OUT_OF_RANGE: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_ILLEGAL_BLOCK; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_CAPACITY_EXCEEDED: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_NO_SENSE; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_NAMESPACE_NOT_READY: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_NOT_READY; asc = SCSI_ASC_LUN_NOT_READY; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; } break; case NVME_SCT_COMMAND_SPECIFIC: switch (nvme_status.sc) { case NVME_SC_INVALID_FORMAT: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_FORMAT_COMMAND_FAILED; ascq = SCSI_ASCQ_FORMAT_COMMAND_FAILED; break; case NVME_SC_CONFLICTING_ATTRIBUTES: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_INVALID_CDB; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; } break; case NVME_SCT_MEDIA_ERROR: switch (nvme_status.sc) { case NVME_SC_WRITE_FAULTS: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_UNRECOVERED_READ_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_UNRECOVERED_READ_ERROR; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_GUARD_CHECK_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED; ascq = SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED; break; case NVME_SC_APPLICATION_TAG_CHECK_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED; ascq = SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED; break; case NVME_SC_REFERENCE_TAG_CHECK_ERROR: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MEDIUM_ERROR; asc = SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED; ascq = SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED; break; case NVME_SC_COMPARE_FAILURE: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_MISCOMPARE; asc = SCSI_ASC_MISCOMPARE_DURING_VERIFY; ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE; break; case NVME_SC_ACCESS_DENIED: status = MPI2_SCSI_STATUS_CHECK_CONDITION; skey = SSD_KEY_ILLEGAL_REQUEST; asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID; ascq = SCSI_ASCQ_INVALID_LUN_ID; break; } break; } returned_sense_len = sizeof(struct scsi_sense_data); if (returned_sense_len < ccb->csio.sense_len) ccb->csio.sense_resid = ccb->csio.sense_len - returned_sense_len; else ccb->csio.sense_resid = 0; scsi_set_sense_data(&ccb->csio.sense_data, SSD_TYPE_FIXED, 1, skey, asc, ascq, SSD_ELEM_NONE); ccb->ccb_h.status |= CAM_AUTOSNS_VALID; return status; } /** mprsas_complete_nvme_unmap * * Complete native NVMe command issued using NVMe Encapsulated * Request Message. */ static u8 mprsas_complete_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm) { Mpi26NVMeEncapsulatedErrorReply_t *mpi_reply; struct nvme_completion *nvme_completion = NULL; u8 scsi_status = MPI2_SCSI_STATUS_GOOD; mpi_reply =(Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply; if (le16toh(mpi_reply->ErrorResponseCount)){ nvme_completion = (struct nvme_completion *)cm->cm_sense; scsi_status = mprsas_nvme_trans_status_code( nvme_completion->status, cm); } return scsi_status; } static void mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm) { MPI2_SCSI_IO_REPLY *rep; union ccb *ccb; struct ccb_scsiio *csio; struct mprsas_softc *sassc; struct scsi_vpd_supported_page_list *vpd_list = NULL; u8 *TLR_bits, TLR_on, *scsi_cdb; int dir = 0, i; u16 alloc_len; struct mprsas_target *target; target_id_t target_id; MPR_FUNCTRACE(sc); mpr_dprint(sc, MPR_TRACE, "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm, cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply, cm->cm_targ->outstanding); callout_stop(&cm->cm_callout); mtx_assert(&sc->mpr_mtx, MA_OWNED); sassc = sc->sassc; ccb = cm->cm_complete_data; csio = &ccb->csio; target_id = csio->ccb_h.target_id; rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply; /* * XXX KDM if the chain allocation fails, does it matter if we do * the sync and unload here? It is simpler to do it in every case, * assuming it doesn't cause problems. */ if (cm->cm_data != NULL) { if (cm->cm_flags & MPR_CM_FLAGS_DATAIN) dir = BUS_DMASYNC_POSTREAD; else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT) dir = BUS_DMASYNC_POSTWRITE; bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir); bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap); } cm->cm_targ->completed++; cm->cm_targ->outstanding--; TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link); ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED); if (cm->cm_state == MPR_CM_STATE_TIMEDOUT) { TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery); if (cm->cm_reply != NULL) mprsas_log_command(cm, MPR_RECOVERY, "completed timedout cm %p ccb %p during recovery " "ioc %x scsi %x state %x xfer %u\n", cm, cm->cm_ccb, le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState, le32toh(rep->TransferCount)); else mprsas_log_command(cm, MPR_RECOVERY, "completed timedout cm %p ccb %p during recovery\n", cm, cm->cm_ccb); } else if (cm->cm_targ->tm != NULL) { if (cm->cm_reply != NULL) mprsas_log_command(cm, MPR_RECOVERY, "completed cm %p ccb %p during recovery " "ioc %x scsi %x state %x xfer %u\n", cm, cm->cm_ccb, le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState, le32toh(rep->TransferCount)); else mprsas_log_command(cm, MPR_RECOVERY, "completed cm %p ccb %p during recovery\n", cm, cm->cm_ccb); } else if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) { mprsas_log_command(cm, MPR_RECOVERY, "reset completed cm %p ccb %p\n", cm, cm->cm_ccb); } if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { /* * We ran into an error after we tried to map the command, * so we're getting a callback without queueing the command * to the hardware. So we set the status here, and it will * be retained below. We'll go through the "fast path", * because there can be no reply when we haven't actually * gone out to the hardware. */ mprsas_set_ccbstatus(ccb, CAM_REQUEUE_REQ); /* * Currently the only error included in the mask is * MPR_CM_FLAGS_CHAIN_FAILED, which means we're out of * chain frames. We need to freeze the queue until we get * a command that completed without this error, which will * hopefully have some chain frames attached that we can * use. If we wanted to get smarter about it, we would * only unfreeze the queue in this condition when we're * sure that we're getting some chain frames back. That's * probably unnecessary. */ if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) { xpt_freeze_simq(sassc->sim, 1); sassc->flags |= MPRSAS_QUEUE_FROZEN; - mpr_dprint(sc, MPR_INFO, "Error sending command, " + mpr_dprint(sc, MPR_XINFO, "Error sending command, " "freezing SIM queue\n"); } } /* * Point to the SCSI CDB, which is dependent on the CAM_CDB_POINTER * flag, and use it in a few places in the rest of this function for * convenience. Use the macro if available. */ #if __FreeBSD_version >= 1100103 scsi_cdb = scsiio_cdb_ptr(csio); #else if (csio->ccb_h.flags & CAM_CDB_POINTER) scsi_cdb = csio->cdb_io.cdb_ptr; else scsi_cdb = csio->cdb_io.cdb_bytes; #endif /* * If this is a Start Stop Unit command and it was issued by the driver * during shutdown, decrement the refcount to account for all of the * commands that were sent. All SSU commands should be completed before * shutdown completes, meaning SSU_refcount will be 0 after SSU_started * is TRUE. */ if (sc->SSU_started && (scsi_cdb[0] == START_STOP_UNIT)) { mpr_dprint(sc, MPR_INFO, "Decrementing SSU count.\n"); sc->SSU_refcount--; } /* Take the fast path to completion */ if (cm->cm_reply == NULL) { if (mprsas_get_ccbstatus(ccb) == CAM_REQ_INPROG) { if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) mprsas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET); else { mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); csio->scsi_status = SCSI_STATUS_OK; } if (sassc->flags & MPRSAS_QUEUE_FROZEN) { ccb->ccb_h.status |= CAM_RELEASE_SIMQ; sassc->flags &= ~MPRSAS_QUEUE_FROZEN; mpr_dprint(sc, MPR_XINFO, "Unfreezing SIM queue\n"); } } /* * There are two scenarios where the status won't be * CAM_REQ_CMP. The first is if MPR_CM_FLAGS_ERROR_MASK is * set, the second is in the MPR_FLAGS_DIAGRESET above. */ if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) { /* * Freeze the dev queue so that commands are * executed in the correct order after error * recovery. */ ccb->ccb_h.status |= CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1); } mpr_free_command(sc, cm); xpt_done(ccb); return; } target = &sassc->targets[target_id]; if (scsi_cdb[0] == UNMAP && target->is_nvme && (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) { rep->SCSIStatus = mprsas_complete_nvme_unmap(sc, cm); csio->scsi_status = rep->SCSIStatus; } mprsas_log_command(cm, MPR_XINFO, "ioc %x scsi %x state %x xfer %u\n", le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState, le32toh(rep->TransferCount)); switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) { case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN: csio->resid = cm->cm_length - le32toh(rep->TransferCount); /* FALLTHROUGH */ case MPI2_IOCSTATUS_SUCCESS: case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR: if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) == MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR) mprsas_log_command(cm, MPR_XINFO, "recovered error\n"); /* Completion failed at the transport level. */ if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS | MPI2_SCSI_STATE_TERMINATED)) { mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); break; } /* In a modern packetized environment, an autosense failure * implies that there's not much else that can be done to * recover the command. */ if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) { mprsas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL); break; } /* * CAM doesn't care about SAS Response Info data, but if this is * the state check if TLR should be done. If not, clear the * TLR_bits for the target. */ if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) && ((le32toh(rep->ResponseInfo) & MPI2_SCSI_RI_MASK_REASONCODE) == MPR_SCSI_RI_INVALID_FRAME)) { sc->mapping_table[target_id].TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR; } /* * Intentionally override the normal SCSI status reporting * for these two cases. These are likely to happen in a * multi-initiator environment, and we want to make sure that * CAM retries these commands rather than fail them. */ if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) || (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) { mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED); break; } /* Handle normal status and sense */ csio->scsi_status = rep->SCSIStatus; if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD) mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); else mprsas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR); if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) { int sense_len, returned_sense_len; returned_sense_len = min(le32toh(rep->SenseCount), sizeof(struct scsi_sense_data)); if (returned_sense_len < csio->sense_len) csio->sense_resid = csio->sense_len - returned_sense_len; else csio->sense_resid = 0; sense_len = min(returned_sense_len, csio->sense_len - csio->sense_resid); bzero(&csio->sense_data, sizeof(csio->sense_data)); bcopy(cm->cm_sense, &csio->sense_data, sense_len); ccb->ccb_h.status |= CAM_AUTOSNS_VALID; } /* * Check if this is an INQUIRY command. If it's a VPD inquiry, * and it's page code 0 (Supported Page List), and there is * inquiry data, and this is for a sequential access device, and * the device is an SSP target, and TLR is supported by the * controller, turn the TLR_bits value ON if page 0x90 is * supported. */ if ((scsi_cdb[0] == INQUIRY) && (scsi_cdb[1] & SI_EVPD) && (scsi_cdb[2] == SVPD_SUPPORTED_PAGE_LIST) && ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) && (csio->data_ptr != NULL) && ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) && (sc->control_TLR) && (sc->mapping_table[target_id].device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET)) { vpd_list = (struct scsi_vpd_supported_page_list *) csio->data_ptr; TLR_bits = &sc->mapping_table[target_id].TLR_bits; *TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR; TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON; alloc_len = ((u16)scsi_cdb[3] << 8) + scsi_cdb[4]; alloc_len -= csio->resid; for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) { if (vpd_list->list[i] == 0x90) { *TLR_bits = TLR_on; break; } } } /* * If this is a SATA direct-access end device, mark it so that * a SCSI StartStopUnit command will be sent to it when the * driver is being shutdown. */ if ((scsi_cdb[0] == INQUIRY) && (csio->data_ptr != NULL) && ((csio->data_ptr[0] & 0x1f) == T_DIRECT) && (sc->mapping_table[target_id].device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE) && ((sc->mapping_table[target_id].device_info & MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) == MPI2_SAS_DEVICE_INFO_END_DEVICE)) { target = &sassc->targets[target_id]; target->supports_SSU = TRUE; mpr_dprint(sc, MPR_XINFO, "Target %d supports SSU\n", target_id); } break; case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE: case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE: /* * If devinfo is 0 this will be a volume. In that case don't * tell CAM that the volume is not there. We want volumes to * be enumerated until they are deleted/removed, not just * failed. */ if (cm->cm_targ->devinfo == 0) mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); else mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); break; case MPI2_IOCSTATUS_INVALID_SGL: mpr_print_scsiio_cmd(sc, cm); mprsas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR); break; case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED: /* * This is one of the responses that comes back when an I/O * has been aborted. If it is because of a timeout that we * initiated, just set the status to CAM_CMD_TIMEOUT. * Otherwise set it to CAM_REQ_ABORTED. The effect on the * command is the same (it gets retried, subject to the * retry counter), the only difference is what gets printed * on the console. */ if (cm->cm_state == MPR_CM_STATE_TIMEDOUT) mprsas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT); else mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED); break; case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN: /* resid is ignored for this condition */ csio->resid = 0; mprsas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR); break; case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED: case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED: /* * These can sometimes be transient transport-related * errors, and sometimes persistent drive-related errors. * We used to retry these without decrementing the retry * count by returning CAM_REQUEUE_REQ. Unfortunately, if * we hit a persistent drive problem that returns one of * these error codes, we would retry indefinitely. So, * return CAM_REQ_CMP_ERROR so that we decrement the retry * count and avoid infinite retries. We're taking the * potential risk of flagging false failures in the event * of a topology-related error (e.g. a SAS expander problem * causes a command addressed to a drive to fail), but * avoiding getting into an infinite retry loop. */ mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); mprsas_log_command(cm, MPR_INFO, "terminated ioc %x loginfo %x scsi %x state %x xfer %u\n", le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo), rep->SCSIStatus, rep->SCSIState, le32toh(rep->TransferCount)); break; case MPI2_IOCSTATUS_INVALID_FUNCTION: case MPI2_IOCSTATUS_INTERNAL_ERROR: case MPI2_IOCSTATUS_INVALID_VPID: case MPI2_IOCSTATUS_INVALID_FIELD: case MPI2_IOCSTATUS_INVALID_STATE: case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED: case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR: case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR: case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH: case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED: default: mprsas_log_command(cm, MPR_XINFO, "completed ioc %x loginfo %x scsi %x state %x xfer %u\n", le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo), rep->SCSIStatus, rep->SCSIState, le32toh(rep->TransferCount)); csio->resid = cm->cm_length; if (scsi_cdb[0] == UNMAP && target->is_nvme && (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); else mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); break; } mpr_sc_failed_io_info(sc, csio, rep, cm->cm_targ); if (sassc->flags & MPRSAS_QUEUE_FROZEN) { ccb->ccb_h.status |= CAM_RELEASE_SIMQ; sassc->flags &= ~MPRSAS_QUEUE_FROZEN; mpr_dprint(sc, MPR_XINFO, "Command completed, unfreezing SIM " "queue\n"); } if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) { ccb->ccb_h.status |= CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1); } mpr_free_command(sc, cm); xpt_done(ccb); } #if __FreeBSD_version >= 900026 static void mprsas_smpio_complete(struct mpr_softc *sc, struct mpr_command *cm) { MPI2_SMP_PASSTHROUGH_REPLY *rpl; MPI2_SMP_PASSTHROUGH_REQUEST *req; uint64_t sasaddr; union ccb *ccb; ccb = cm->cm_complete_data; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and SMP * commands require two S/G elements only. That should be handled * in the standard request size. */ if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x on SMP " "request!\n", __func__, cm->cm_flags); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); goto bailout; } rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply; if (rpl == NULL) { mpr_dprint(sc, MPR_ERROR, "%s: NULL cm_reply!\n", __func__); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); goto bailout; } req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req; sasaddr = le32toh(req->SASAddress.Low); sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32; if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS || rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) { mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus %04x SASStatus %02x\n", __func__, le16toh(rpl->IOCStatus), rpl->SASStatus); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); goto bailout; } mpr_dprint(sc, MPR_XINFO, "%s: SMP request to SAS address %#jx " "completed successfully\n", __func__, (uintmax_t)sasaddr); if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED) mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); else mprsas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR); bailout: /* * We sync in both directions because we had DMAs in the S/G list * in both directions. */ bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap); mpr_free_command(sc, cm); xpt_done(ccb); } static void mprsas_send_smpcmd(struct mprsas_softc *sassc, union ccb *ccb, uint64_t sasaddr) { struct mpr_command *cm; uint8_t *request, *response; MPI2_SMP_PASSTHROUGH_REQUEST *req; struct mpr_softc *sc; struct sglist *sg; int error; sc = sassc->sc; sg = NULL; error = 0; #if (__FreeBSD_version >= 1000028) || \ ((__FreeBSD_version >= 902001) && (__FreeBSD_version < 1000000)) switch (ccb->ccb_h.flags & CAM_DATA_MASK) { case CAM_DATA_PADDR: case CAM_DATA_SG_PADDR: /* * XXX We don't yet support physical addresses here. */ mpr_dprint(sc, MPR_ERROR, "%s: physical addresses not " "supported\n", __func__); mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID); xpt_done(ccb); return; case CAM_DATA_SG: /* * The chip does not support more than one buffer for the * request or response. */ if ((ccb->smpio.smp_request_sglist_cnt > 1) || (ccb->smpio.smp_response_sglist_cnt > 1)) { mpr_dprint(sc, MPR_ERROR, "%s: multiple request or " "response buffer segments not supported for SMP\n", __func__); mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID); xpt_done(ccb); return; } /* * The CAM_SCATTER_VALID flag was originally implemented * for the XPT_SCSI_IO CCB, which only has one data pointer. * We have two. So, just take that flag to mean that we * might have S/G lists, and look at the S/G segment count * to figure out whether that is the case for each individual * buffer. */ if (ccb->smpio.smp_request_sglist_cnt != 0) { bus_dma_segment_t *req_sg; req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request; request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr; } else request = ccb->smpio.smp_request; if (ccb->smpio.smp_response_sglist_cnt != 0) { bus_dma_segment_t *rsp_sg; rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response; response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr; } else response = ccb->smpio.smp_response; break; case CAM_DATA_VADDR: request = ccb->smpio.smp_request; response = ccb->smpio.smp_response; break; default: mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID); xpt_done(ccb); return; } #else /* __FreeBSD_version < 1000028 */ /* * XXX We don't yet support physical addresses here. */ if (ccb->ccb_h.flags & (CAM_DATA_PHYS|CAM_SG_LIST_PHYS)) { mpr_dprint(sc, MPR_ERROR, "%s: physical addresses not " "supported\n", __func__); mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID); xpt_done(ccb); return; } /* * If the user wants to send an S/G list, check to make sure they * have single buffers. */ if (ccb->ccb_h.flags & CAM_SCATTER_VALID) { /* * The chip does not support more than one buffer for the * request or response. */ if ((ccb->smpio.smp_request_sglist_cnt > 1) || (ccb->smpio.smp_response_sglist_cnt > 1)) { mpr_dprint(sc, MPR_ERROR, "%s: multiple request or " "response buffer segments not supported for SMP\n", __func__); mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID); xpt_done(ccb); return; } /* * The CAM_SCATTER_VALID flag was originally implemented * for the XPT_SCSI_IO CCB, which only has one data pointer. * We have two. So, just take that flag to mean that we * might have S/G lists, and look at the S/G segment count * to figure out whether that is the case for each individual * buffer. */ if (ccb->smpio.smp_request_sglist_cnt != 0) { bus_dma_segment_t *req_sg; req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request; request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr; } else request = ccb->smpio.smp_request; if (ccb->smpio.smp_response_sglist_cnt != 0) { bus_dma_segment_t *rsp_sg; rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response; response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr; } else response = ccb->smpio.smp_response; } else { request = ccb->smpio.smp_request; response = ccb->smpio.smp_response; } #endif /* __FreeBSD_version < 1000028 */ cm = mpr_alloc_command(sc); if (cm == NULL) { mpr_dprint(sc, MPR_ERROR, "%s: cannot allocate command\n", __func__); mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL); xpt_done(ccb); return; } req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req; bzero(req, sizeof(*req)); req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH; /* Allow the chip to use any route to this SAS address. */ req->PhysicalPort = 0xff; req->RequestDataLength = htole16(ccb->smpio.smp_request_len); req->SGLFlags = MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI; mpr_dprint(sc, MPR_XINFO, "%s: sending SMP request to SAS address " "%#jx\n", __func__, (uintmax_t)sasaddr); mpr_init_sge(cm, req, &req->SGL); /* * Set up a uio to pass into mpr_map_command(). This allows us to * do one map command, and one busdma call in there. */ cm->cm_uio.uio_iov = cm->cm_iovec; cm->cm_uio.uio_iovcnt = 2; cm->cm_uio.uio_segflg = UIO_SYSSPACE; /* * The read/write flag isn't used by busdma, but set it just in * case. This isn't exactly accurate, either, since we're going in * both directions. */ cm->cm_uio.uio_rw = UIO_WRITE; cm->cm_iovec[0].iov_base = request; cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength); cm->cm_iovec[1].iov_base = response; cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len; cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len + cm->cm_iovec[1].iov_len; /* * Trigger a warning message in mpr_data_cb() for the user if we * wind up exceeding two S/G segments. The chip expects one * segment for the request and another for the response. */ cm->cm_max_segs = 2; cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; cm->cm_complete = mprsas_smpio_complete; cm->cm_complete_data = ccb; /* * Tell the mapping code that we're using a uio, and that this is * an SMP passthrough request. There is a little special-case * logic there (in mpr_data_cb()) to handle the bidirectional * transfer. */ cm->cm_flags |= MPR_CM_FLAGS_USE_UIO | MPR_CM_FLAGS_SMP_PASS | MPR_CM_FLAGS_DATAIN | MPR_CM_FLAGS_DATAOUT; /* The chip data format is little endian. */ req->SASAddress.High = htole32(sasaddr >> 32); req->SASAddress.Low = htole32(sasaddr); /* * XXX Note that we don't have a timeout/abort mechanism here. * From the manual, it looks like task management requests only * work for SCSI IO and SATA passthrough requests. We may need to * have a mechanism to retry requests in the event of a chip reset * at least. Hopefully the chip will insure that any errors short * of that are relayed back to the driver. */ error = mpr_map_command(sc, cm); if ((error != 0) && (error != EINPROGRESS)) { mpr_dprint(sc, MPR_ERROR, "%s: error %d returned from " "mpr_map_command()\n", __func__, error); goto bailout_error; } return; bailout_error: mpr_free_command(sc, cm); mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL); xpt_done(ccb); return; } static void mprsas_action_smpio(struct mprsas_softc *sassc, union ccb *ccb) { struct mpr_softc *sc; struct mprsas_target *targ; uint64_t sasaddr = 0; sc = sassc->sc; /* * Make sure the target exists. */ KASSERT(ccb->ccb_h.target_id < sassc->maxtargets, ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id)); targ = &sassc->targets[ccb->ccb_h.target_id]; if (targ->handle == 0x0) { mpr_dprint(sc, MPR_ERROR, "%s: target %d does not exist!\n", __func__, ccb->ccb_h.target_id); mprsas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); xpt_done(ccb); return; } /* * If this device has an embedded SMP target, we'll talk to it * directly. * figure out what the expander's address is. */ if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0) sasaddr = targ->sasaddr; /* * If we don't have a SAS address for the expander yet, try * grabbing it from the page 0x83 information cached in the * transport layer for this target. LSI expanders report the * expander SAS address as the port-associated SAS address in * Inquiry VPD page 0x83. Maxim expanders don't report it in page * 0x83. * * XXX KDM disable this for now, but leave it commented out so that * it is obvious that this is another possible way to get the SAS * address. * * The parent handle method below is a little more reliable, and * the other benefit is that it works for devices other than SES * devices. So you can send a SMP request to a da(4) device and it * will get routed to the expander that device is attached to. * (Assuming the da(4) device doesn't contain an SMP target...) */ #if 0 if (sasaddr == 0) sasaddr = xpt_path_sas_addr(ccb->ccb_h.path); #endif /* * If we still don't have a SAS address for the expander, look for * the parent device of this device, which is probably the expander. */ if (sasaddr == 0) { #ifdef OLD_MPR_PROBE struct mprsas_target *parent_target; #endif if (targ->parent_handle == 0x0) { mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have " "a valid parent handle!\n", __func__, targ->handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } #ifdef OLD_MPR_PROBE parent_target = mprsas_find_target_by_handle(sassc, 0, targ->parent_handle); if (parent_target == NULL) { mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have " "a valid parent target!\n", __func__, targ->handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } if ((parent_target->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) { mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d " "does not have an SMP target!\n", __func__, targ->handle, parent_target->handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } sasaddr = parent_target->sasaddr; #else /* OLD_MPR_PROBE */ if ((targ->parent_devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) { mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d " "does not have an SMP target!\n", __func__, targ->handle, targ->parent_handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } if (targ->parent_sasaddr == 0x0) { mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent handle " "%d does not have a valid SAS address!\n", __func__, targ->handle, targ->parent_handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } sasaddr = targ->parent_sasaddr; #endif /* OLD_MPR_PROBE */ } if (sasaddr == 0) { mpr_dprint(sc, MPR_INFO, "%s: unable to find SAS address for " "handle %d\n", __func__, targ->handle); mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } mprsas_send_smpcmd(sassc, ccb, sasaddr); return; bailout: xpt_done(ccb); } #endif //__FreeBSD_version >= 900026 static void mprsas_action_resetdev(struct mprsas_softc *sassc, union ccb *ccb) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; struct mpr_softc *sc; struct mpr_command *tm; struct mprsas_target *targ; MPR_FUNCTRACE(sassc->sc); mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED); KASSERT(ccb->ccb_h.target_id < sassc->maxtargets, ("Target %d out of " "bounds in XPT_RESET_DEV\n", ccb->ccb_h.target_id)); sc = sassc->sc; tm = mpr_alloc_command(sc); if (tm == NULL) { mpr_dprint(sc, MPR_ERROR, "command alloc failure in " "mprsas_action_resetdev\n"); mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL); xpt_done(ccb); return; } targ = &sassc->targets[ccb->ccb_h.target_id]; req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; req->DevHandle = htole16(targ->handle); req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT; req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET; /* SAS Hard Link Reset / SATA Link Reset */ req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET; tm->cm_data = NULL; tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; tm->cm_complete = mprsas_resetdev_complete; tm->cm_complete_data = ccb; mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n", __func__, targ->tid); tm->cm_targ = targ; targ->flags |= MPRSAS_TARGET_INRESET; mpr_map_command(sc, tm); } static void mprsas_resetdev_complete(struct mpr_softc *sc, struct mpr_command *tm) { MPI2_SCSI_TASK_MANAGE_REPLY *resp; union ccb *ccb; MPR_FUNCTRACE(sc); mtx_assert(&sc->mpr_mtx, MA_OWNED); resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply; ccb = tm->cm_complete_data; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * task management commands don't have S/G lists. */ if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { MPI2_SCSI_TASK_MANAGE_REQUEST *req; req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req; mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for reset of " "handle %#04x! This should not happen!\n", __func__, tm->cm_flags, req->DevHandle); mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); goto bailout; } mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__, le16toh(resp->IOCStatus), le32toh(resp->ResponseCode)); if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) { mprsas_set_ccbstatus(ccb, CAM_REQ_CMP); mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid, CAM_LUN_WILDCARD); } else mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); bailout: mprsas_free_tm(sc, tm); xpt_done(ccb); } static void mprsas_poll(struct cam_sim *sim) { struct mprsas_softc *sassc; sassc = cam_sim_softc(sim); if (sassc->sc->mpr_debug & MPR_TRACE) { /* frequent debug messages during a panic just slow * everything down too much. */ mpr_dprint(sassc->sc, MPR_XINFO, "%s clearing MPR_TRACE\n", __func__); sassc->sc->mpr_debug &= ~MPR_TRACE; } mpr_intr_locked(sassc->sc); } static void mprsas_async(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) { struct mpr_softc *sc; sc = (struct mpr_softc *)callback_arg; switch (code) { #if (__FreeBSD_version >= 1000006) || \ ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000)) case AC_ADVINFO_CHANGED: { struct mprsas_target *target; struct mprsas_softc *sassc; struct scsi_read_capacity_data_long rcap_buf; struct ccb_dev_advinfo cdai; struct mprsas_lun *lun; lun_id_t lunid; int found_lun; uintptr_t buftype; buftype = (uintptr_t)arg; found_lun = 0; sassc = sc->sassc; /* * We're only interested in read capacity data changes. */ if (buftype != CDAI_TYPE_RCAPLONG) break; /* * See the comment in mpr_attach_sas() for a detailed * explanation. In these versions of FreeBSD we register * for all events and filter out the events that don't * apply to us. */ #if (__FreeBSD_version < 1000703) || \ ((__FreeBSD_version >= 1100000) && (__FreeBSD_version < 1100002)) if (xpt_path_path_id(path) != sassc->sim->path_id) break; #endif /* * We should have a handle for this, but check to make sure. */ KASSERT(xpt_path_target_id(path) < sassc->maxtargets, ("Target %d out of bounds in mprsas_async\n", xpt_path_target_id(path))); target = &sassc->targets[xpt_path_target_id(path)]; if (target->handle == 0) break; lunid = xpt_path_lun_id(path); SLIST_FOREACH(lun, &target->luns, lun_link) { if (lun->lun_id == lunid) { found_lun = 1; break; } } if (found_lun == 0) { lun = malloc(sizeof(struct mprsas_lun), M_MPR, M_NOWAIT | M_ZERO); if (lun == NULL) { mpr_dprint(sc, MPR_ERROR, "Unable to alloc " "LUN for EEDP support.\n"); break; } lun->lun_id = lunid; SLIST_INSERT_HEAD(&target->luns, lun, lun_link); } bzero(&rcap_buf, sizeof(rcap_buf)); xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL); cdai.ccb_h.func_code = XPT_DEV_ADVINFO; cdai.ccb_h.flags = CAM_DIR_IN; cdai.buftype = CDAI_TYPE_RCAPLONG; #if (__FreeBSD_version >= 1100061) || \ ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000)) cdai.flags = CDAI_FLAG_NONE; #else cdai.flags = 0; #endif cdai.bufsiz = sizeof(rcap_buf); cdai.buf = (uint8_t *)&rcap_buf; xpt_action((union ccb *)&cdai); if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0) cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE); if ((mprsas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP) && (rcap_buf.prot & SRC16_PROT_EN)) { lun->eedp_formatted = TRUE; lun->eedp_block_size = scsi_4btoul(rcap_buf.length); } else { lun->eedp_formatted = FALSE; lun->eedp_block_size = 0; } break; } #endif case AC_FOUND_DEVICE: { struct ccb_getdev *cgd; /* * See the comment in mpr_attach_sas() for a detailed * explanation. In these versions of FreeBSD we register * for all events and filter out the events that don't * apply to us. */ #if (__FreeBSD_version < 1000703) || \ ((__FreeBSD_version >= 1100000) && (__FreeBSD_version < 1100002)) if (xpt_path_path_id(path) != sc->sassc->sim->path_id) break; #endif cgd = arg; #if (__FreeBSD_version < 901503) || \ ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) mprsas_check_eedp(sc, path, cgd); #endif break; } default: break; } } #if (__FreeBSD_version < 901503) || \ ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) static void mprsas_check_eedp(struct mpr_softc *sc, struct cam_path *path, struct ccb_getdev *cgd) { struct mprsas_softc *sassc = sc->sassc; struct ccb_scsiio *csio; struct scsi_read_capacity_16 *scsi_cmd; struct scsi_read_capacity_eedp *rcap_buf; path_id_t pathid; target_id_t targetid; lun_id_t lunid; union ccb *ccb; struct cam_path *local_path; struct mprsas_target *target; struct mprsas_lun *lun; uint8_t found_lun; char path_str[64]; pathid = cam_sim_path(sassc->sim); targetid = xpt_path_target_id(path); lunid = xpt_path_lun_id(path); KASSERT(targetid < sassc->maxtargets, ("Target %d out of bounds in " "mprsas_check_eedp\n", targetid)); target = &sassc->targets[targetid]; if (target->handle == 0x0) return; /* * Determine if the device is EEDP capable. * * If this flag is set in the inquiry data, the device supports * protection information, and must support the 16 byte read capacity * command, otherwise continue without sending read cap 16. */ if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0) return; /* * Issue a READ CAPACITY 16 command. This info is used to determine if * the LUN is formatted for EEDP support. */ ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { mpr_dprint(sc, MPR_ERROR, "Unable to alloc CCB for EEDP " "support.\n"); return; } if (xpt_create_path(&local_path, xpt_periph, pathid, targetid, lunid) != CAM_REQ_CMP) { mpr_dprint(sc, MPR_ERROR, "Unable to create path for EEDP " "support.\n"); xpt_free_ccb(ccb); return; } /* * If LUN is already in list, don't create a new one. */ found_lun = FALSE; SLIST_FOREACH(lun, &target->luns, lun_link) { if (lun->lun_id == lunid) { found_lun = TRUE; break; } } if (!found_lun) { lun = malloc(sizeof(struct mprsas_lun), M_MPR, M_NOWAIT | M_ZERO); if (lun == NULL) { mpr_dprint(sc, MPR_ERROR, "Unable to alloc LUN for " "EEDP support.\n"); xpt_free_path(local_path); xpt_free_ccb(ccb); return; } lun->lun_id = lunid; SLIST_INSERT_HEAD(&target->luns, lun, lun_link); } xpt_path_string(local_path, path_str, sizeof(path_str)); mpr_dprint(sc, MPR_INFO, "Sending read cap: path %s handle %d\n", path_str, target->handle); /* * Issue a READ CAPACITY 16 command for the LUN. The * mprsas_read_cap_done function will load the read cap info into the * LUN struct. */ rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp), M_MPR, M_NOWAIT | M_ZERO); if (rcap_buf == NULL) { mpr_dprint(sc, MPR_ERROR, "Unable to alloc read capacity " "buffer for EEDP support.\n"); xpt_free_path(ccb->ccb_h.path); xpt_free_ccb(ccb); return; } xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT); csio = &ccb->csio; csio->ccb_h.func_code = XPT_SCSI_IO; csio->ccb_h.flags = CAM_DIR_IN; csio->ccb_h.retry_count = 4; csio->ccb_h.cbfcnp = mprsas_read_cap_done; csio->ccb_h.timeout = 60000; csio->data_ptr = (uint8_t *)rcap_buf; csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp); csio->sense_len = MPR_SENSE_LEN; csio->cdb_len = sizeof(*scsi_cmd); csio->tag_action = MSG_SIMPLE_Q_TAG; scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = 0x9E; scsi_cmd->service_action = SRC16_SERVICE_ACTION; ((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp); ccb->ccb_h.ppriv_ptr1 = sassc; xpt_action(ccb); } static void mprsas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb) { struct mprsas_softc *sassc; struct mprsas_target *target; struct mprsas_lun *lun; struct scsi_read_capacity_eedp *rcap_buf; if (done_ccb == NULL) return; /* Driver need to release devq, it Scsi command is * generated by driver internally. * Currently there is a single place where driver * calls scsi command internally. In future if driver * calls more scsi command internally, it needs to release * devq internally, since those command will not go back to * cam_periph. */ if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) { done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN; xpt_release_devq(done_ccb->ccb_h.path, /*count*/ 1, /*run_queue*/TRUE); } rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr; /* * Get the LUN ID for the path and look it up in the LUN list for the * target. */ sassc = (struct mprsas_softc *)done_ccb->ccb_h.ppriv_ptr1; KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets, ("Target %d out " "of bounds in mprsas_read_cap_done\n", done_ccb->ccb_h.target_id)); target = &sassc->targets[done_ccb->ccb_h.target_id]; SLIST_FOREACH(lun, &target->luns, lun_link) { if (lun->lun_id != done_ccb->ccb_h.target_lun) continue; /* * Got the LUN in the target's LUN list. Fill it in with EEDP * info. If the READ CAP 16 command had some SCSI error (common * if command is not supported), mark the lun as not supporting * EEDP and set the block size to 0. */ if ((mprsas_get_ccbstatus(done_ccb) != CAM_REQ_CMP) || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) { lun->eedp_formatted = FALSE; lun->eedp_block_size = 0; break; } if (rcap_buf->protect & 0x01) { mpr_dprint(sassc->sc, MPR_INFO, "LUN %d for target ID " "%d is formatted for EEDP support.\n", done_ccb->ccb_h.target_lun, done_ccb->ccb_h.target_id); lun->eedp_formatted = TRUE; lun->eedp_block_size = scsi_4btoul(rcap_buf->length); } break; } // Finished with this CCB and path. free(rcap_buf, M_MPR); xpt_free_path(done_ccb->ccb_h.path); xpt_free_ccb(done_ccb); } #endif /* (__FreeBSD_version < 901503) || \ ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */ void mprsas_prepare_for_tm(struct mpr_softc *sc, struct mpr_command *tm, struct mprsas_target *target, lun_id_t lun_id) { union ccb *ccb; path_id_t path_id; /* * Set the INRESET flag for this target so that no I/O will be sent to * the target until the reset has completed. If an I/O request does * happen, the devq will be frozen. The CCB holds the path which is * used to release the devq. The devq is released and the CCB is freed * when the TM completes. */ ccb = xpt_alloc_ccb_nowait(); if (ccb) { path_id = cam_sim_path(sc->sassc->sim); if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id, target->tid, lun_id) != CAM_REQ_CMP) { xpt_free_ccb(ccb); } else { tm->cm_ccb = ccb; tm->cm_targ = target; target->flags |= MPRSAS_TARGET_INRESET; } } } int mprsas_startup(struct mpr_softc *sc) { /* * Send the port enable message and set the wait_for_port_enable flag. * This flag helps to keep the simq frozen until all discovery events * are processed. */ sc->wait_for_port_enable = 1; mprsas_send_portenable(sc); return (0); } static int mprsas_send_portenable(struct mpr_softc *sc) { MPI2_PORT_ENABLE_REQUEST *request; struct mpr_command *cm; MPR_FUNCTRACE(sc); if ((cm = mpr_alloc_command(sc)) == NULL) return (EBUSY); request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req; request->Function = MPI2_FUNCTION_PORT_ENABLE; request->MsgFlags = 0; request->VP_ID = 0; cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; cm->cm_complete = mprsas_portenable_complete; cm->cm_data = NULL; cm->cm_sge = NULL; mpr_map_command(sc, cm); mpr_dprint(sc, MPR_XINFO, "mpr_send_portenable finished cm %p req %p complete %p\n", cm, cm->cm_req, cm->cm_complete); return (0); } static void mprsas_portenable_complete(struct mpr_softc *sc, struct mpr_command *cm) { MPI2_PORT_ENABLE_REPLY *reply; struct mprsas_softc *sassc; MPR_FUNCTRACE(sc); sassc = sc->sassc; /* * Currently there should be no way we can hit this case. It only * happens when we have a failure to allocate chain frames, and * port enable commands don't have S/G lists. */ if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for port enable! " "This should not happen!\n", __func__, cm->cm_flags); } reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply; if (reply == NULL) mpr_dprint(sc, MPR_FAULT, "Portenable NULL reply\n"); else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) mpr_dprint(sc, MPR_FAULT, "Portenable failed\n"); mpr_free_command(sc, cm); if (sc->mpr_ich.ich_arg != NULL) { mpr_dprint(sc, MPR_XINFO, "disestablish config intrhook\n"); config_intrhook_disestablish(&sc->mpr_ich); sc->mpr_ich.ich_arg = NULL; } /* * Done waiting for port enable to complete. Decrement the refcount. * If refcount is 0, discovery is complete and a rescan of the bus can * take place. */ sc->wait_for_port_enable = 0; sc->port_enable_complete = 1; wakeup(&sc->port_enable_complete); mprsas_startup_decrement(sassc); } int mprsas_check_id(struct mprsas_softc *sassc, int id) { struct mpr_softc *sc = sassc->sc; char *ids; char *name; ids = &sc->exclude_ids[0]; while((name = strsep(&ids, ",")) != NULL) { if (name[0] == '\0') continue; if (strtol(name, NULL, 0) == (long)id) return (1); } return (0); } void mprsas_realloc_targets(struct mpr_softc *sc, int maxtargets) { struct mprsas_softc *sassc; struct mprsas_lun *lun, *lun_tmp; struct mprsas_target *targ; int i; sassc = sc->sassc; /* * The number of targets is based on IOC Facts, so free all of * the allocated LUNs for each target and then the target buffer * itself. */ for (i=0; i< maxtargets; i++) { targ = &sassc->targets[i]; SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) { free(lun, M_MPR); } } free(sassc->targets, M_MPR); sassc->targets = malloc(sizeof(struct mprsas_target) * maxtargets, M_MPR, M_WAITOK|M_ZERO); if (!sassc->targets) { panic("%s failed to alloc targets with error %d\n", __func__, ENOMEM); } } Index: projects/runtime-coverage/sys/fs/fdescfs/fdesc.h =================================================================== --- projects/runtime-coverage/sys/fs/fdescfs/fdesc.h (revision 321584) +++ projects/runtime-coverage/sys/fs/fdescfs/fdesc.h (revision 321585) @@ -1,69 +1,71 @@ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software donated to Berkeley by * Jan-Simon Pendry. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)fdesc.h 8.5 (Berkeley) 1/21/94 * * $FreeBSD$ */ -#ifdef _KERNEL +#ifndef _FS_FDESC_H_ +#define _FS_FDESC_H_ + /* Private mount flags for fdescfs. */ #define FMNT_UNMOUNTF 0x01 struct fdescmount { struct vnode *f_root; /* Root node */ int flags; }; #define FD_ROOT 1 #define FD_DESC 3 typedef enum { Froot, Fdesc } fdntype; struct fdescnode { LIST_ENTRY(fdescnode) fd_hash; /* Hash list */ struct vnode *fd_vnode; /* Back ptr to vnode */ fdntype fd_type; /* Type of this node */ unsigned fd_fd; /* Fd to be dup'ed */ int fd_ix; /* filesystem index */ }; extern struct mtx fdesc_hashmtx; #define VFSTOFDESC(mp) ((struct fdescmount *)((mp)->mnt_data)) #define VTOFDESC(vp) ((struct fdescnode *)(vp)->v_data) extern vfs_init_t fdesc_init; extern vfs_uninit_t fdesc_uninit; extern int fdesc_allocvp(fdntype, unsigned, int, struct mount *, struct vnode **); -#endif /* _KERNEL */ +#endif /* !_FS_FDESC_H_ */ Index: projects/runtime-coverage/sys/fs/nfsclient/nfs_clbio.c =================================================================== --- projects/runtime-coverage/sys/fs/nfsclient/nfs_clbio.c (revision 321584) +++ projects/runtime-coverage/sys/fs/nfsclient/nfs_clbio.c (revision 321585) @@ -1,1896 +1,1897 @@ /*- * Copyright (c) 1989, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Rick Macklem at The University of Guelph. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)nfs_bio.c 8.9 (Berkeley) 3/30/95 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern int newnfs_directio_allow_mmap; extern struct nfsstatsv1 nfsstatsv1; extern struct mtx ncl_iod_mutex; extern int ncl_numasync; extern enum nfsiod_state ncl_iodwant[NFS_MAXASYNCDAEMON]; extern struct nfsmount *ncl_iodmount[NFS_MAXASYNCDAEMON]; extern int newnfs_directio_enable; extern int nfs_keep_dirty_on_error; int ncl_pbuf_freecnt = -1; /* start out unlimited */ static struct buf *nfs_getcacheblk(struct vnode *vp, daddr_t bn, int size, struct thread *td); static int nfs_directio_write(struct vnode *vp, struct uio *uiop, struct ucred *cred, int ioflag); /* * Vnode op for VM getpages. */ SYSCTL_DECL(_vfs_nfs); static int use_buf_pager = 1; SYSCTL_INT(_vfs_nfs, OID_AUTO, use_buf_pager, CTLFLAG_RWTUN, &use_buf_pager, 0, "Use buffer pager instead of direct readrpc call"); static daddr_t ncl_gbp_getblkno(struct vnode *vp, vm_ooffset_t off) { return (off / vp->v_bufobj.bo_bsize); } static int ncl_gbp_getblksz(struct vnode *vp, daddr_t lbn) { struct nfsnode *np; u_quad_t nsize; int biosize, bcount; np = VTONFS(vp); mtx_lock(&np->n_mtx); nsize = np->n_size; mtx_unlock(&np->n_mtx); biosize = vp->v_bufobj.bo_bsize; bcount = biosize; if ((off_t)lbn * biosize >= nsize) bcount = 0; else if ((off_t)(lbn + 1) * biosize > nsize) bcount = nsize - (off_t)lbn * biosize; return (bcount); } int ncl_getpages(struct vop_getpages_args *ap) { int i, error, nextoff, size, toff, count, npages; struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; struct vnode *vp; struct thread *td; struct ucred *cred; struct nfsmount *nmp; vm_object_t object; vm_page_t *pages; struct nfsnode *np; vp = ap->a_vp; np = VTONFS(vp); td = curthread; cred = curthread->td_ucred; nmp = VFSTONFS(vp->v_mount); pages = ap->a_m; npages = ap->a_count; if ((object = vp->v_object) == NULL) { printf("ncl_getpages: called with non-merged cache vnode\n"); return (VM_PAGER_ERROR); } if (newnfs_directio_enable && !newnfs_directio_allow_mmap) { mtx_lock(&np->n_mtx); if ((np->n_flag & NNONCACHE) && (vp->v_type == VREG)) { mtx_unlock(&np->n_mtx); printf("ncl_getpages: called on non-cacheable vnode\n"); return (VM_PAGER_ERROR); } else mtx_unlock(&np->n_mtx); } mtx_lock(&nmp->nm_mtx); if ((nmp->nm_flag & NFSMNT_NFSV3) != 0 && (nmp->nm_state & NFSSTA_GOTFSINFO) == 0) { mtx_unlock(&nmp->nm_mtx); /* We'll never get here for v4, because we always have fsinfo */ (void)ncl_fsinfo(nmp, vp, cred, td); } else mtx_unlock(&nmp->nm_mtx); if (use_buf_pager) return (vfs_bio_getpages(vp, pages, npages, ap->a_rbehind, ap->a_rahead, ncl_gbp_getblkno, ncl_gbp_getblksz)); /* * If the requested page is partially valid, just return it and * allow the pager to zero-out the blanks. Partially valid pages * can only occur at the file EOF. * * XXXGL: is that true for NFS, where short read can occur??? */ VM_OBJECT_WLOCK(object); if (pages[npages - 1]->valid != 0 && --npages == 0) goto out; VM_OBJECT_WUNLOCK(object); /* * We use only the kva address for the buffer, but this is extremely * convenient and fast. */ bp = getpbuf(&ncl_pbuf_freecnt); kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); VM_CNT_INC(v_vnodein); VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = IDX_TO_OFF(pages[0]->pindex); uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; uio.uio_td = td; error = ncl_readrpc(vp, &uio, cred); pmap_qremove(kva, npages); relpbuf(bp, &ncl_pbuf_freecnt); if (error && (uio.uio_resid == count)) { printf("ncl_getpages: error %d\n", error); return (VM_PAGER_ERROR); } /* * Calculate the number of bytes read and validate only that number * of bytes. Note that due to pending writes, size may be 0. This * does not mean that the remaining data is invalid! */ size = count - uio.uio_resid; VM_OBJECT_WLOCK(object); for (i = 0, toff = 0; i < npages; i++, toff = nextoff) { vm_page_t m; nextoff = toff + PAGE_SIZE; m = pages[i]; if (nextoff <= size) { /* * Read operation filled an entire page */ m->valid = VM_PAGE_BITS_ALL; KASSERT(m->dirty == 0, ("nfs_getpages: page %p is dirty", m)); } else if (size > toff) { /* * Read operation filled a partial page. */ m->valid = 0; vm_page_set_valid_range(m, 0, size - toff); KASSERT(m->dirty == 0, ("nfs_getpages: page %p is dirty", m)); } else { /* * Read operation was short. If no error * occurred we may have hit a zero-fill * section. We leave valid set to 0, and page * is freed by vm_page_readahead_finish() if * its index is not equal to requested, or * page is zeroed and set valid by * vm_pager_get_pages() for requested page. */ ; } } out: VM_OBJECT_WUNLOCK(object); if (ap->a_rbehind) *ap->a_rbehind = 0; if (ap->a_rahead) *ap->a_rahead = 0; return (VM_PAGER_OK); } /* * Vnode op for VM putpages. */ int ncl_putpages(struct vop_putpages_args *ap) { struct uio uio; struct iovec iov; int i, error, npages, count; off_t offset; int *rtvals; struct vnode *vp; struct thread *td; struct ucred *cred; struct nfsmount *nmp; struct nfsnode *np; vm_page_t *pages; vp = ap->a_vp; np = VTONFS(vp); td = curthread; /* XXX */ /* Set the cred to n_writecred for the write rpcs. */ if (np->n_writecred != NULL) cred = crhold(np->n_writecred); else cred = crhold(curthread->td_ucred); /* XXX */ nmp = VFSTONFS(vp->v_mount); pages = ap->a_m; count = ap->a_count; rtvals = ap->a_rtvals; npages = btoc(count); offset = IDX_TO_OFF(pages[0]->pindex); mtx_lock(&nmp->nm_mtx); if ((nmp->nm_flag & NFSMNT_NFSV3) != 0 && (nmp->nm_state & NFSSTA_GOTFSINFO) == 0) { mtx_unlock(&nmp->nm_mtx); (void)ncl_fsinfo(nmp, vp, cred, td); } else mtx_unlock(&nmp->nm_mtx); mtx_lock(&np->n_mtx); if (newnfs_directio_enable && !newnfs_directio_allow_mmap && (np->n_flag & NNONCACHE) && (vp->v_type == VREG)) { mtx_unlock(&np->n_mtx); printf("ncl_putpages: called on noncache-able vnode\n"); mtx_lock(&np->n_mtx); } - - for (i = 0; i < npages; i++) - rtvals[i] = VM_PAGER_ERROR; - /* * When putting pages, do not extend file past EOF. */ if (offset + count > np->n_size) { count = np->n_size - offset; if (count < 0) count = 0; } mtx_unlock(&np->n_mtx); + for (i = 0; i < npages; i++) + rtvals[i] = VM_PAGER_ERROR; + VM_CNT_INC(v_vnodeout); VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = unmapped_buf; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = offset; uio.uio_resid = count; uio.uio_segflg = UIO_NOCOPY; uio.uio_rw = UIO_WRITE; uio.uio_td = td; error = VOP_WRITE(vp, &uio, vnode_pager_putpages_ioflags(ap->a_sync), cred); crfree(cred); - if (error == 0 || !nfs_keep_dirty_on_error) - vnode_pager_undirty_pages(pages, rtvals, count - uio.uio_resid); + if (error == 0 || !nfs_keep_dirty_on_error) { + vnode_pager_undirty_pages(pages, rtvals, count - uio.uio_resid, + np->n_size - offset, npages * PAGE_SIZE); + } return (rtvals[0]); } /* * For nfs, cache consistency can only be maintained approximately. * Although RFC1094 does not specify the criteria, the following is * believed to be compatible with the reference port. * For nfs: * If the file's modify time on the server has changed since the * last read rpc or you have written to the file, * you may have lost data cache consistency with the * server, so flush all of the file's data out of the cache. * Then force a getattr rpc to ensure that you have up to date * attributes. * NB: This implies that cache data can be read when up to * NFS_ATTRTIMEO seconds out of date. If you find that you need current * attributes this could be forced by setting n_attrstamp to 0 before * the VOP_GETATTR() call. */ static inline int nfs_bioread_check_cons(struct vnode *vp, struct thread *td, struct ucred *cred) { int error = 0; struct vattr vattr; struct nfsnode *np = VTONFS(vp); int old_lock; /* * Grab the exclusive lock before checking whether the cache is * consistent. * XXX - We can make this cheaper later (by acquiring cheaper locks). * But for now, this suffices. */ old_lock = ncl_upgrade_vnlock(vp); if (vp->v_iflag & VI_DOOMED) { error = EBADF; goto out; } mtx_lock(&np->n_mtx); if (np->n_flag & NMODIFIED) { mtx_unlock(&np->n_mtx); if (vp->v_type != VREG) { if (vp->v_type != VDIR) panic("nfs: bioread, not dir"); ncl_invaldir(vp); error = ncl_vinvalbuf(vp, V_SAVE, td, 1); if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) error = EBADF; if (error != 0) goto out; } np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); error = VOP_GETATTR(vp, &vattr, cred); if (error) goto out; mtx_lock(&np->n_mtx); np->n_mtime = vattr.va_mtime; mtx_unlock(&np->n_mtx); } else { mtx_unlock(&np->n_mtx); error = VOP_GETATTR(vp, &vattr, cred); if (error) return (error); mtx_lock(&np->n_mtx); if ((np->n_flag & NSIZECHANGED) || (NFS_TIMESPEC_COMPARE(&np->n_mtime, &vattr.va_mtime))) { mtx_unlock(&np->n_mtx); if (vp->v_type == VDIR) ncl_invaldir(vp); error = ncl_vinvalbuf(vp, V_SAVE, td, 1); if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) error = EBADF; if (error != 0) goto out; mtx_lock(&np->n_mtx); np->n_mtime = vattr.va_mtime; np->n_flag &= ~NSIZECHANGED; } mtx_unlock(&np->n_mtx); } out: ncl_downgrade_vnlock(vp, old_lock); return (error); } /* * Vnode op for read using bio */ int ncl_bioread(struct vnode *vp, struct uio *uio, int ioflag, struct ucred *cred) { struct nfsnode *np = VTONFS(vp); int biosize, i; struct buf *bp, *rabp; struct thread *td; struct nfsmount *nmp = VFSTONFS(vp->v_mount); daddr_t lbn, rabn; int bcount; int seqcount; int nra, error = 0, n = 0, on = 0; off_t tmp_off; KASSERT(uio->uio_rw == UIO_READ, ("ncl_read mode")); if (uio->uio_resid == 0) return (0); if (uio->uio_offset < 0) /* XXX VDIR cookies can be negative */ return (EINVAL); td = uio->uio_td; mtx_lock(&nmp->nm_mtx); if ((nmp->nm_flag & NFSMNT_NFSV3) != 0 && (nmp->nm_state & NFSSTA_GOTFSINFO) == 0) { mtx_unlock(&nmp->nm_mtx); (void)ncl_fsinfo(nmp, vp, cred, td); mtx_lock(&nmp->nm_mtx); } if (nmp->nm_rsize == 0 || nmp->nm_readdirsize == 0) (void) newnfs_iosize(nmp); tmp_off = uio->uio_offset + uio->uio_resid; if (vp->v_type != VDIR && (tmp_off > nmp->nm_maxfilesize || tmp_off < uio->uio_offset)) { mtx_unlock(&nmp->nm_mtx); return (EFBIG); } mtx_unlock(&nmp->nm_mtx); if (newnfs_directio_enable && (ioflag & IO_DIRECT) && (vp->v_type == VREG)) /* No caching/ no readaheads. Just read data into the user buffer */ return ncl_readrpc(vp, uio, cred); biosize = vp->v_bufobj.bo_bsize; seqcount = (int)((off_t)(ioflag >> IO_SEQSHIFT) * biosize / BKVASIZE); error = nfs_bioread_check_cons(vp, td, cred); if (error) return error; do { u_quad_t nsize; mtx_lock(&np->n_mtx); nsize = np->n_size; mtx_unlock(&np->n_mtx); switch (vp->v_type) { case VREG: NFSINCRGLOBAL(nfsstatsv1.biocache_reads); lbn = uio->uio_offset / biosize; on = uio->uio_offset - (lbn * biosize); /* * Start the read ahead(s), as required. */ if (nmp->nm_readahead > 0) { for (nra = 0; nra < nmp->nm_readahead && nra < seqcount && (off_t)(lbn + 1 + nra) * biosize < nsize; nra++) { rabn = lbn + 1 + nra; if (incore(&vp->v_bufobj, rabn) == NULL) { rabp = nfs_getcacheblk(vp, rabn, biosize, td); if (!rabp) { error = newnfs_sigintr(nmp, td); return (error ? error : EINTR); } if ((rabp->b_flags & (B_CACHE|B_DELWRI)) == 0) { rabp->b_flags |= B_ASYNC; rabp->b_iocmd = BIO_READ; vfs_busy_pages(rabp, 0); if (ncl_asyncio(nmp, rabp, cred, td)) { rabp->b_flags |= B_INVAL; rabp->b_ioflags |= BIO_ERROR; vfs_unbusy_pages(rabp); brelse(rabp); break; } } else { brelse(rabp); } } } } /* Note that bcount is *not* DEV_BSIZE aligned. */ bcount = biosize; if ((off_t)lbn * biosize >= nsize) { bcount = 0; } else if ((off_t)(lbn + 1) * biosize > nsize) { bcount = nsize - (off_t)lbn * biosize; } bp = nfs_getcacheblk(vp, lbn, bcount, td); if (!bp) { error = newnfs_sigintr(nmp, td); return (error ? error : EINTR); } /* * If B_CACHE is not set, we must issue the read. If this * fails, we return an error. */ if ((bp->b_flags & B_CACHE) == 0) { bp->b_iocmd = BIO_READ; vfs_busy_pages(bp, 0); error = ncl_doio(vp, bp, cred, td, 0); if (error) { brelse(bp); return (error); } } /* * on is the offset into the current bp. Figure out how many * bytes we can copy out of the bp. Note that bcount is * NOT DEV_BSIZE aligned. * * Then figure out how many bytes we can copy into the uio. */ n = 0; if (on < bcount) n = MIN((unsigned)(bcount - on), uio->uio_resid); break; case VLNK: NFSINCRGLOBAL(nfsstatsv1.biocache_readlinks); bp = nfs_getcacheblk(vp, (daddr_t)0, NFS_MAXPATHLEN, td); if (!bp) { error = newnfs_sigintr(nmp, td); return (error ? error : EINTR); } if ((bp->b_flags & B_CACHE) == 0) { bp->b_iocmd = BIO_READ; vfs_busy_pages(bp, 0); error = ncl_doio(vp, bp, cred, td, 0); if (error) { bp->b_ioflags |= BIO_ERROR; brelse(bp); return (error); } } n = MIN(uio->uio_resid, NFS_MAXPATHLEN - bp->b_resid); on = 0; break; case VDIR: NFSINCRGLOBAL(nfsstatsv1.biocache_readdirs); if (np->n_direofoffset && uio->uio_offset >= np->n_direofoffset) { return (0); } lbn = (uoff_t)uio->uio_offset / NFS_DIRBLKSIZ; on = uio->uio_offset & (NFS_DIRBLKSIZ - 1); bp = nfs_getcacheblk(vp, lbn, NFS_DIRBLKSIZ, td); if (!bp) { error = newnfs_sigintr(nmp, td); return (error ? error : EINTR); } if ((bp->b_flags & B_CACHE) == 0) { bp->b_iocmd = BIO_READ; vfs_busy_pages(bp, 0); error = ncl_doio(vp, bp, cred, td, 0); if (error) { brelse(bp); } while (error == NFSERR_BAD_COOKIE) { ncl_invaldir(vp); error = ncl_vinvalbuf(vp, 0, td, 1); if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) return (EBADF); /* * Yuck! The directory has been modified on the * server. The only way to get the block is by * reading from the beginning to get all the * offset cookies. * * Leave the last bp intact unless there is an error. * Loop back up to the while if the error is another * NFSERR_BAD_COOKIE (double yuch!). */ for (i = 0; i <= lbn && !error; i++) { if (np->n_direofoffset && (i * NFS_DIRBLKSIZ) >= np->n_direofoffset) return (0); bp = nfs_getcacheblk(vp, i, NFS_DIRBLKSIZ, td); if (!bp) { error = newnfs_sigintr(nmp, td); return (error ? error : EINTR); } if ((bp->b_flags & B_CACHE) == 0) { bp->b_iocmd = BIO_READ; vfs_busy_pages(bp, 0); error = ncl_doio(vp, bp, cred, td, 0); /* * no error + B_INVAL == directory EOF, * use the block. */ if (error == 0 && (bp->b_flags & B_INVAL)) break; } /* * An error will throw away the block and the * for loop will break out. If no error and this * is not the block we want, we throw away the * block and go for the next one via the for loop. */ if (error || i < lbn) brelse(bp); } } /* * The above while is repeated if we hit another cookie * error. If we hit an error and it wasn't a cookie error, * we give up. */ if (error) return (error); } /* * If not eof and read aheads are enabled, start one. * (You need the current block first, so that you have the * directory offset cookie of the next block.) */ if (nmp->nm_readahead > 0 && (bp->b_flags & B_INVAL) == 0 && (np->n_direofoffset == 0 || (lbn + 1) * NFS_DIRBLKSIZ < np->n_direofoffset) && incore(&vp->v_bufobj, lbn + 1) == NULL) { rabp = nfs_getcacheblk(vp, lbn + 1, NFS_DIRBLKSIZ, td); if (rabp) { if ((rabp->b_flags & (B_CACHE|B_DELWRI)) == 0) { rabp->b_flags |= B_ASYNC; rabp->b_iocmd = BIO_READ; vfs_busy_pages(rabp, 0); if (ncl_asyncio(nmp, rabp, cred, td)) { rabp->b_flags |= B_INVAL; rabp->b_ioflags |= BIO_ERROR; vfs_unbusy_pages(rabp); brelse(rabp); } } else { brelse(rabp); } } } /* * Unlike VREG files, whos buffer size ( bp->b_bcount ) is * chopped for the EOF condition, we cannot tell how large * NFS directories are going to be until we hit EOF. So * an NFS directory buffer is *not* chopped to its EOF. Now, * it just so happens that b_resid will effectively chop it * to EOF. *BUT* this information is lost if the buffer goes * away and is reconstituted into a B_CACHE state ( due to * being VMIO ) later. So we keep track of the directory eof * in np->n_direofoffset and chop it off as an extra step * right here. */ n = lmin(uio->uio_resid, NFS_DIRBLKSIZ - bp->b_resid - on); if (np->n_direofoffset && n > np->n_direofoffset - uio->uio_offset) n = np->n_direofoffset - uio->uio_offset; break; default: printf(" ncl_bioread: type %x unexpected\n", vp->v_type); bp = NULL; break; } if (n > 0) { error = vn_io_fault_uiomove(bp->b_data + on, (int)n, uio); } if (vp->v_type == VLNK) n = 0; if (bp != NULL) brelse(bp); } while (error == 0 && uio->uio_resid > 0 && n > 0); return (error); } /* * The NFS write path cannot handle iovecs with len > 1. So we need to * break up iovecs accordingly (restricting them to wsize). * For the SYNC case, we can do this with 1 copy (user buffer -> mbuf). * For the ASYNC case, 2 copies are needed. The first a copy from the * user buffer to a staging buffer and then a second copy from the staging * buffer to mbufs. This can be optimized by copying from the user buffer * directly into mbufs and passing the chain down, but that requires a * fair amount of re-working of the relevant codepaths (and can be done * later). */ static int nfs_directio_write(vp, uiop, cred, ioflag) struct vnode *vp; struct uio *uiop; struct ucred *cred; int ioflag; { int error; struct nfsmount *nmp = VFSTONFS(vp->v_mount); struct thread *td = uiop->uio_td; int size; int wsize; mtx_lock(&nmp->nm_mtx); wsize = nmp->nm_wsize; mtx_unlock(&nmp->nm_mtx); if (ioflag & IO_SYNC) { int iomode, must_commit; struct uio uio; struct iovec iov; do_sync: while (uiop->uio_resid > 0) { size = MIN(uiop->uio_resid, wsize); size = MIN(uiop->uio_iov->iov_len, size); iov.iov_base = uiop->uio_iov->iov_base; iov.iov_len = size; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = uiop->uio_offset; uio.uio_resid = size; uio.uio_segflg = UIO_USERSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; iomode = NFSWRITE_FILESYNC; error = ncl_writerpc(vp, &uio, cred, &iomode, &must_commit, 0); KASSERT((must_commit == 0), ("ncl_directio_write: Did not commit write")); if (error) return (error); uiop->uio_offset += size; uiop->uio_resid -= size; if (uiop->uio_iov->iov_len <= size) { uiop->uio_iovcnt--; uiop->uio_iov++; } else { uiop->uio_iov->iov_base = (char *)uiop->uio_iov->iov_base + size; uiop->uio_iov->iov_len -= size; } } } else { struct uio *t_uio; struct iovec *t_iov; struct buf *bp; /* * Break up the write into blocksize chunks and hand these * over to nfsiod's for write back. * Unfortunately, this incurs a copy of the data. Since * the user could modify the buffer before the write is * initiated. * * The obvious optimization here is that one of the 2 copies * in the async write path can be eliminated by copying the * data here directly into mbufs and passing the mbuf chain * down. But that will require a fair amount of re-working * of the code and can be done if there's enough interest * in NFS directio access. */ while (uiop->uio_resid > 0) { size = MIN(uiop->uio_resid, wsize); size = MIN(uiop->uio_iov->iov_len, size); bp = getpbuf(&ncl_pbuf_freecnt); t_uio = malloc(sizeof(struct uio), M_NFSDIRECTIO, M_WAITOK); t_iov = malloc(sizeof(struct iovec), M_NFSDIRECTIO, M_WAITOK); t_iov->iov_base = malloc(size, M_NFSDIRECTIO, M_WAITOK); t_iov->iov_len = size; t_uio->uio_iov = t_iov; t_uio->uio_iovcnt = 1; t_uio->uio_offset = uiop->uio_offset; t_uio->uio_resid = size; t_uio->uio_segflg = UIO_SYSSPACE; t_uio->uio_rw = UIO_WRITE; t_uio->uio_td = td; KASSERT(uiop->uio_segflg == UIO_USERSPACE || uiop->uio_segflg == UIO_SYSSPACE, ("nfs_directio_write: Bad uio_segflg")); if (uiop->uio_segflg == UIO_USERSPACE) { error = copyin(uiop->uio_iov->iov_base, t_iov->iov_base, size); if (error != 0) goto err_free; } else /* * UIO_SYSSPACE may never happen, but handle * it just in case it does. */ bcopy(uiop->uio_iov->iov_base, t_iov->iov_base, size); bp->b_flags |= B_DIRECT; bp->b_iocmd = BIO_WRITE; if (cred != NOCRED) { crhold(cred); bp->b_wcred = cred; } else bp->b_wcred = NOCRED; bp->b_caller1 = (void *)t_uio; bp->b_vp = vp; error = ncl_asyncio(nmp, bp, NOCRED, td); err_free: if (error) { free(t_iov->iov_base, M_NFSDIRECTIO); free(t_iov, M_NFSDIRECTIO); free(t_uio, M_NFSDIRECTIO); bp->b_vp = NULL; relpbuf(bp, &ncl_pbuf_freecnt); if (error == EINTR) return (error); goto do_sync; } uiop->uio_offset += size; uiop->uio_resid -= size; if (uiop->uio_iov->iov_len <= size) { uiop->uio_iovcnt--; uiop->uio_iov++; } else { uiop->uio_iov->iov_base = (char *)uiop->uio_iov->iov_base + size; uiop->uio_iov->iov_len -= size; } } } return (0); } /* * Vnode op for write using bio */ int ncl_write(struct vop_write_args *ap) { int biosize; struct uio *uio = ap->a_uio; struct thread *td = uio->uio_td; struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct ucred *cred = ap->a_cred; int ioflag = ap->a_ioflag; struct buf *bp; struct vattr vattr; struct nfsmount *nmp = VFSTONFS(vp->v_mount); daddr_t lbn; int bcount, noncontig_write, obcount; int bp_cached, n, on, error = 0, error1, wouldcommit; size_t orig_resid, local_resid; off_t orig_size, tmp_off; KASSERT(uio->uio_rw == UIO_WRITE, ("ncl_write mode")); KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, ("ncl_write proc")); if (vp->v_type != VREG) return (EIO); mtx_lock(&np->n_mtx); if (np->n_flag & NWRITEERR) { np->n_flag &= ~NWRITEERR; mtx_unlock(&np->n_mtx); return (np->n_error); } else mtx_unlock(&np->n_mtx); mtx_lock(&nmp->nm_mtx); if ((nmp->nm_flag & NFSMNT_NFSV3) != 0 && (nmp->nm_state & NFSSTA_GOTFSINFO) == 0) { mtx_unlock(&nmp->nm_mtx); (void)ncl_fsinfo(nmp, vp, cred, td); mtx_lock(&nmp->nm_mtx); } if (nmp->nm_wsize == 0) (void) newnfs_iosize(nmp); mtx_unlock(&nmp->nm_mtx); /* * Synchronously flush pending buffers if we are in synchronous * mode or if we are appending. */ if (ioflag & (IO_APPEND | IO_SYNC)) { mtx_lock(&np->n_mtx); if (np->n_flag & NMODIFIED) { mtx_unlock(&np->n_mtx); #ifdef notyet /* Needs matching nonblock semantics elsewhere, too. */ /* * Require non-blocking, synchronous writes to * dirty files to inform the program it needs * to fsync(2) explicitly. */ if (ioflag & IO_NDELAY) return (EAGAIN); #endif np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); error = ncl_vinvalbuf(vp, V_SAVE | ((ioflag & IO_VMIO) != 0 ? V_VMIO : 0), td, 1); if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) error = EBADF; if (error != 0) return (error); } else mtx_unlock(&np->n_mtx); } orig_resid = uio->uio_resid; mtx_lock(&np->n_mtx); orig_size = np->n_size; mtx_unlock(&np->n_mtx); /* * If IO_APPEND then load uio_offset. We restart here if we cannot * get the append lock. */ if (ioflag & IO_APPEND) { np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); error = VOP_GETATTR(vp, &vattr, cred); if (error) return (error); mtx_lock(&np->n_mtx); uio->uio_offset = np->n_size; mtx_unlock(&np->n_mtx); } if (uio->uio_offset < 0) return (EINVAL); tmp_off = uio->uio_offset + uio->uio_resid; if (tmp_off > nmp->nm_maxfilesize || tmp_off < uio->uio_offset) return (EFBIG); if (uio->uio_resid == 0) return (0); if (newnfs_directio_enable && (ioflag & IO_DIRECT) && vp->v_type == VREG) return nfs_directio_write(vp, uio, cred, ioflag); /* * Maybe this should be above the vnode op call, but so long as * file servers have no limits, i don't think it matters */ if (vn_rlimit_fsize(vp, uio, td)) return (EFBIG); biosize = vp->v_bufobj.bo_bsize; /* * Find all of this file's B_NEEDCOMMIT buffers. If our writes * would exceed the local maximum per-file write commit size when * combined with those, we must decide whether to flush, * go synchronous, or return error. We don't bother checking * IO_UNIT -- we just make all writes atomic anyway, as there's * no point optimizing for something that really won't ever happen. */ wouldcommit = 0; if (!(ioflag & IO_SYNC)) { int nflag; mtx_lock(&np->n_mtx); nflag = np->n_flag; mtx_unlock(&np->n_mtx); if (nflag & NMODIFIED) { BO_LOCK(&vp->v_bufobj); if (vp->v_bufobj.bo_dirty.bv_cnt != 0) { TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) { if (bp->b_flags & B_NEEDCOMMIT) wouldcommit += bp->b_bcount; } } BO_UNLOCK(&vp->v_bufobj); } } do { if (!(ioflag & IO_SYNC)) { wouldcommit += biosize; if (wouldcommit > nmp->nm_wcommitsize) { np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); error = ncl_vinvalbuf(vp, V_SAVE | ((ioflag & IO_VMIO) != 0 ? V_VMIO : 0), td, 1); if (error == 0 && (vp->v_iflag & VI_DOOMED) != 0) error = EBADF; if (error != 0) return (error); wouldcommit = biosize; } } NFSINCRGLOBAL(nfsstatsv1.biocache_writes); lbn = uio->uio_offset / biosize; on = uio->uio_offset - (lbn * biosize); n = MIN((unsigned)(biosize - on), uio->uio_resid); again: /* * Handle direct append and file extension cases, calculate * unaligned buffer size. */ mtx_lock(&np->n_mtx); if ((np->n_flag & NHASBEENLOCKED) == 0 && (nmp->nm_flag & NFSMNT_NONCONTIGWR) != 0) noncontig_write = 1; else noncontig_write = 0; if ((uio->uio_offset == np->n_size || (noncontig_write != 0 && lbn == (np->n_size / biosize) && uio->uio_offset + n > np->n_size)) && n) { mtx_unlock(&np->n_mtx); /* * Get the buffer (in its pre-append state to maintain * B_CACHE if it was previously set). Resize the * nfsnode after we have locked the buffer to prevent * readers from reading garbage. */ obcount = np->n_size - (lbn * biosize); bp = nfs_getcacheblk(vp, lbn, obcount, td); if (bp != NULL) { long save; mtx_lock(&np->n_mtx); np->n_size = uio->uio_offset + n; np->n_flag |= NMODIFIED; vnode_pager_setsize(vp, np->n_size); mtx_unlock(&np->n_mtx); save = bp->b_flags & B_CACHE; bcount = on + n; allocbuf(bp, bcount); bp->b_flags |= save; if (noncontig_write != 0 && on > obcount) vfs_bio_bzero_buf(bp, obcount, on - obcount); } } else { /* * Obtain the locked cache block first, and then * adjust the file's size as appropriate. */ bcount = on + n; if ((off_t)lbn * biosize + bcount < np->n_size) { if ((off_t)(lbn + 1) * biosize < np->n_size) bcount = biosize; else bcount = np->n_size - (off_t)lbn * biosize; } mtx_unlock(&np->n_mtx); bp = nfs_getcacheblk(vp, lbn, bcount, td); mtx_lock(&np->n_mtx); if (uio->uio_offset + n > np->n_size) { np->n_size = uio->uio_offset + n; np->n_flag |= NMODIFIED; vnode_pager_setsize(vp, np->n_size); } mtx_unlock(&np->n_mtx); } if (!bp) { error = newnfs_sigintr(nmp, td); if (!error) error = EINTR; break; } /* * Issue a READ if B_CACHE is not set. In special-append * mode, B_CACHE is based on the buffer prior to the write * op and is typically set, avoiding the read. If a read * is required in special append mode, the server will * probably send us a short-read since we extended the file * on our end, resulting in b_resid == 0 and, thusly, * B_CACHE getting set. * * We can also avoid issuing the read if the write covers * the entire buffer. We have to make sure the buffer state * is reasonable in this case since we will not be initiating * I/O. See the comments in kern/vfs_bio.c's getblk() for * more information. * * B_CACHE may also be set due to the buffer being cached * normally. */ bp_cached = 1; if (on == 0 && n == bcount) { if ((bp->b_flags & B_CACHE) == 0) bp_cached = 0; bp->b_flags |= B_CACHE; bp->b_flags &= ~B_INVAL; bp->b_ioflags &= ~BIO_ERROR; } if ((bp->b_flags & B_CACHE) == 0) { bp->b_iocmd = BIO_READ; vfs_busy_pages(bp, 0); error = ncl_doio(vp, bp, cred, td, 0); if (error) { brelse(bp); break; } } if (bp->b_wcred == NOCRED) bp->b_wcred = crhold(cred); mtx_lock(&np->n_mtx); np->n_flag |= NMODIFIED; mtx_unlock(&np->n_mtx); /* * If dirtyend exceeds file size, chop it down. This should * not normally occur but there is an append race where it * might occur XXX, so we log it. * * If the chopping creates a reverse-indexed or degenerate * situation with dirtyoff/end, we 0 both of them. */ if (bp->b_dirtyend > bcount) { printf("NFS append race @%lx:%d\n", (long)bp->b_blkno * DEV_BSIZE, bp->b_dirtyend - bcount); bp->b_dirtyend = bcount; } if (bp->b_dirtyoff >= bp->b_dirtyend) bp->b_dirtyoff = bp->b_dirtyend = 0; /* * If the new write will leave a contiguous dirty * area, just update the b_dirtyoff and b_dirtyend, * otherwise force a write rpc of the old dirty area. * * If there has been a file lock applied to this file * or vfs.nfs.old_noncontig_writing is set, do the following: * While it is possible to merge discontiguous writes due to * our having a B_CACHE buffer ( and thus valid read data * for the hole), we don't because it could lead to * significant cache coherency problems with multiple clients, * especially if locking is implemented later on. * * If vfs.nfs.old_noncontig_writing is not set and there has * not been file locking done on this file: * Relax coherency a bit for the sake of performance and * expand the current dirty region to contain the new * write even if it means we mark some non-dirty data as * dirty. */ if (noncontig_write == 0 && bp->b_dirtyend > 0 && (on > bp->b_dirtyend || (on + n) < bp->b_dirtyoff)) { if (bwrite(bp) == EINTR) { error = EINTR; break; } goto again; } local_resid = uio->uio_resid; error = vn_io_fault_uiomove((char *)bp->b_data + on, n, uio); if (error != 0 && !bp_cached) { /* * This block has no other content then what * possibly was written by the faulty uiomove. * Release it, forgetting the data pages, to * prevent the leak of uninitialized data to * usermode. */ bp->b_ioflags |= BIO_ERROR; brelse(bp); uio->uio_offset -= local_resid - uio->uio_resid; uio->uio_resid = local_resid; break; } /* * Since this block is being modified, it must be written * again and not just committed. Since write clustering does * not work for the stage 1 data write, only the stage 2 * commit rpc, we have to clear B_CLUSTEROK as well. */ bp->b_flags &= ~(B_NEEDCOMMIT | B_CLUSTEROK); /* * Get the partial update on the progress made from * uiomove, if an error occurred. */ if (error != 0) n = local_resid - uio->uio_resid; /* * Only update dirtyoff/dirtyend if not a degenerate * condition. */ if (n > 0) { if (bp->b_dirtyend > 0) { bp->b_dirtyoff = min(on, bp->b_dirtyoff); bp->b_dirtyend = max((on + n), bp->b_dirtyend); } else { bp->b_dirtyoff = on; bp->b_dirtyend = on + n; } vfs_bio_set_valid(bp, on, n); } /* * If IO_SYNC do bwrite(). * * IO_INVAL appears to be unused. The idea appears to be * to turn off caching in this case. Very odd. XXX */ if ((ioflag & IO_SYNC)) { if (ioflag & IO_INVAL) bp->b_flags |= B_NOCACHE; error1 = bwrite(bp); if (error1 != 0) { if (error == 0) error = error1; break; } } else if ((n + on) == biosize || (ioflag & IO_ASYNC) != 0) { bp->b_flags |= B_ASYNC; (void) ncl_writebp(bp, 0, NULL); } else { bdwrite(bp); } if (error != 0) break; } while (uio->uio_resid > 0 && n > 0); if (error != 0) { if (ioflag & IO_UNIT) { VATTR_NULL(&vattr); vattr.va_size = orig_size; /* IO_SYNC is handled implicitely */ (void)VOP_SETATTR(vp, &vattr, cred); uio->uio_offset -= orig_resid - uio->uio_resid; uio->uio_resid = orig_resid; } } return (error); } /* * Get an nfs cache block. * * Allocate a new one if the block isn't currently in the cache * and return the block marked busy. If the calling process is * interrupted by a signal for an interruptible mount point, return * NULL. * * The caller must carefully deal with the possible B_INVAL state of * the buffer. ncl_doio() clears B_INVAL (and ncl_asyncio() clears it * indirectly), so synchronous reads can be issued without worrying about * the B_INVAL state. We have to be a little more careful when dealing * with writes (see comments in nfs_write()) when extending a file past * its EOF. */ static struct buf * nfs_getcacheblk(struct vnode *vp, daddr_t bn, int size, struct thread *td) { struct buf *bp; struct mount *mp; struct nfsmount *nmp; mp = vp->v_mount; nmp = VFSTONFS(mp); if (nmp->nm_flag & NFSMNT_INT) { sigset_t oldset; newnfs_set_sigmask(td, &oldset); bp = getblk(vp, bn, size, PCATCH, 0, 0); newnfs_restore_sigmask(td, &oldset); while (bp == NULL) { if (newnfs_sigintr(nmp, td)) return (NULL); bp = getblk(vp, bn, size, 0, 2 * hz, 0); } } else { bp = getblk(vp, bn, size, 0, 0, 0); } if (vp->v_type == VREG) bp->b_blkno = bn * (vp->v_bufobj.bo_bsize / DEV_BSIZE); return (bp); } /* * Flush and invalidate all dirty buffers. If another process is already * doing the flush, just wait for completion. */ int ncl_vinvalbuf(struct vnode *vp, int flags, struct thread *td, int intrflg) { struct nfsnode *np = VTONFS(vp); struct nfsmount *nmp = VFSTONFS(vp->v_mount); int error = 0, slpflag, slptimeo; int old_lock = 0; ASSERT_VOP_LOCKED(vp, "ncl_vinvalbuf"); if ((nmp->nm_flag & NFSMNT_INT) == 0) intrflg = 0; if ((nmp->nm_mountp->mnt_kern_flag & MNTK_UNMOUNTF)) intrflg = 1; if (intrflg) { slpflag = PCATCH; slptimeo = 2 * hz; } else { slpflag = 0; slptimeo = 0; } old_lock = ncl_upgrade_vnlock(vp); if (vp->v_iflag & VI_DOOMED) { /* * Since vgonel() uses the generic vinvalbuf() to flush * dirty buffers and it does not call this function, it * is safe to just return OK when VI_DOOMED is set. */ ncl_downgrade_vnlock(vp, old_lock); return (0); } /* * Now, flush as required. */ if ((flags & (V_SAVE | V_VMIO)) == V_SAVE && vp->v_bufobj.bo_object != NULL) { VM_OBJECT_WLOCK(vp->v_bufobj.bo_object); vm_object_page_clean(vp->v_bufobj.bo_object, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(vp->v_bufobj.bo_object); /* * If the page clean was interrupted, fail the invalidation. * Not doing so, we run the risk of losing dirty pages in the * vinvalbuf() call below. */ if (intrflg && (error = newnfs_sigintr(nmp, td))) goto out; } error = vinvalbuf(vp, flags, slpflag, 0); while (error) { if (intrflg && (error = newnfs_sigintr(nmp, td))) goto out; error = vinvalbuf(vp, flags, 0, slptimeo); } if (NFSHASPNFS(nmp)) { nfscl_layoutcommit(vp, td); /* * Invalidate the attribute cache, since writes to a DS * won't update the size attribute. */ mtx_lock(&np->n_mtx); np->n_attrstamp = 0; } else mtx_lock(&np->n_mtx); if (np->n_directio_asyncwr == 0) np->n_flag &= ~NMODIFIED; mtx_unlock(&np->n_mtx); out: ncl_downgrade_vnlock(vp, old_lock); return error; } /* * Initiate asynchronous I/O. Return an error if no nfsiods are available. * This is mainly to avoid queueing async I/O requests when the nfsiods * are all hung on a dead server. * * Note: ncl_asyncio() does not clear (BIO_ERROR|B_INVAL) but when the bp * is eventually dequeued by the async daemon, ncl_doio() *will*. */ int ncl_asyncio(struct nfsmount *nmp, struct buf *bp, struct ucred *cred, struct thread *td) { int iod; int gotiod; int slpflag = 0; int slptimeo = 0; int error, error2; /* * Commits are usually short and sweet so lets save some cpu and * leave the async daemons for more important rpc's (such as reads * and writes). * * Readdirplus RPCs do vget()s to acquire the vnodes for entries * in the directory in order to update attributes. This can deadlock * with another thread that is waiting for async I/O to be done by * an nfsiod thread while holding a lock on one of these vnodes. * To avoid this deadlock, don't allow the async nfsiod threads to * perform Readdirplus RPCs. */ mtx_lock(&ncl_iod_mutex); if ((bp->b_iocmd == BIO_WRITE && (bp->b_flags & B_NEEDCOMMIT) && (nmp->nm_bufqiods > ncl_numasync / 2)) || (bp->b_vp->v_type == VDIR && (nmp->nm_flag & NFSMNT_RDIRPLUS))) { mtx_unlock(&ncl_iod_mutex); return(EIO); } again: if (nmp->nm_flag & NFSMNT_INT) slpflag = PCATCH; gotiod = FALSE; /* * Find a free iod to process this request. */ for (iod = 0; iod < ncl_numasync; iod++) if (ncl_iodwant[iod] == NFSIOD_AVAILABLE) { gotiod = TRUE; break; } /* * Try to create one if none are free. */ if (!gotiod) ncl_nfsiodnew(); else { /* * Found one, so wake it up and tell it which * mount to process. */ NFS_DPF(ASYNCIO, ("ncl_asyncio: waking iod %d for mount %p\n", iod, nmp)); ncl_iodwant[iod] = NFSIOD_NOT_AVAILABLE; ncl_iodmount[iod] = nmp; nmp->nm_bufqiods++; wakeup(&ncl_iodwant[iod]); } /* * If none are free, we may already have an iod working on this mount * point. If so, it will process our request. */ if (!gotiod) { if (nmp->nm_bufqiods > 0) { NFS_DPF(ASYNCIO, ("ncl_asyncio: %d iods are already processing mount %p\n", nmp->nm_bufqiods, nmp)); gotiod = TRUE; } } /* * If we have an iod which can process the request, then queue * the buffer. */ if (gotiod) { /* * Ensure that the queue never grows too large. We still want * to asynchronize so we block rather then return EIO. */ while (nmp->nm_bufqlen >= 2*ncl_numasync) { NFS_DPF(ASYNCIO, ("ncl_asyncio: waiting for mount %p queue to drain\n", nmp)); nmp->nm_bufqwant = TRUE; error = newnfs_msleep(td, &nmp->nm_bufq, &ncl_iod_mutex, slpflag | PRIBIO, "nfsaio", slptimeo); if (error) { error2 = newnfs_sigintr(nmp, td); if (error2) { mtx_unlock(&ncl_iod_mutex); return (error2); } if (slpflag == PCATCH) { slpflag = 0; slptimeo = 2 * hz; } } /* * We might have lost our iod while sleeping, * so check and loop if necessary. */ goto again; } /* We might have lost our nfsiod */ if (nmp->nm_bufqiods == 0) { NFS_DPF(ASYNCIO, ("ncl_asyncio: no iods after mount %p queue was drained, looping\n", nmp)); goto again; } if (bp->b_iocmd == BIO_READ) { if (bp->b_rcred == NOCRED && cred != NOCRED) bp->b_rcred = crhold(cred); } else { if (bp->b_wcred == NOCRED && cred != NOCRED) bp->b_wcred = crhold(cred); } if (bp->b_flags & B_REMFREE) bremfreef(bp); BUF_KERNPROC(bp); TAILQ_INSERT_TAIL(&nmp->nm_bufq, bp, b_freelist); nmp->nm_bufqlen++; if ((bp->b_flags & B_DIRECT) && bp->b_iocmd == BIO_WRITE) { mtx_lock(&(VTONFS(bp->b_vp))->n_mtx); VTONFS(bp->b_vp)->n_flag |= NMODIFIED; VTONFS(bp->b_vp)->n_directio_asyncwr++; mtx_unlock(&(VTONFS(bp->b_vp))->n_mtx); } mtx_unlock(&ncl_iod_mutex); return (0); } mtx_unlock(&ncl_iod_mutex); /* * All the iods are busy on other mounts, so return EIO to * force the caller to process the i/o synchronously. */ NFS_DPF(ASYNCIO, ("ncl_asyncio: no iods available, i/o is synchronous\n")); return (EIO); } void ncl_doio_directwrite(struct buf *bp) { int iomode, must_commit; struct uio *uiop = (struct uio *)bp->b_caller1; char *iov_base = uiop->uio_iov->iov_base; iomode = NFSWRITE_FILESYNC; uiop->uio_td = NULL; /* NULL since we're in nfsiod */ ncl_writerpc(bp->b_vp, uiop, bp->b_wcred, &iomode, &must_commit, 0); KASSERT((must_commit == 0), ("ncl_doio_directwrite: Did not commit write")); free(iov_base, M_NFSDIRECTIO); free(uiop->uio_iov, M_NFSDIRECTIO); free(uiop, M_NFSDIRECTIO); if ((bp->b_flags & B_DIRECT) && bp->b_iocmd == BIO_WRITE) { struct nfsnode *np = VTONFS(bp->b_vp); mtx_lock(&np->n_mtx); if (NFSHASPNFS(VFSTONFS(vnode_mount(bp->b_vp)))) { /* * Invalidate the attribute cache, since writes to a DS * won't update the size attribute. */ np->n_attrstamp = 0; } np->n_directio_asyncwr--; if (np->n_directio_asyncwr == 0) { np->n_flag &= ~NMODIFIED; if ((np->n_flag & NFSYNCWAIT)) { np->n_flag &= ~NFSYNCWAIT; wakeup((caddr_t)&np->n_directio_asyncwr); } } mtx_unlock(&np->n_mtx); } bp->b_vp = NULL; relpbuf(bp, &ncl_pbuf_freecnt); } /* * Do an I/O operation to/from a cache block. This may be called * synchronously or from an nfsiod. */ int ncl_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td, int called_from_strategy) { struct uio *uiop; struct nfsnode *np; struct nfsmount *nmp; int error = 0, iomode, must_commit = 0; struct uio uio; struct iovec io; struct proc *p = td ? td->td_proc : NULL; uint8_t iocmd; np = VTONFS(vp); nmp = VFSTONFS(vp->v_mount); uiop = &uio; uiop->uio_iov = &io; uiop->uio_iovcnt = 1; uiop->uio_segflg = UIO_SYSSPACE; uiop->uio_td = td; /* * clear BIO_ERROR and B_INVAL state prior to initiating the I/O. We * do this here so we do not have to do it in all the code that * calls us. */ bp->b_flags &= ~B_INVAL; bp->b_ioflags &= ~BIO_ERROR; KASSERT(!(bp->b_flags & B_DONE), ("ncl_doio: bp %p already marked done", bp)); iocmd = bp->b_iocmd; if (iocmd == BIO_READ) { io.iov_len = uiop->uio_resid = bp->b_bcount; io.iov_base = bp->b_data; uiop->uio_rw = UIO_READ; switch (vp->v_type) { case VREG: uiop->uio_offset = ((off_t)bp->b_blkno) * DEV_BSIZE; NFSINCRGLOBAL(nfsstatsv1.read_bios); error = ncl_readrpc(vp, uiop, cr); if (!error) { if (uiop->uio_resid) { /* * If we had a short read with no error, we must have * hit a file hole. We should zero-fill the remainder. * This can also occur if the server hits the file EOF. * * Holes used to be able to occur due to pending * writes, but that is not possible any longer. */ int nread = bp->b_bcount - uiop->uio_resid; ssize_t left = uiop->uio_resid; if (left > 0) bzero((char *)bp->b_data + nread, left); uiop->uio_resid = 0; } } /* ASSERT_VOP_LOCKED(vp, "ncl_doio"); */ if (p && (vp->v_vflag & VV_TEXT)) { mtx_lock(&np->n_mtx); if (NFS_TIMESPEC_COMPARE(&np->n_mtime, &np->n_vattr.na_mtime)) { mtx_unlock(&np->n_mtx); PROC_LOCK(p); killproc(p, "text file modification"); PROC_UNLOCK(p); } else mtx_unlock(&np->n_mtx); } break; case VLNK: uiop->uio_offset = (off_t)0; NFSINCRGLOBAL(nfsstatsv1.readlink_bios); error = ncl_readlinkrpc(vp, uiop, cr); break; case VDIR: NFSINCRGLOBAL(nfsstatsv1.readdir_bios); uiop->uio_offset = ((u_quad_t)bp->b_lblkno) * NFS_DIRBLKSIZ; if ((nmp->nm_flag & NFSMNT_RDIRPLUS) != 0) { error = ncl_readdirplusrpc(vp, uiop, cr, td); if (error == NFSERR_NOTSUPP) nmp->nm_flag &= ~NFSMNT_RDIRPLUS; } if ((nmp->nm_flag & NFSMNT_RDIRPLUS) == 0) error = ncl_readdirrpc(vp, uiop, cr, td); /* * end-of-directory sets B_INVAL but does not generate an * error. */ if (error == 0 && uiop->uio_resid == bp->b_bcount) bp->b_flags |= B_INVAL; break; default: printf("ncl_doio: type %x unexpected\n", vp->v_type); break; } if (error) { bp->b_ioflags |= BIO_ERROR; bp->b_error = error; } } else { /* * If we only need to commit, try to commit */ if (bp->b_flags & B_NEEDCOMMIT) { int retv; off_t off; off = ((u_quad_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff; retv = ncl_commit(vp, off, bp->b_dirtyend-bp->b_dirtyoff, bp->b_wcred, td); if (retv == 0) { bp->b_dirtyoff = bp->b_dirtyend = 0; bp->b_flags &= ~(B_NEEDCOMMIT | B_CLUSTEROK); bp->b_resid = 0; bufdone(bp); return (0); } if (retv == NFSERR_STALEWRITEVERF) { ncl_clearcommit(vp->v_mount); } } /* * Setup for actual write */ mtx_lock(&np->n_mtx); if ((off_t)bp->b_blkno * DEV_BSIZE + bp->b_dirtyend > np->n_size) bp->b_dirtyend = np->n_size - (off_t)bp->b_blkno * DEV_BSIZE; mtx_unlock(&np->n_mtx); if (bp->b_dirtyend > bp->b_dirtyoff) { io.iov_len = uiop->uio_resid = bp->b_dirtyend - bp->b_dirtyoff; uiop->uio_offset = (off_t)bp->b_blkno * DEV_BSIZE + bp->b_dirtyoff; io.iov_base = (char *)bp->b_data + bp->b_dirtyoff; uiop->uio_rw = UIO_WRITE; NFSINCRGLOBAL(nfsstatsv1.write_bios); if ((bp->b_flags & (B_ASYNC | B_NEEDCOMMIT | B_NOCACHE | B_CLUSTER)) == B_ASYNC) iomode = NFSWRITE_UNSTABLE; else iomode = NFSWRITE_FILESYNC; error = ncl_writerpc(vp, uiop, cr, &iomode, &must_commit, called_from_strategy); /* * When setting B_NEEDCOMMIT also set B_CLUSTEROK to try * to cluster the buffers needing commit. This will allow * the system to submit a single commit rpc for the whole * cluster. We can do this even if the buffer is not 100% * dirty (relative to the NFS blocksize), so we optimize the * append-to-file-case. * * (when clearing B_NEEDCOMMIT, B_CLUSTEROK must also be * cleared because write clustering only works for commit * rpc's, not for the data portion of the write). */ if (!error && iomode == NFSWRITE_UNSTABLE) { bp->b_flags |= B_NEEDCOMMIT; if (bp->b_dirtyoff == 0 && bp->b_dirtyend == bp->b_bcount) bp->b_flags |= B_CLUSTEROK; } else { bp->b_flags &= ~(B_NEEDCOMMIT | B_CLUSTEROK); } /* * For an interrupted write, the buffer is still valid * and the write hasn't been pushed to the server yet, * so we can't set BIO_ERROR and report the interruption * by setting B_EINTR. For the B_ASYNC case, B_EINTR * is not relevant, so the rpc attempt is essentially * a noop. For the case of a V3 write rpc not being * committed to stable storage, the block is still * dirty and requires either a commit rpc or another * write rpc with iomode == NFSV3WRITE_FILESYNC before * the block is reused. This is indicated by setting * the B_DELWRI and B_NEEDCOMMIT flags. * * EIO is returned by ncl_writerpc() to indicate a recoverable * write error and is handled as above, except that * B_EINTR isn't set. One cause of this is a stale stateid * error for the RPC that indicates recovery is required, * when called with called_from_strategy != 0. * * If the buffer is marked B_PAGING, it does not reside on * the vp's paging queues so we cannot call bdirty(). The * bp in this case is not an NFS cache block so we should * be safe. XXX * * The logic below breaks up errors into recoverable and * unrecoverable. For the former, we clear B_INVAL|B_NOCACHE * and keep the buffer around for potential write retries. * For the latter (eg ESTALE), we toss the buffer away (B_INVAL) * and save the error in the nfsnode. This is less than ideal * but necessary. Keeping such buffers around could potentially * cause buffer exhaustion eventually (they can never be written * out, so will get constantly be re-dirtied). It also causes * all sorts of vfs panics. For non-recoverable write errors, * also invalidate the attrcache, so we'll be forced to go over * the wire for this object, returning an error to user on next * call (most of the time). */ if (error == EINTR || error == EIO || error == ETIMEDOUT || (!error && (bp->b_flags & B_NEEDCOMMIT))) { bp->b_flags &= ~(B_INVAL|B_NOCACHE); if ((bp->b_flags & B_PAGING) == 0) { bdirty(bp); bp->b_flags &= ~B_DONE; } if ((error == EINTR || error == ETIMEDOUT) && (bp->b_flags & B_ASYNC) == 0) bp->b_flags |= B_EINTR; } else { if (error) { bp->b_ioflags |= BIO_ERROR; bp->b_flags |= B_INVAL; bp->b_error = np->n_error = error; mtx_lock(&np->n_mtx); np->n_flag |= NWRITEERR; np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); mtx_unlock(&np->n_mtx); } bp->b_dirtyoff = bp->b_dirtyend = 0; } } else { bp->b_resid = 0; bufdone(bp); return (0); } } bp->b_resid = uiop->uio_resid; if (must_commit) ncl_clearcommit(vp->v_mount); bufdone(bp); return (error); } /* * Used to aid in handling ftruncate() operations on the NFS client side. * Truncation creates a number of special problems for NFS. We have to * throw away VM pages and buffer cache buffers that are beyond EOF, and * we have to properly handle VM pages or (potentially dirty) buffers * that straddle the truncation point. */ int ncl_meta_setsize(struct vnode *vp, struct ucred *cred, struct thread *td, u_quad_t nsize) { struct nfsnode *np = VTONFS(vp); u_quad_t tsize; int biosize = vp->v_bufobj.bo_bsize; int error = 0; mtx_lock(&np->n_mtx); tsize = np->n_size; np->n_size = nsize; mtx_unlock(&np->n_mtx); if (nsize < tsize) { struct buf *bp; daddr_t lbn; int bufsize; /* * vtruncbuf() doesn't get the buffer overlapping the * truncation point. We may have a B_DELWRI and/or B_CACHE * buffer that now needs to be truncated. */ error = vtruncbuf(vp, cred, nsize, biosize); lbn = nsize / biosize; bufsize = nsize - (lbn * biosize); bp = nfs_getcacheblk(vp, lbn, bufsize, td); if (!bp) return EINTR; if (bp->b_dirtyoff > bp->b_bcount) bp->b_dirtyoff = bp->b_bcount; if (bp->b_dirtyend > bp->b_bcount) bp->b_dirtyend = bp->b_bcount; bp->b_flags |= B_RELBUF; /* don't leave garbage around */ brelse(bp); } else { vnode_pager_setsize(vp, nsize); } return(error); } Index: projects/runtime-coverage/sys/fs/smbfs/smbfs_io.c =================================================================== --- projects/runtime-coverage/sys/fs/smbfs/smbfs_io.c (revision 321584) +++ projects/runtime-coverage/sys/fs/smbfs/smbfs_io.c (revision 321585) @@ -1,676 +1,678 @@ /*- * Copyright (c) 2000-2001 Boris Popov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* #include */ #include #include #include #include #include #include /*#define SMBFS_RWGENERIC*/ extern int smbfs_pbuf_freecnt; static int smbfs_fastlookup = 1; SYSCTL_DECL(_vfs_smbfs); SYSCTL_INT(_vfs_smbfs, OID_AUTO, fastlookup, CTLFLAG_RW, &smbfs_fastlookup, 0, ""); #define DE_SIZE (sizeof(struct dirent)) static int smbfs_readvdir(struct vnode *vp, struct uio *uio, struct ucred *cred) { struct dirent de; struct componentname cn; struct smb_cred *scred; struct smbfs_fctx *ctx; struct vnode *newvp; struct smbnode *np = VTOSMB(vp); int error/*, *eofflag = ap->a_eofflag*/; long offset, limit; np = VTOSMB(vp); SMBVDEBUG("dirname='%s'\n", np->n_name); scred = smbfs_malloc_scred(); smb_makescred(scred, uio->uio_td, cred); offset = uio->uio_offset / DE_SIZE; /* offset in the directory */ limit = uio->uio_resid / DE_SIZE; if (uio->uio_resid < DE_SIZE || uio->uio_offset < 0) { error = EINVAL; goto out; } while (limit && offset < 2) { limit--; bzero((caddr_t)&de, DE_SIZE); de.d_reclen = DE_SIZE; de.d_fileno = (offset == 0) ? np->n_ino : (np->n_parent ? np->n_parentino : 2); if (de.d_fileno == 0) de.d_fileno = 0x7ffffffd + offset; de.d_namlen = offset + 1; de.d_name[0] = '.'; de.d_name[1] = '.'; de.d_name[offset + 1] = '\0'; de.d_type = DT_DIR; error = uiomove(&de, DE_SIZE, uio); if (error) goto out; offset++; uio->uio_offset += DE_SIZE; } if (limit == 0) { error = 0; goto out; } if (offset != np->n_dirofs || np->n_dirseq == NULL) { SMBVDEBUG("Reopening search %ld:%ld\n", offset, np->n_dirofs); if (np->n_dirseq) { smbfs_findclose(np->n_dirseq, scred); np->n_dirseq = NULL; } np->n_dirofs = 2; error = smbfs_findopen(np, "*", 1, SMB_FA_SYSTEM | SMB_FA_HIDDEN | SMB_FA_DIR, scred, &ctx); if (error) { SMBVDEBUG("can not open search, error = %d", error); goto out; } np->n_dirseq = ctx; } else ctx = np->n_dirseq; while (np->n_dirofs < offset) { error = smbfs_findnext(ctx, offset - np->n_dirofs++, scred); if (error) { smbfs_findclose(np->n_dirseq, scred); np->n_dirseq = NULL; error = ENOENT ? 0 : error; goto out; } } error = 0; for (; limit; limit--, offset++) { error = smbfs_findnext(ctx, limit, scred); if (error) break; np->n_dirofs++; bzero((caddr_t)&de, DE_SIZE); de.d_reclen = DE_SIZE; de.d_fileno = ctx->f_attr.fa_ino; de.d_type = (ctx->f_attr.fa_attr & SMB_FA_DIR) ? DT_DIR : DT_REG; de.d_namlen = ctx->f_nmlen; bcopy(ctx->f_name, de.d_name, de.d_namlen); de.d_name[de.d_namlen] = '\0'; if (smbfs_fastlookup) { error = smbfs_nget(vp->v_mount, vp, ctx->f_name, ctx->f_nmlen, &ctx->f_attr, &newvp); if (!error) { cn.cn_nameptr = de.d_name; cn.cn_namelen = de.d_namlen; cache_enter(vp, newvp, &cn); vput(newvp); } } error = uiomove(&de, DE_SIZE, uio); if (error) break; } if (error == ENOENT) error = 0; uio->uio_offset = offset * DE_SIZE; out: smbfs_free_scred(scred); return error; } int smbfs_readvnode(struct vnode *vp, struct uio *uiop, struct ucred *cred) { struct smbmount *smp = VFSTOSMBFS(vp->v_mount); struct smbnode *np = VTOSMB(vp); struct thread *td; struct vattr vattr; struct smb_cred *scred; int error, lks; /* * Protect against method which is not supported for now */ if (uiop->uio_segflg == UIO_NOCOPY) return EOPNOTSUPP; if (vp->v_type != VREG && vp->v_type != VDIR) { SMBFSERR("vn types other than VREG or VDIR are unsupported !\n"); return EIO; } if (uiop->uio_resid == 0) return 0; if (uiop->uio_offset < 0) return EINVAL; /* if (uiop->uio_offset + uiop->uio_resid > smp->nm_maxfilesize) return EFBIG;*/ td = uiop->uio_td; if (vp->v_type == VDIR) { lks = LK_EXCLUSIVE; /* lockstatus(vp->v_vnlock); */ if (lks == LK_SHARED) vn_lock(vp, LK_UPGRADE | LK_RETRY); error = smbfs_readvdir(vp, uiop, cred); if (lks == LK_SHARED) vn_lock(vp, LK_DOWNGRADE | LK_RETRY); return error; } /* biosize = SSTOCN(smp->sm_share)->sc_txmax;*/ if (np->n_flag & NMODIFIED) { smbfs_attr_cacheremove(vp); error = VOP_GETATTR(vp, &vattr, cred); if (error) return error; np->n_mtime.tv_sec = vattr.va_mtime.tv_sec; } else { error = VOP_GETATTR(vp, &vattr, cred); if (error) return error; if (np->n_mtime.tv_sec != vattr.va_mtime.tv_sec) { error = smbfs_vinvalbuf(vp, td); if (error) return error; np->n_mtime.tv_sec = vattr.va_mtime.tv_sec; } } scred = smbfs_malloc_scred(); smb_makescred(scred, td, cred); error = smb_read(smp->sm_share, np->n_fid, uiop, scred); smbfs_free_scred(scred); return (error); } int smbfs_writevnode(struct vnode *vp, struct uio *uiop, struct ucred *cred, int ioflag) { struct smbmount *smp = VTOSMBFS(vp); struct smbnode *np = VTOSMB(vp); struct smb_cred *scred; struct thread *td; int error = 0; if (vp->v_type != VREG) { SMBERROR("vn types other than VREG unsupported !\n"); return EIO; } SMBVDEBUG("ofs=%jd,resid=%zd\n", (intmax_t)uiop->uio_offset, uiop->uio_resid); if (uiop->uio_offset < 0) return EINVAL; /* if (uiop->uio_offset + uiop->uio_resid > smp->nm_maxfilesize) return (EFBIG);*/ td = uiop->uio_td; if (ioflag & (IO_APPEND | IO_SYNC)) { if (np->n_flag & NMODIFIED) { smbfs_attr_cacheremove(vp); error = smbfs_vinvalbuf(vp, td); if (error) return error; } if (ioflag & IO_APPEND) { #ifdef notyet /* * File size can be changed by another client */ smbfs_attr_cacheremove(vp); error = VOP_GETATTR(vp, &vattr, cred); if (error) return (error); #endif uiop->uio_offset = np->n_size; } } if (uiop->uio_resid == 0) return 0; if (vn_rlimit_fsize(vp, uiop, td)) return (EFBIG); scred = smbfs_malloc_scred(); smb_makescred(scred, td, cred); error = smb_write(smp->sm_share, np->n_fid, uiop, scred); smbfs_free_scred(scred); SMBVDEBUG("after: ofs=%jd,resid=%zd\n", (intmax_t)uiop->uio_offset, uiop->uio_resid); if (!error) { if (uiop->uio_offset > np->n_size) { np->n_size = uiop->uio_offset; vnode_pager_setsize(vp, np->n_size); } } return error; } /* * Do an I/O operation to/from a cache block. */ int smbfs_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td) { struct smbmount *smp = VFSTOSMBFS(vp->v_mount); struct smbnode *np = VTOSMB(vp); struct uio *uiop; struct iovec io; struct smb_cred *scred; int error = 0; uiop = malloc(sizeof(struct uio), M_SMBFSDATA, M_WAITOK); uiop->uio_iov = &io; uiop->uio_iovcnt = 1; uiop->uio_segflg = UIO_SYSSPACE; uiop->uio_td = td; scred = smbfs_malloc_scred(); smb_makescred(scred, td, cr); if (bp->b_iocmd == BIO_READ) { io.iov_len = uiop->uio_resid = bp->b_bcount; io.iov_base = bp->b_data; uiop->uio_rw = UIO_READ; switch (vp->v_type) { case VREG: uiop->uio_offset = ((off_t)bp->b_blkno) * DEV_BSIZE; error = smb_read(smp->sm_share, np->n_fid, uiop, scred); if (error) break; if (uiop->uio_resid) { int left = uiop->uio_resid; int nread = bp->b_bcount - left; if (left > 0) bzero((char *)bp->b_data + nread, left); } break; default: printf("smbfs_doio: type %x unexpected\n",vp->v_type); break; } if (error) { bp->b_error = error; bp->b_ioflags |= BIO_ERROR; } } else { /* write */ if (((bp->b_blkno * DEV_BSIZE) + bp->b_dirtyend) > np->n_size) bp->b_dirtyend = np->n_size - (bp->b_blkno * DEV_BSIZE); if (bp->b_dirtyend > bp->b_dirtyoff) { io.iov_len = uiop->uio_resid = bp->b_dirtyend - bp->b_dirtyoff; uiop->uio_offset = ((off_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff; io.iov_base = (char *)bp->b_data + bp->b_dirtyoff; uiop->uio_rw = UIO_WRITE; error = smb_write(smp->sm_share, np->n_fid, uiop, scred); /* * For an interrupted write, the buffer is still valid * and the write hasn't been pushed to the server yet, * so we can't set BIO_ERROR and report the interruption * by setting B_EINTR. For the B_ASYNC case, B_EINTR * is not relevant, so the rpc attempt is essentially * a noop. For the case of a V3 write rpc not being * committed to stable storage, the block is still * dirty and requires either a commit rpc or another * write rpc with iomode == NFSV3WRITE_FILESYNC before * the block is reused. This is indicated by setting * the B_DELWRI and B_NEEDCOMMIT flags. */ if (error == EINTR || (!error && (bp->b_flags & B_NEEDCOMMIT))) { int s; s = splbio(); bp->b_flags &= ~(B_INVAL|B_NOCACHE); if ((bp->b_flags & B_ASYNC) == 0) bp->b_flags |= B_EINTR; if ((bp->b_flags & B_PAGING) == 0) { bdirty(bp); bp->b_flags &= ~B_DONE; } if ((bp->b_flags & B_ASYNC) == 0) bp->b_flags |= B_EINTR; splx(s); } else { if (error) { bp->b_ioflags |= BIO_ERROR; bp->b_error = error; } bp->b_dirtyoff = bp->b_dirtyend = 0; } } else { bp->b_resid = 0; bufdone(bp); free(uiop, M_SMBFSDATA); smbfs_free_scred(scred); return 0; } } bp->b_resid = uiop->uio_resid; bufdone(bp); free(uiop, M_SMBFSDATA); smbfs_free_scred(scred); return error; } /* * Vnode op for VM getpages. * Wish wish .... get rid from multiple IO routines */ int smbfs_getpages(ap) struct vop_getpages_args /* { struct vnode *a_vp; vm_page_t *a_m; int a_count; int a_reqpage; } */ *ap; { #ifdef SMBFS_RWGENERIC return vop_stdgetpages(ap); #else int i, error, nextoff, size, toff, npages, count; struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; struct vnode *vp; struct thread *td; struct ucred *cred; struct smbmount *smp; struct smbnode *np; struct smb_cred *scred; vm_object_t object; vm_page_t *pages; vp = ap->a_vp; if ((object = vp->v_object) == NULL) { printf("smbfs_getpages: called with non-merged cache vnode??\n"); return VM_PAGER_ERROR; } td = curthread; /* XXX */ cred = td->td_ucred; /* XXX */ np = VTOSMB(vp); smp = VFSTOSMBFS(vp->v_mount); pages = ap->a_m; npages = ap->a_count; /* * If the requested page is partially valid, just return it and * allow the pager to zero-out the blanks. Partially valid pages * can only occur at the file EOF. * * XXXGL: is that true for SMB filesystem? */ VM_OBJECT_WLOCK(object); if (pages[npages - 1]->valid != 0 && --npages == 0) goto out; VM_OBJECT_WUNLOCK(object); scred = smbfs_malloc_scred(); smb_makescred(scred, td, cred); bp = getpbuf(&smbfs_pbuf_freecnt); kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); VM_CNT_INC(v_vnodein); VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = IDX_TO_OFF(pages[0]->pindex); uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; uio.uio_td = td; error = smb_read(smp->sm_share, np->n_fid, &uio, scred); smbfs_free_scred(scred); pmap_qremove(kva, npages); relpbuf(bp, &smbfs_pbuf_freecnt); if (error && (uio.uio_resid == count)) { printf("smbfs_getpages: error %d\n",error); return VM_PAGER_ERROR; } size = count - uio.uio_resid; VM_OBJECT_WLOCK(object); for (i = 0, toff = 0; i < npages; i++, toff = nextoff) { vm_page_t m; nextoff = toff + PAGE_SIZE; m = pages[i]; if (nextoff <= size) { /* * Read operation filled an entire page */ m->valid = VM_PAGE_BITS_ALL; KASSERT(m->dirty == 0, ("smbfs_getpages: page %p is dirty", m)); } else if (size > toff) { /* * Read operation filled a partial page. */ m->valid = 0; vm_page_set_valid_range(m, 0, size - toff); KASSERT(m->dirty == 0, ("smbfs_getpages: page %p is dirty", m)); } else { /* * Read operation was short. If no error occurred * we may have hit a zero-fill section. We simply * leave valid set to 0. */ ; } } out: VM_OBJECT_WUNLOCK(object); if (ap->a_rbehind) *ap->a_rbehind = 0; if (ap->a_rahead) *ap->a_rahead = 0; return (VM_PAGER_OK); #endif /* SMBFS_RWGENERIC */ } /* * Vnode op for VM putpages. * possible bug: all IO done in sync mode * Note that vop_close always invalidate pages before close, so it's * not necessary to open vnode. */ int smbfs_putpages(ap) struct vop_putpages_args /* { struct vnode *a_vp; vm_page_t *a_m; int a_count; int a_sync; int *a_rtvals; } */ *ap; { int error; struct vnode *vp = ap->a_vp; struct thread *td; struct ucred *cred; #ifdef SMBFS_RWGENERIC td = curthread; /* XXX */ cred = td->td_ucred; /* XXX */ VOP_OPEN(vp, FWRITE, cred, td, NULL); error = vop_stdputpages(ap); VOP_CLOSE(vp, FWRITE, cred, td); return error; #else struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; int i, npages, count; int *rtvals; struct smbmount *smp; struct smbnode *np; struct smb_cred *scred; vm_page_t *pages; td = curthread; /* XXX */ cred = td->td_ucred; /* XXX */ /* VOP_OPEN(vp, FWRITE, cred, td, NULL);*/ np = VTOSMB(vp); smp = VFSTOSMBFS(vp->v_mount); pages = ap->a_m; count = ap->a_count; rtvals = ap->a_rtvals; npages = btoc(count); for (i = 0; i < npages; i++) { rtvals[i] = VM_PAGER_ERROR; } bp = getpbuf(&smbfs_pbuf_freecnt); kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); VM_CNT_INC(v_vnodeout); VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = (caddr_t) kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = IDX_TO_OFF(pages[0]->pindex); uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; SMBVDEBUG("ofs=%jd,resid=%zd\n", (intmax_t)uio.uio_offset, uio.uio_resid); scred = smbfs_malloc_scred(); smb_makescred(scred, td, cred); error = smb_write(smp->sm_share, np->n_fid, &uio, scred); smbfs_free_scred(scred); /* VOP_CLOSE(vp, FWRITE, cred, td);*/ SMBVDEBUG("paged write done: %d\n", error); pmap_qremove(kva, npages); relpbuf(bp, &smbfs_pbuf_freecnt); - if (!error) - vnode_pager_undirty_pages(pages, rtvals, count - uio.uio_resid); - return rtvals[0]; + if (error == 0) { + vnode_pager_undirty_pages(pages, rtvals, count - uio.uio_resid, + npages * PAGE_SIZE, npages * PAGE_SIZE); + } + return (rtvals[0]); #endif /* SMBFS_RWGENERIC */ } /* * Flush and invalidate all dirty buffers. If another process is already * doing the flush, just wait for completion. */ int smbfs_vinvalbuf(struct vnode *vp, struct thread *td) { struct smbnode *np = VTOSMB(vp); int error = 0; if (vp->v_iflag & VI_DOOMED) return 0; while (np->n_flag & NFLUSHINPROG) { np->n_flag |= NFLUSHWANT; error = tsleep(&np->n_flag, PRIBIO + 2, "smfsvinv", 2 * hz); error = smb_td_intr(td); if (error == EINTR) return EINTR; } np->n_flag |= NFLUSHINPROG; if (vp->v_bufobj.bo_object != NULL) { VM_OBJECT_WLOCK(vp->v_bufobj.bo_object); vm_object_page_clean(vp->v_bufobj.bo_object, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(vp->v_bufobj.bo_object); } error = vinvalbuf(vp, V_SAVE, PCATCH, 0); while (error) { if (error == ERESTART || error == EINTR) { np->n_flag &= ~NFLUSHINPROG; if (np->n_flag & NFLUSHWANT) { np->n_flag &= ~NFLUSHWANT; wakeup(&np->n_flag); } return EINTR; } error = vinvalbuf(vp, V_SAVE, PCATCH, 0); } np->n_flag &= ~(NMODIFIED | NFLUSHINPROG); if (np->n_flag & NFLUSHWANT) { np->n_flag &= ~NFLUSHWANT; wakeup(&np->n_flag); } return (error); } Index: projects/runtime-coverage/sys/net/if_lagg.c =================================================================== --- projects/runtime-coverage/sys/net/if_lagg.c (revision 321584) +++ projects/runtime-coverage/sys/net/if_lagg.c (revision 321585) @@ -1,2186 +1,2187 @@ /* $OpenBSD: if_trunk.c,v 1.30 2007/01/31 06:20:19 reyk Exp $ */ /* * Copyright (c) 2005, 2006 Reyk Floeter * Copyright (c) 2007 Andrew Thompson * Copyright (c) 2014, 2016 Marcelo Araujo * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_ratelimit.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(INET) || defined(INET6) #include #include #endif #ifdef INET #include #include #endif #ifdef INET6 #include #include #include #endif #include #include #include /* Special flags we should propagate to the lagg ports. */ static struct { int flag; int (*func)(struct ifnet *, int); } lagg_pflags[] = { {IFF_PROMISC, ifpromisc}, {IFF_ALLMULTI, if_allmulti}, {0, NULL} }; VNET_DEFINE(SLIST_HEAD(__trhead, lagg_softc), lagg_list); /* list of laggs */ #define V_lagg_list VNET(lagg_list) static VNET_DEFINE(struct mtx, lagg_list_mtx); #define V_lagg_list_mtx VNET(lagg_list_mtx) #define LAGG_LIST_LOCK_INIT(x) mtx_init(&V_lagg_list_mtx, \ "if_lagg list", NULL, MTX_DEF) #define LAGG_LIST_LOCK_DESTROY(x) mtx_destroy(&V_lagg_list_mtx) #define LAGG_LIST_LOCK(x) mtx_lock(&V_lagg_list_mtx) #define LAGG_LIST_UNLOCK(x) mtx_unlock(&V_lagg_list_mtx) eventhandler_tag lagg_detach_cookie = NULL; static int lagg_clone_create(struct if_clone *, int, caddr_t); static void lagg_clone_destroy(struct ifnet *); static VNET_DEFINE(struct if_clone *, lagg_cloner); #define V_lagg_cloner VNET(lagg_cloner) static const char laggname[] = "lagg"; static void lagg_capabilities(struct lagg_softc *); static int lagg_port_create(struct lagg_softc *, struct ifnet *); static int lagg_port_destroy(struct lagg_port *, int); static struct mbuf *lagg_input(struct ifnet *, struct mbuf *); static void lagg_linkstate(struct lagg_softc *); static void lagg_port_state(struct ifnet *, int); static int lagg_port_ioctl(struct ifnet *, u_long, caddr_t); static int lagg_port_output(struct ifnet *, struct mbuf *, const struct sockaddr *, struct route *); static void lagg_port_ifdetach(void *arg __unused, struct ifnet *); #ifdef LAGG_PORT_STACKING static int lagg_port_checkstacking(struct lagg_softc *); #endif static void lagg_port2req(struct lagg_port *, struct lagg_reqport *); static void lagg_init(void *); static void lagg_stop(struct lagg_softc *); static int lagg_ioctl(struct ifnet *, u_long, caddr_t); #ifdef RATELIMIT static int lagg_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, struct m_snd_tag **); #endif static int lagg_setmulti(struct lagg_port *); static int lagg_clrmulti(struct lagg_port *); static int lagg_setcaps(struct lagg_port *, int cap); static int lagg_setflag(struct lagg_port *, int, int, int (*func)(struct ifnet *, int)); static int lagg_setflags(struct lagg_port *, int status); static uint64_t lagg_get_counter(struct ifnet *ifp, ift_counter cnt); static int lagg_transmit(struct ifnet *, struct mbuf *); static void lagg_qflush(struct ifnet *); static int lagg_media_change(struct ifnet *); static void lagg_media_status(struct ifnet *, struct ifmediareq *); static struct lagg_port *lagg_link_active(struct lagg_softc *, struct lagg_port *); /* Simple round robin */ static void lagg_rr_attach(struct lagg_softc *); static int lagg_rr_start(struct lagg_softc *, struct mbuf *); static struct mbuf *lagg_rr_input(struct lagg_softc *, struct lagg_port *, struct mbuf *); /* Active failover */ static int lagg_fail_start(struct lagg_softc *, struct mbuf *); static struct mbuf *lagg_fail_input(struct lagg_softc *, struct lagg_port *, struct mbuf *); /* Loadbalancing */ static void lagg_lb_attach(struct lagg_softc *); static void lagg_lb_detach(struct lagg_softc *); static int lagg_lb_port_create(struct lagg_port *); static void lagg_lb_port_destroy(struct lagg_port *); static int lagg_lb_start(struct lagg_softc *, struct mbuf *); static struct mbuf *lagg_lb_input(struct lagg_softc *, struct lagg_port *, struct mbuf *); static int lagg_lb_porttable(struct lagg_softc *, struct lagg_port *); /* Broadcast */ static int lagg_bcast_start(struct lagg_softc *, struct mbuf *); static struct mbuf *lagg_bcast_input(struct lagg_softc *, struct lagg_port *, struct mbuf *); /* 802.3ad LACP */ static void lagg_lacp_attach(struct lagg_softc *); static void lagg_lacp_detach(struct lagg_softc *); static int lagg_lacp_start(struct lagg_softc *, struct mbuf *); static struct mbuf *lagg_lacp_input(struct lagg_softc *, struct lagg_port *, struct mbuf *); static void lagg_lacp_lladdr(struct lagg_softc *); /* lagg protocol table */ static const struct lagg_proto { lagg_proto pr_num; void (*pr_attach)(struct lagg_softc *); void (*pr_detach)(struct lagg_softc *); int (*pr_start)(struct lagg_softc *, struct mbuf *); struct mbuf * (*pr_input)(struct lagg_softc *, struct lagg_port *, struct mbuf *); int (*pr_addport)(struct lagg_port *); void (*pr_delport)(struct lagg_port *); void (*pr_linkstate)(struct lagg_port *); void (*pr_init)(struct lagg_softc *); void (*pr_stop)(struct lagg_softc *); void (*pr_lladdr)(struct lagg_softc *); void (*pr_request)(struct lagg_softc *, void *); void (*pr_portreq)(struct lagg_port *, void *); } lagg_protos[] = { { .pr_num = LAGG_PROTO_NONE }, { .pr_num = LAGG_PROTO_ROUNDROBIN, .pr_attach = lagg_rr_attach, .pr_start = lagg_rr_start, .pr_input = lagg_rr_input, }, { .pr_num = LAGG_PROTO_FAILOVER, .pr_start = lagg_fail_start, .pr_input = lagg_fail_input, }, { .pr_num = LAGG_PROTO_LOADBALANCE, .pr_attach = lagg_lb_attach, .pr_detach = lagg_lb_detach, .pr_start = lagg_lb_start, .pr_input = lagg_lb_input, .pr_addport = lagg_lb_port_create, .pr_delport = lagg_lb_port_destroy, }, { .pr_num = LAGG_PROTO_LACP, .pr_attach = lagg_lacp_attach, .pr_detach = lagg_lacp_detach, .pr_start = lagg_lacp_start, .pr_input = lagg_lacp_input, .pr_addport = lacp_port_create, .pr_delport = lacp_port_destroy, .pr_linkstate = lacp_linkstate, .pr_init = lacp_init, .pr_stop = lacp_stop, .pr_lladdr = lagg_lacp_lladdr, .pr_request = lacp_req, .pr_portreq = lacp_portreq, }, { .pr_num = LAGG_PROTO_BROADCAST, .pr_start = lagg_bcast_start, .pr_input = lagg_bcast_input, }, }; SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, OID_AUTO, lagg, CTLFLAG_RW, 0, "Link Aggregation"); /* Allow input on any failover links */ static VNET_DEFINE(int, lagg_failover_rx_all); #define V_lagg_failover_rx_all VNET(lagg_failover_rx_all) SYSCTL_INT(_net_link_lagg, OID_AUTO, failover_rx_all, CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(lagg_failover_rx_all), 0, "Accept input from any interface in a failover lagg"); /* Default value for using flowid */ static VNET_DEFINE(int, def_use_flowid) = 1; #define V_def_use_flowid VNET(def_use_flowid) SYSCTL_INT(_net_link_lagg, OID_AUTO, default_use_flowid, CTLFLAG_RWTUN, &VNET_NAME(def_use_flowid), 0, "Default setting for using flow id for load sharing"); /* Default value for flowid shift */ static VNET_DEFINE(int, def_flowid_shift) = 16; #define V_def_flowid_shift VNET(def_flowid_shift) SYSCTL_INT(_net_link_lagg, OID_AUTO, default_flowid_shift, CTLFLAG_RWTUN, &VNET_NAME(def_flowid_shift), 0, "Default setting for flowid shift for load sharing"); static void vnet_lagg_init(const void *unused __unused) { LAGG_LIST_LOCK_INIT(); SLIST_INIT(&V_lagg_list); V_lagg_cloner = if_clone_simple(laggname, lagg_clone_create, lagg_clone_destroy, 0); } VNET_SYSINIT(vnet_lagg_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, vnet_lagg_init, NULL); static void vnet_lagg_uninit(const void *unused __unused) { if_clone_detach(V_lagg_cloner); LAGG_LIST_LOCK_DESTROY(); } VNET_SYSUNINIT(vnet_lagg_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, vnet_lagg_uninit, NULL); static int lagg_modevent(module_t mod, int type, void *data) { switch (type) { case MOD_LOAD: lagg_input_p = lagg_input; lagg_linkstate_p = lagg_port_state; lagg_detach_cookie = EVENTHANDLER_REGISTER( ifnet_departure_event, lagg_port_ifdetach, NULL, EVENTHANDLER_PRI_ANY); break; case MOD_UNLOAD: EVENTHANDLER_DEREGISTER(ifnet_departure_event, lagg_detach_cookie); lagg_input_p = NULL; lagg_linkstate_p = NULL; break; default: return (EOPNOTSUPP); } return (0); } static moduledata_t lagg_mod = { "if_lagg", lagg_modevent, 0 }; DECLARE_MODULE(if_lagg, lagg_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); MODULE_VERSION(if_lagg, 1); static void lagg_proto_attach(struct lagg_softc *sc, lagg_proto pr) { LAGG_XLOCK_ASSERT(sc); KASSERT(sc->sc_proto == LAGG_PROTO_NONE, ("%s: sc %p has proto", __func__, sc)); if (sc->sc_ifflags & IFF_DEBUG) if_printf(sc->sc_ifp, "using proto %u\n", pr); if (lagg_protos[pr].pr_attach != NULL) lagg_protos[pr].pr_attach(sc); sc->sc_proto = pr; } static void lagg_proto_detach(struct lagg_softc *sc) { lagg_proto pr; LAGG_XLOCK_ASSERT(sc); LAGG_WLOCK_ASSERT(sc); pr = sc->sc_proto; sc->sc_proto = LAGG_PROTO_NONE; if (lagg_protos[pr].pr_detach != NULL) lagg_protos[pr].pr_detach(sc); else LAGG_WUNLOCK(sc); } static int lagg_proto_start(struct lagg_softc *sc, struct mbuf *m) { return (lagg_protos[sc->sc_proto].pr_start(sc, m)); } static struct mbuf * lagg_proto_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { return (lagg_protos[sc->sc_proto].pr_input(sc, lp, m)); } static int lagg_proto_addport(struct lagg_softc *sc, struct lagg_port *lp) { if (lagg_protos[sc->sc_proto].pr_addport == NULL) return (0); else return (lagg_protos[sc->sc_proto].pr_addport(lp)); } static void lagg_proto_delport(struct lagg_softc *sc, struct lagg_port *lp) { if (lagg_protos[sc->sc_proto].pr_delport != NULL) lagg_protos[sc->sc_proto].pr_delport(lp); } static void lagg_proto_linkstate(struct lagg_softc *sc, struct lagg_port *lp) { if (lagg_protos[sc->sc_proto].pr_linkstate != NULL) lagg_protos[sc->sc_proto].pr_linkstate(lp); } static void lagg_proto_init(struct lagg_softc *sc) { if (lagg_protos[sc->sc_proto].pr_init != NULL) lagg_protos[sc->sc_proto].pr_init(sc); } static void lagg_proto_stop(struct lagg_softc *sc) { if (lagg_protos[sc->sc_proto].pr_stop != NULL) lagg_protos[sc->sc_proto].pr_stop(sc); } static void lagg_proto_lladdr(struct lagg_softc *sc) { if (lagg_protos[sc->sc_proto].pr_lladdr != NULL) lagg_protos[sc->sc_proto].pr_lladdr(sc); } static void lagg_proto_request(struct lagg_softc *sc, void *v) { if (lagg_protos[sc->sc_proto].pr_request != NULL) lagg_protos[sc->sc_proto].pr_request(sc, v); } static void lagg_proto_portreq(struct lagg_softc *sc, struct lagg_port *lp, void *v) { if (lagg_protos[sc->sc_proto].pr_portreq != NULL) lagg_protos[sc->sc_proto].pr_portreq(lp, v); } /* * This routine is run via an vlan * config EVENT */ static void lagg_register_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) { struct lagg_softc *sc = ifp->if_softc; struct lagg_port *lp; if (ifp->if_softc != arg) /* Not our event */ return; LAGG_SLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) EVENTHANDLER_INVOKE(vlan_config, lp->lp_ifp, vtag); LAGG_SUNLOCK(sc); } /* * This routine is run via an vlan * unconfig EVENT */ static void lagg_unregister_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) { struct lagg_softc *sc = ifp->if_softc; struct lagg_port *lp; if (ifp->if_softc != arg) /* Not our event */ return; LAGG_SLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) EVENTHANDLER_INVOKE(vlan_unconfig, lp->lp_ifp, vtag); LAGG_SUNLOCK(sc); } static int lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params) { struct lagg_softc *sc; struct ifnet *ifp; static const u_char eaddr[6]; /* 00:00:00:00:00:00 */ sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_ZERO); ifp = sc->sc_ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { free(sc, M_DEVBUF); return (ENOSPC); } LAGG_LOCK_INIT(sc); LAGG_SX_INIT(sc); LAGG_XLOCK(sc); if (V_def_use_flowid) sc->sc_opts |= LAGG_OPT_USE_FLOWID; sc->flowid_shift = V_def_flowid_shift; /* Hash all layers by default */ sc->sc_flags = MBUF_HASHFLAG_L2|MBUF_HASHFLAG_L3|MBUF_HASHFLAG_L4; lagg_proto_attach(sc, LAGG_PROTO_DEFAULT); SLIST_INIT(&sc->sc_ports); /* Initialise pseudo media types */ ifmedia_init(&sc->sc_media, 0, lagg_media_change, lagg_media_status); ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); if_initname(ifp, laggname, unit); ifp->if_softc = sc; ifp->if_transmit = lagg_transmit; ifp->if_qflush = lagg_qflush; ifp->if_init = lagg_init; ifp->if_ioctl = lagg_ioctl; ifp->if_get_counter = lagg_get_counter; ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST; #ifdef RATELIMIT ifp->if_snd_tag_alloc = lagg_snd_tag_alloc; ifp->if_capenable = ifp->if_capabilities = IFCAP_HWSTATS | IFCAP_TXRTLMT; #else ifp->if_capenable = ifp->if_capabilities = IFCAP_HWSTATS; #endif /* * Attach as an ordinary ethernet device, children will be attached * as special device IFT_IEEE8023ADLAG. */ ether_ifattach(ifp, eaddr); sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, lagg_register_vlan, sc, EVENTHANDLER_PRI_FIRST); sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, lagg_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); /* Insert into the global list of laggs */ LAGG_LIST_LOCK(); SLIST_INSERT_HEAD(&V_lagg_list, sc, sc_entries); LAGG_LIST_UNLOCK(); LAGG_XUNLOCK(sc); return (0); } static void lagg_clone_destroy(struct ifnet *ifp) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; struct lagg_port *lp; LAGG_XLOCK(sc); sc->sc_destroying = 1; lagg_stop(sc); ifp->if_flags &= ~IFF_UP; EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); /* Shutdown and remove lagg ports */ while ((lp = SLIST_FIRST(&sc->sc_ports)) != NULL) lagg_port_destroy(lp, 1); /* Unhook the aggregation protocol */ LAGG_WLOCK(sc); lagg_proto_detach(sc); LAGG_UNLOCK_ASSERT(sc); LAGG_XUNLOCK(sc); ifmedia_removeall(&sc->sc_media); ether_ifdetach(ifp); if_free(ifp); LAGG_LIST_LOCK(); SLIST_REMOVE(&V_lagg_list, sc, lagg_softc, sc_entries); LAGG_LIST_UNLOCK(); LAGG_SX_DESTROY(sc); LAGG_LOCK_DESTROY(sc); free(sc, M_DEVBUF); } static void lagg_capabilities(struct lagg_softc *sc) { struct lagg_port *lp; int cap, ena, pena; uint64_t hwa; struct ifnet_hw_tsomax hw_tsomax; LAGG_XLOCK_ASSERT(sc); /* Get common enabled capabilities for the lagg ports */ ena = ~0; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) ena &= lp->lp_ifp->if_capenable; ena = (ena == ~0 ? 0 : ena); /* * Apply common enabled capabilities back to the lagg ports. * May require several iterations if they are dependent. */ do { pena = ena; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_setcaps(lp, ena); ena &= lp->lp_ifp->if_capenable; } } while (pena != ena); /* Get other capabilities from the lagg ports */ cap = ~0; hwa = ~(uint64_t)0; memset(&hw_tsomax, 0, sizeof(hw_tsomax)); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { cap &= lp->lp_ifp->if_capabilities; hwa &= lp->lp_ifp->if_hwassist; if_hw_tsomax_common(lp->lp_ifp, &hw_tsomax); } cap = (cap == ~0 ? 0 : cap); hwa = (hwa == ~(uint64_t)0 ? 0 : hwa); if (sc->sc_ifp->if_capabilities != cap || sc->sc_ifp->if_capenable != ena || sc->sc_ifp->if_hwassist != hwa || if_hw_tsomax_update(sc->sc_ifp, &hw_tsomax) != 0) { sc->sc_ifp->if_capabilities = cap; sc->sc_ifp->if_capenable = ena; sc->sc_ifp->if_hwassist = hwa; getmicrotime(&sc->sc_ifp->if_lastchange); if (sc->sc_ifflags & IFF_DEBUG) if_printf(sc->sc_ifp, "capabilities 0x%08x enabled 0x%08x\n", cap, ena); } } static int lagg_port_create(struct lagg_softc *sc, struct ifnet *ifp) { struct lagg_softc *sc_ptr; struct lagg_port *lp, *tlp; int error, i; uint64_t *pval; LAGG_XLOCK_ASSERT(sc); /* Limit the maximal number of lagg ports */ if (sc->sc_count >= LAGG_MAX_PORTS) return (ENOSPC); /* Check if port has already been associated to a lagg */ if (ifp->if_lagg != NULL) { /* Port is already in the current lagg? */ lp = (struct lagg_port *)ifp->if_lagg; if (lp->lp_softc == sc) return (EEXIST); return (EBUSY); } /* XXX Disallow non-ethernet interfaces (this should be any of 802) */ if (ifp->if_type != IFT_ETHER && ifp->if_type != IFT_L2VLAN) return (EPROTONOSUPPORT); /* Allow the first Ethernet member to define the MTU */ if (SLIST_EMPTY(&sc->sc_ports)) sc->sc_ifp->if_mtu = ifp->if_mtu; else if (sc->sc_ifp->if_mtu != ifp->if_mtu) { if_printf(sc->sc_ifp, "invalid MTU for %s\n", ifp->if_xname); return (EINVAL); } lp = malloc(sizeof(struct lagg_port), M_DEVBUF, M_WAITOK|M_ZERO); lp->lp_softc = sc; /* Check if port is a stacked lagg */ LAGG_LIST_LOCK(); SLIST_FOREACH(sc_ptr, &V_lagg_list, sc_entries) { if (ifp == sc_ptr->sc_ifp) { LAGG_LIST_UNLOCK(); free(lp, M_DEVBUF); return (EINVAL); /* XXX disable stacking for the moment, its untested */ #ifdef LAGG_PORT_STACKING lp->lp_flags |= LAGG_PORT_STACK; if (lagg_port_checkstacking(sc_ptr) >= LAGG_MAX_STACKING) { LAGG_LIST_UNLOCK(); free(lp, M_DEVBUF); return (E2BIG); } #endif } } LAGG_LIST_UNLOCK(); if_ref(ifp); lp->lp_ifp = ifp; bcopy(IF_LLADDR(ifp), lp->lp_lladdr, ETHER_ADDR_LEN); lp->lp_ifcapenable = ifp->if_capenable; if (SLIST_EMPTY(&sc->sc_ports)) { LAGG_WLOCK(sc); bcopy(IF_LLADDR(ifp), IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); lagg_proto_lladdr(sc); LAGG_WUNLOCK(sc); EVENTHANDLER_INVOKE(iflladdr_event, sc->sc_ifp); } else { if_setlladdr(ifp, IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); } lagg_setflags(lp, 1); LAGG_WLOCK(sc); if (SLIST_EMPTY(&sc->sc_ports)) sc->sc_primary = lp; /* Change the interface type */ lp->lp_iftype = ifp->if_type; ifp->if_type = IFT_IEEE8023ADLAG; ifp->if_lagg = lp; lp->lp_ioctl = ifp->if_ioctl; ifp->if_ioctl = lagg_port_ioctl; lp->lp_output = ifp->if_output; ifp->if_output = lagg_port_output; /* Read port counters */ pval = lp->port_counters.val; for (i = 0; i < IFCOUNTERS; i++, pval++) *pval = ifp->if_get_counter(ifp, i); /* * Insert into the list of ports. * Keep ports sorted by if_index. It is handy, when configuration * is predictable and `ifconfig laggN create ...` command * will lead to the same result each time. */ SLIST_FOREACH(tlp, &sc->sc_ports, lp_entries) { if (tlp->lp_ifp->if_index < ifp->if_index && ( SLIST_NEXT(tlp, lp_entries) == NULL || SLIST_NEXT(tlp, lp_entries)->lp_ifp->if_index > ifp->if_index)) break; } if (tlp != NULL) SLIST_INSERT_AFTER(tlp, lp, lp_entries); else SLIST_INSERT_HEAD(&sc->sc_ports, lp, lp_entries); sc->sc_count++; lagg_setmulti(lp); + LAGG_WUNLOCK(sc); + if ((error = lagg_proto_addport(sc, lp)) != 0) { /* Remove the port, without calling pr_delport. */ + LAGG_WLOCK(sc); lagg_port_destroy(lp, 0); LAGG_UNLOCK_ASSERT(sc); return (error); } - - LAGG_WUNLOCK(sc); /* Update lagg capabilities */ lagg_capabilities(sc); lagg_linkstate(sc); return (0); } #ifdef LAGG_PORT_STACKING static int lagg_port_checkstacking(struct lagg_softc *sc) { struct lagg_softc *sc_ptr; struct lagg_port *lp; int m = 0; LAGG_SXLOCK_ASSERT(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_flags & LAGG_PORT_STACK) { sc_ptr = (struct lagg_softc *)lp->lp_ifp->if_softc; m = MAX(m, lagg_port_checkstacking(sc_ptr)); } } return (m + 1); } #endif static int lagg_port_destroy(struct lagg_port *lp, int rundelport) { struct lagg_softc *sc = lp->lp_softc; struct lagg_port *lp_ptr, *lp0; struct ifnet *ifp = lp->lp_ifp; uint64_t *pval, vdiff; int i; LAGG_XLOCK_ASSERT(sc); if (rundelport) { LAGG_WLOCK(sc); lagg_proto_delport(sc, lp); } else LAGG_WLOCK_ASSERT(sc); if (lp->lp_detaching == 0) lagg_clrmulti(lp); /* Restore interface */ ifp->if_type = lp->lp_iftype; ifp->if_ioctl = lp->lp_ioctl; ifp->if_output = lp->lp_output; ifp->if_lagg = NULL; /* Update detached port counters */ pval = lp->port_counters.val; for (i = 0; i < IFCOUNTERS; i++, pval++) { vdiff = ifp->if_get_counter(ifp, i) - *pval; sc->detached_counters.val[i] += vdiff; } /* Finally, remove the port from the lagg */ SLIST_REMOVE(&sc->sc_ports, lp, lagg_port, lp_entries); sc->sc_count--; /* Update the primary interface */ if (lp == sc->sc_primary) { uint8_t lladdr[ETHER_ADDR_LEN]; if ((lp0 = SLIST_FIRST(&sc->sc_ports)) == NULL) bzero(&lladdr, ETHER_ADDR_LEN); else bcopy(lp0->lp_lladdr, lladdr, ETHER_ADDR_LEN); sc->sc_primary = lp0; if (sc->sc_destroying == 0) { bcopy(lladdr, IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); lagg_proto_lladdr(sc); LAGG_WUNLOCK(sc); EVENTHANDLER_INVOKE(iflladdr_event, sc->sc_ifp); } else LAGG_WUNLOCK(sc); /* * Update lladdr for each port (new primary needs update * as well, to switch from old lladdr to its 'real' one) */ SLIST_FOREACH(lp_ptr, &sc->sc_ports, lp_entries) if_setlladdr(lp_ptr->lp_ifp, lladdr, ETHER_ADDR_LEN); } else LAGG_WUNLOCK(sc); if (lp->lp_ifflags) if_printf(ifp, "%s: lp_ifflags unclean\n", __func__); if (lp->lp_detaching == 0) { lagg_setflags(lp, 0); lagg_setcaps(lp, lp->lp_ifcapenable); if_setlladdr(ifp, lp->lp_lladdr, ETHER_ADDR_LEN); } if_rele(ifp); free(lp, M_DEVBUF); /* Update lagg capabilities */ lagg_capabilities(sc); lagg_linkstate(sc); return (0); } static int lagg_port_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct lagg_reqport *rp = (struct lagg_reqport *)data; struct lagg_softc *sc; struct lagg_port *lp = NULL; int error = 0; /* Should be checked by the caller */ if (ifp->if_type != IFT_IEEE8023ADLAG || (lp = ifp->if_lagg) == NULL || (sc = lp->lp_softc) == NULL) goto fallback; switch (cmd) { case SIOCGLAGGPORT: if (rp->rp_portname[0] == '\0' || ifunit(rp->rp_portname) != ifp) { error = EINVAL; break; } LAGG_SLOCK(sc); if ((lp = ifp->if_lagg) == NULL || lp->lp_softc != sc) { error = ENOENT; LAGG_SUNLOCK(sc); break; } lagg_port2req(lp, rp); LAGG_SUNLOCK(sc); break; case SIOCSIFCAP: if (lp->lp_ioctl == NULL) { error = EINVAL; break; } error = (*lp->lp_ioctl)(ifp, cmd, data); if (error) break; /* Update lagg interface capabilities */ LAGG_XLOCK(sc); lagg_capabilities(sc); LAGG_XUNLOCK(sc); VLAN_CAPABILITIES(sc->sc_ifp); break; case SIOCSIFMTU: /* Do not allow the MTU to be changed once joined */ error = EINVAL; break; default: goto fallback; } return (error); fallback: if (lp != NULL && lp->lp_ioctl != NULL) return ((*lp->lp_ioctl)(ifp, cmd, data)); return (EINVAL); } /* * Requests counter @cnt data. * * Counter value is calculated the following way: * 1) for each port, sum difference between current and "initial" measurements. * 2) add lagg logical interface counters. * 3) add data from detached_counters array. * * We also do the following things on ports attach/detach: * 1) On port attach we store all counters it has into port_counter array. * 2) On port detach we add the different between "initial" and * current counters data to detached_counters array. */ static uint64_t lagg_get_counter(struct ifnet *ifp, ift_counter cnt) { struct lagg_softc *sc; struct lagg_port *lp; struct ifnet *lpifp; struct rm_priotracker tracker; uint64_t newval, oldval, vsum; /* Revise this when we've got non-generic counters. */ KASSERT(cnt < IFCOUNTERS, ("%s: invalid cnt %d", __func__, cnt)); sc = (struct lagg_softc *)ifp->if_softc; LAGG_RLOCK(sc, &tracker); vsum = 0; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { /* Saved attached value */ oldval = lp->port_counters.val[cnt]; /* current value */ lpifp = lp->lp_ifp; newval = lpifp->if_get_counter(lpifp, cnt); /* Calculate diff and save new */ vsum += newval - oldval; } /* * Add counter data which might be added by upper * layer protocols operating on logical interface. */ vsum += if_get_counter_default(ifp, cnt); /* * Add counter data from detached ports counters */ vsum += sc->detached_counters.val[cnt]; LAGG_RUNLOCK(sc, &tracker); return (vsum); } /* * For direct output to child ports. */ static int lagg_port_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro) { struct lagg_port *lp = ifp->if_lagg; switch (dst->sa_family) { case pseudo_AF_HDRCMPLT: case AF_UNSPEC: return ((*lp->lp_output)(ifp, m, dst, ro)); } /* drop any other frames */ m_freem(m); return (ENETDOWN); } static void lagg_port_ifdetach(void *arg __unused, struct ifnet *ifp) { struct lagg_port *lp; struct lagg_softc *sc; if ((lp = ifp->if_lagg) == NULL) return; /* If the ifnet is just being renamed, don't do anything. */ if (ifp->if_flags & IFF_RENAMING) return; sc = lp->lp_softc; LAGG_XLOCK(sc); lp->lp_detaching = 1; lagg_port_destroy(lp, 1); LAGG_XUNLOCK(sc); VLAN_CAPABILITIES(sc->sc_ifp); } static void lagg_port2req(struct lagg_port *lp, struct lagg_reqport *rp) { struct lagg_softc *sc = lp->lp_softc; strlcpy(rp->rp_ifname, sc->sc_ifname, sizeof(rp->rp_ifname)); strlcpy(rp->rp_portname, lp->lp_ifp->if_xname, sizeof(rp->rp_portname)); rp->rp_prio = lp->lp_prio; rp->rp_flags = lp->lp_flags; lagg_proto_portreq(sc, lp, &rp->rp_psc); /* Add protocol specific flags */ switch (sc->sc_proto) { case LAGG_PROTO_FAILOVER: if (lp == sc->sc_primary) rp->rp_flags |= LAGG_PORT_MASTER; if (lp == lagg_link_active(sc, sc->sc_primary)) rp->rp_flags |= LAGG_PORT_ACTIVE; break; case LAGG_PROTO_ROUNDROBIN: case LAGG_PROTO_LOADBALANCE: case LAGG_PROTO_BROADCAST: if (LAGG_PORTACTIVE(lp)) rp->rp_flags |= LAGG_PORT_ACTIVE; break; case LAGG_PROTO_LACP: /* LACP has a different definition of active */ if (lacp_isactive(lp)) rp->rp_flags |= LAGG_PORT_ACTIVE; if (lacp_iscollecting(lp)) rp->rp_flags |= LAGG_PORT_COLLECTING; if (lacp_isdistributing(lp)) rp->rp_flags |= LAGG_PORT_DISTRIBUTING; break; } } static void lagg_init(void *xsc) { struct lagg_softc *sc = (struct lagg_softc *)xsc; struct ifnet *ifp = sc->sc_ifp; struct lagg_port *lp; LAGG_XLOCK(sc); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { LAGG_XUNLOCK(sc); return; } ifp->if_drv_flags |= IFF_DRV_RUNNING; /* * Update the port lladdrs if needed. * This might be if_setlladdr() notification * that lladdr has been changed. */ SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (memcmp(IF_LLADDR(ifp), IF_LLADDR(lp->lp_ifp), ETHER_ADDR_LEN) != 0) if_setlladdr(lp->lp_ifp, IF_LLADDR(ifp), ETHER_ADDR_LEN); } lagg_proto_init(sc); LAGG_XUNLOCK(sc); } static void lagg_stop(struct lagg_softc *sc) { struct ifnet *ifp = sc->sc_ifp; LAGG_XLOCK_ASSERT(sc); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return; ifp->if_drv_flags &= ~IFF_DRV_RUNNING; lagg_proto_stop(sc); } static int lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; struct lagg_reqall *ra = (struct lagg_reqall *)data; struct lagg_reqopts *ro = (struct lagg_reqopts *)data; struct lagg_reqport *rp = (struct lagg_reqport *)data, rpbuf; struct lagg_reqflags *rf = (struct lagg_reqflags *)data; struct ifreq *ifr = (struct ifreq *)data; struct lagg_port *lp; struct ifnet *tpif; struct thread *td = curthread; char *buf, *outbuf; int count, buflen, len, error = 0; bzero(&rpbuf, sizeof(rpbuf)); switch (cmd) { case SIOCGLAGG: LAGG_SLOCK(sc); buflen = sc->sc_count * sizeof(struct lagg_reqport); outbuf = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO); ra->ra_proto = sc->sc_proto; lagg_proto_request(sc, &ra->ra_psc); count = 0; buf = outbuf; len = min(ra->ra_size, buflen); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (len < sizeof(rpbuf)) break; lagg_port2req(lp, &rpbuf); memcpy(buf, &rpbuf, sizeof(rpbuf)); count++; buf += sizeof(rpbuf); len -= sizeof(rpbuf); } LAGG_SUNLOCK(sc); ra->ra_ports = count; ra->ra_size = count * sizeof(rpbuf); error = copyout(outbuf, ra->ra_port, ra->ra_size); free(outbuf, M_TEMP); break; case SIOCSLAGG: error = priv_check(td, PRIV_NET_LAGG); if (error) break; if (ra->ra_proto >= LAGG_PROTO_MAX) { error = EPROTONOSUPPORT; break; } LAGG_XLOCK(sc); LAGG_WLOCK(sc); lagg_proto_detach(sc); LAGG_UNLOCK_ASSERT(sc); lagg_proto_attach(sc, ra->ra_proto); LAGG_XUNLOCK(sc); break; case SIOCGLAGGOPTS: LAGG_SLOCK(sc); ro->ro_opts = sc->sc_opts; if (sc->sc_proto == LAGG_PROTO_LACP) { struct lacp_softc *lsc; lsc = (struct lacp_softc *)sc->sc_psc; if (lsc->lsc_debug.lsc_tx_test != 0) ro->ro_opts |= LAGG_OPT_LACP_TXTEST; if (lsc->lsc_debug.lsc_rx_test != 0) ro->ro_opts |= LAGG_OPT_LACP_RXTEST; if (lsc->lsc_strict_mode != 0) ro->ro_opts |= LAGG_OPT_LACP_STRICT; if (lsc->lsc_fast_timeout != 0) ro->ro_opts |= LAGG_OPT_LACP_TIMEOUT; ro->ro_active = sc->sc_active; } else { ro->ro_active = 0; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) ro->ro_active += LAGG_PORTACTIVE(lp); } ro->ro_bkt = sc->sc_bkt; ro->ro_flapping = sc->sc_flapping; ro->ro_flowid_shift = sc->flowid_shift; LAGG_SUNLOCK(sc); break; case SIOCSLAGGOPTS: if (sc->sc_proto == LAGG_PROTO_ROUNDROBIN) { if (ro->ro_bkt == 0) sc->sc_bkt = 1; // Minimum 1 packet per iface. else sc->sc_bkt = ro->ro_bkt; } error = priv_check(td, PRIV_NET_LAGG); if (error) break; if (ro->ro_opts == 0) break; /* * Set options. LACP options are stored in sc->sc_psc, * not in sc_opts. */ int valid, lacp; switch (ro->ro_opts) { case LAGG_OPT_USE_FLOWID: case -LAGG_OPT_USE_FLOWID: case LAGG_OPT_FLOWIDSHIFT: valid = 1; lacp = 0; break; case LAGG_OPT_LACP_TXTEST: case -LAGG_OPT_LACP_TXTEST: case LAGG_OPT_LACP_RXTEST: case -LAGG_OPT_LACP_RXTEST: case LAGG_OPT_LACP_STRICT: case -LAGG_OPT_LACP_STRICT: case LAGG_OPT_LACP_TIMEOUT: case -LAGG_OPT_LACP_TIMEOUT: valid = lacp = 1; break; default: valid = lacp = 0; break; } LAGG_XLOCK(sc); if (valid == 0 || (lacp == 1 && sc->sc_proto != LAGG_PROTO_LACP)) { /* Invalid combination of options specified. */ error = EINVAL; LAGG_XUNLOCK(sc); break; /* Return from SIOCSLAGGOPTS. */ } /* * Store new options into sc->sc_opts except for * FLOWIDSHIFT and LACP options. */ if (lacp == 0) { if (ro->ro_opts == LAGG_OPT_FLOWIDSHIFT) sc->flowid_shift = ro->ro_flowid_shift; else if (ro->ro_opts > 0) sc->sc_opts |= ro->ro_opts; else sc->sc_opts &= ~ro->ro_opts; } else { struct lacp_softc *lsc; struct lacp_port *lp; lsc = (struct lacp_softc *)sc->sc_psc; switch (ro->ro_opts) { case LAGG_OPT_LACP_TXTEST: lsc->lsc_debug.lsc_tx_test = 1; break; case -LAGG_OPT_LACP_TXTEST: lsc->lsc_debug.lsc_tx_test = 0; break; case LAGG_OPT_LACP_RXTEST: lsc->lsc_debug.lsc_rx_test = 1; break; case -LAGG_OPT_LACP_RXTEST: lsc->lsc_debug.lsc_rx_test = 0; break; case LAGG_OPT_LACP_STRICT: lsc->lsc_strict_mode = 1; break; case -LAGG_OPT_LACP_STRICT: lsc->lsc_strict_mode = 0; break; case LAGG_OPT_LACP_TIMEOUT: LACP_LOCK(lsc); LIST_FOREACH(lp, &lsc->lsc_ports, lp_next) lp->lp_state |= LACP_STATE_TIMEOUT; LACP_UNLOCK(lsc); lsc->lsc_fast_timeout = 1; break; case -LAGG_OPT_LACP_TIMEOUT: LACP_LOCK(lsc); LIST_FOREACH(lp, &lsc->lsc_ports, lp_next) lp->lp_state &= ~LACP_STATE_TIMEOUT; LACP_UNLOCK(lsc); lsc->lsc_fast_timeout = 0; break; } } LAGG_XUNLOCK(sc); break; case SIOCGLAGGFLAGS: rf->rf_flags = 0; LAGG_SLOCK(sc); if (sc->sc_flags & MBUF_HASHFLAG_L2) rf->rf_flags |= LAGG_F_HASHL2; if (sc->sc_flags & MBUF_HASHFLAG_L3) rf->rf_flags |= LAGG_F_HASHL3; if (sc->sc_flags & MBUF_HASHFLAG_L4) rf->rf_flags |= LAGG_F_HASHL4; LAGG_SUNLOCK(sc); break; case SIOCSLAGGHASH: error = priv_check(td, PRIV_NET_LAGG); if (error) break; if ((rf->rf_flags & LAGG_F_HASHMASK) == 0) { error = EINVAL; break; } LAGG_XLOCK(sc); sc->sc_flags = 0; if (rf->rf_flags & LAGG_F_HASHL2) sc->sc_flags |= MBUF_HASHFLAG_L2; if (rf->rf_flags & LAGG_F_HASHL3) sc->sc_flags |= MBUF_HASHFLAG_L3; if (rf->rf_flags & LAGG_F_HASHL4) sc->sc_flags |= MBUF_HASHFLAG_L4; LAGG_XUNLOCK(sc); break; case SIOCGLAGGPORT: if (rp->rp_portname[0] == '\0' || (tpif = ifunit_ref(rp->rp_portname)) == NULL) { error = EINVAL; break; } LAGG_SLOCK(sc); if ((lp = (struct lagg_port *)tpif->if_lagg) == NULL || lp->lp_softc != sc) { error = ENOENT; LAGG_SUNLOCK(sc); if_rele(tpif); break; } lagg_port2req(lp, rp); LAGG_SUNLOCK(sc); if_rele(tpif); break; case SIOCSLAGGPORT: error = priv_check(td, PRIV_NET_LAGG); if (error) break; if (rp->rp_portname[0] == '\0' || (tpif = ifunit_ref(rp->rp_portname)) == NULL) { error = EINVAL; break; } #ifdef INET6 /* * A laggport interface should not have inet6 address * because two interfaces with a valid link-local * scope zone must not be merged in any form. This * restriction is needed to prevent violation of * link-local scope zone. Attempts to add a laggport * interface which has inet6 addresses triggers * removal of all inet6 addresses on the member * interface. */ if (in6ifa_llaonifp(tpif)) { in6_ifdetach(tpif); if_printf(sc->sc_ifp, "IPv6 addresses on %s have been removed " "before adding it as a member to prevent " "IPv6 address scope violation.\n", tpif->if_xname); } #endif LAGG_XLOCK(sc); error = lagg_port_create(sc, tpif); LAGG_XUNLOCK(sc); if_rele(tpif); VLAN_CAPABILITIES(ifp); break; case SIOCSLAGGDELPORT: error = priv_check(td, PRIV_NET_LAGG); if (error) break; if (rp->rp_portname[0] == '\0' || (tpif = ifunit_ref(rp->rp_portname)) == NULL) { error = EINVAL; break; } LAGG_XLOCK(sc); if ((lp = (struct lagg_port *)tpif->if_lagg) == NULL || lp->lp_softc != sc) { error = ENOENT; LAGG_XUNLOCK(sc); if_rele(tpif); break; } error = lagg_port_destroy(lp, 1); LAGG_XUNLOCK(sc); if_rele(tpif); VLAN_CAPABILITIES(ifp); break; case SIOCSIFFLAGS: /* Set flags on ports too */ LAGG_XLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_setflags(lp, 1); } if (!(ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { /* * If interface is marked down and it is running, * then stop and disable it. */ lagg_stop(sc); LAGG_XUNLOCK(sc); } else if ((ifp->if_flags & IFF_UP) && !(ifp->if_drv_flags & IFF_DRV_RUNNING)) { /* * If interface is marked up and it is stopped, then * start it. */ LAGG_XUNLOCK(sc); (*ifp->if_init)(sc); } else LAGG_XUNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: LAGG_WLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_clrmulti(lp); lagg_setmulti(lp); } LAGG_WUNLOCK(sc); error = 0; break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); break; case SIOCSIFCAP: LAGG_XLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_ioctl != NULL) (*lp->lp_ioctl)(lp->lp_ifp, cmd, data); } lagg_capabilities(sc); LAGG_XUNLOCK(sc); VLAN_CAPABILITIES(ifp); error = 0; break; case SIOCSIFMTU: /* Do not allow the MTU to be directly changed */ error = EINVAL; break; default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } #ifdef RATELIMIT static int lagg_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **ppmt) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; struct lagg_port *lp; struct lagg_lb *lb; uint32_t p; switch (sc->sc_proto) { case LAGG_PROTO_FAILOVER: lp = lagg_link_active(sc, sc->sc_primary); break; case LAGG_PROTO_LOADBALANCE: if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) == 0 || params->hdr.flowtype == M_HASHTYPE_NONE) return (EOPNOTSUPP); p = params->hdr.flowid >> sc->flowid_shift; p %= sc->sc_count; lb = (struct lagg_lb *)sc->sc_psc; lp = lb->lb_ports[p]; lp = lagg_link_active(sc, lp); break; case LAGG_PROTO_LACP: if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) == 0 || params->hdr.flowtype == M_HASHTYPE_NONE) return (EOPNOTSUPP); lp = lacp_select_tx_port_by_hash(sc, params->hdr.flowid); break; default: return (EOPNOTSUPP); } if (lp == NULL) return (EOPNOTSUPP); ifp = lp->lp_ifp; if (ifp == NULL || ifp->if_snd_tag_alloc == NULL || (ifp->if_capenable & IFCAP_TXRTLMT) == 0) return (EOPNOTSUPP); /* forward allocation request */ return (ifp->if_snd_tag_alloc(ifp, params, ppmt)); } #endif static int lagg_setmulti(struct lagg_port *lp) { struct lagg_softc *sc = lp->lp_softc; struct ifnet *ifp = lp->lp_ifp; struct ifnet *scifp = sc->sc_ifp; struct lagg_mc *mc; struct ifmultiaddr *ifma; int error; LAGG_WLOCK_ASSERT(sc); IF_ADDR_WLOCK(scifp); TAILQ_FOREACH(ifma, &scifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; mc = malloc(sizeof(struct lagg_mc), M_DEVBUF, M_NOWAIT); if (mc == NULL) { IF_ADDR_WUNLOCK(scifp); return (ENOMEM); } bcopy(ifma->ifma_addr, &mc->mc_addr, ifma->ifma_addr->sa_len); mc->mc_addr.sdl_index = ifp->if_index; mc->mc_ifma = NULL; SLIST_INSERT_HEAD(&lp->lp_mc_head, mc, mc_entries); } IF_ADDR_WUNLOCK(scifp); SLIST_FOREACH (mc, &lp->lp_mc_head, mc_entries) { error = if_addmulti(ifp, (struct sockaddr *)&mc->mc_addr, &mc->mc_ifma); if (error) return (error); } return (0); } static int lagg_clrmulti(struct lagg_port *lp) { struct lagg_mc *mc; LAGG_WLOCK_ASSERT(lp->lp_softc); while ((mc = SLIST_FIRST(&lp->lp_mc_head)) != NULL) { SLIST_REMOVE(&lp->lp_mc_head, mc, lagg_mc, mc_entries); if (mc->mc_ifma && lp->lp_detaching == 0) if_delmulti_ifma(mc->mc_ifma); free(mc, M_DEVBUF); } return (0); } static int lagg_setcaps(struct lagg_port *lp, int cap) { struct ifreq ifr; if (lp->lp_ifp->if_capenable == cap) return (0); if (lp->lp_ioctl == NULL) return (ENXIO); ifr.ifr_reqcap = cap; return ((*lp->lp_ioctl)(lp->lp_ifp, SIOCSIFCAP, (caddr_t)&ifr)); } /* Handle a ref counted flag that should be set on the lagg port as well */ static int lagg_setflag(struct lagg_port *lp, int flag, int status, int (*func)(struct ifnet *, int)) { struct lagg_softc *sc = lp->lp_softc; struct ifnet *scifp = sc->sc_ifp; struct ifnet *ifp = lp->lp_ifp; int error; LAGG_XLOCK_ASSERT(sc); status = status ? (scifp->if_flags & flag) : 0; /* Now "status" contains the flag value or 0 */ /* * See if recorded ports status is different from what * we want it to be. If it is, flip it. We record ports * status in lp_ifflags so that we won't clear ports flag * we haven't set. In fact, we don't clear or set ports * flags directly, but get or release references to them. * That's why we can be sure that recorded flags still are * in accord with actual ports flags. */ if (status != (lp->lp_ifflags & flag)) { error = (*func)(ifp, status); if (error) return (error); lp->lp_ifflags &= ~flag; lp->lp_ifflags |= status; } return (0); } /* * Handle IFF_* flags that require certain changes on the lagg port * if "status" is true, update ports flags respective to the lagg * if "status" is false, forcedly clear the flags set on port. */ static int lagg_setflags(struct lagg_port *lp, int status) { int error, i; for (i = 0; lagg_pflags[i].flag; i++) { error = lagg_setflag(lp, lagg_pflags[i].flag, status, lagg_pflags[i].func); if (error) return (error); } return (0); } static int lagg_transmit(struct ifnet *ifp, struct mbuf *m) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; int error, len, mcast; struct rm_priotracker tracker; len = m->m_pkthdr.len; mcast = (m->m_flags & (M_MCAST | M_BCAST)) ? 1 : 0; LAGG_RLOCK(sc, &tracker); /* We need a Tx algorithm and at least one port */ if (sc->sc_proto == LAGG_PROTO_NONE || sc->sc_count == 0) { LAGG_RUNLOCK(sc, &tracker); m_freem(m); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return (ENXIO); } ETHER_BPF_MTAP(ifp, m); error = lagg_proto_start(sc, m); LAGG_RUNLOCK(sc, &tracker); if (error != 0) if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return (error); } /* * The ifp->if_qflush entry point for lagg(4) is no-op. */ static void lagg_qflush(struct ifnet *ifp __unused) { } static struct mbuf * lagg_input(struct ifnet *ifp, struct mbuf *m) { struct lagg_port *lp = ifp->if_lagg; struct lagg_softc *sc = lp->lp_softc; struct ifnet *scifp = sc->sc_ifp; struct rm_priotracker tracker; LAGG_RLOCK(sc, &tracker); if ((scifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || (lp->lp_flags & LAGG_PORT_DISABLED) || sc->sc_proto == LAGG_PROTO_NONE) { LAGG_RUNLOCK(sc, &tracker); m_freem(m); return (NULL); } ETHER_BPF_MTAP(scifp, m); if (lp->lp_detaching != 0) { m_freem(m); m = NULL; } else m = lagg_proto_input(sc, lp, m); if (m != NULL) { if (scifp->if_flags & IFF_MONITOR) { m_freem(m); m = NULL; } } LAGG_RUNLOCK(sc, &tracker); return (m); } static int lagg_media_change(struct ifnet *ifp) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; if (sc->sc_ifflags & IFF_DEBUG) printf("%s\n", __func__); /* Ignore */ return (0); } static void lagg_media_status(struct ifnet *ifp, struct ifmediareq *imr) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; struct lagg_port *lp; imr->ifm_status = IFM_AVALID; imr->ifm_active = IFM_ETHER | IFM_AUTO; LAGG_SLOCK(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (LAGG_PORTACTIVE(lp)) imr->ifm_status |= IFM_ACTIVE; } LAGG_SUNLOCK(sc); } static void lagg_linkstate(struct lagg_softc *sc) { struct lagg_port *lp; int new_link = LINK_STATE_DOWN; uint64_t speed; LAGG_XLOCK_ASSERT(sc); /* Our link is considered up if at least one of our ports is active */ SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_ifp->if_link_state == LINK_STATE_UP) { new_link = LINK_STATE_UP; break; } } if_link_state_change(sc->sc_ifp, new_link); /* Update if_baudrate to reflect the max possible speed */ switch (sc->sc_proto) { case LAGG_PROTO_FAILOVER: sc->sc_ifp->if_baudrate = sc->sc_primary != NULL ? sc->sc_primary->lp_ifp->if_baudrate : 0; break; case LAGG_PROTO_ROUNDROBIN: case LAGG_PROTO_LOADBALANCE: case LAGG_PROTO_BROADCAST: speed = 0; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) speed += lp->lp_ifp->if_baudrate; sc->sc_ifp->if_baudrate = speed; break; case LAGG_PROTO_LACP: /* LACP updates if_baudrate itself */ break; } } static void lagg_port_state(struct ifnet *ifp, int state) { struct lagg_port *lp = (struct lagg_port *)ifp->if_lagg; struct lagg_softc *sc = NULL; if (lp != NULL) sc = lp->lp_softc; if (sc == NULL) return; LAGG_XLOCK(sc); lagg_linkstate(sc); lagg_proto_linkstate(sc, lp); LAGG_XUNLOCK(sc); } struct lagg_port * lagg_link_active(struct lagg_softc *sc, struct lagg_port *lp) { struct lagg_port *lp_next, *rval = NULL; /* * Search a port which reports an active link state. */ if (lp == NULL) goto search; if (LAGG_PORTACTIVE(lp)) { rval = lp; goto found; } if ((lp_next = SLIST_NEXT(lp, lp_entries)) != NULL && LAGG_PORTACTIVE(lp_next)) { rval = lp_next; goto found; } search: SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { if (LAGG_PORTACTIVE(lp_next)) { rval = lp_next; goto found; } } found: return (rval); } int lagg_enqueue(struct ifnet *ifp, struct mbuf *m) { return (ifp->if_transmit)(ifp, m); } /* * Simple round robin aggregation */ static void lagg_rr_attach(struct lagg_softc *sc) { sc->sc_seq = 0; sc->sc_bkt_count = sc->sc_bkt; } static int lagg_rr_start(struct lagg_softc *sc, struct mbuf *m) { struct lagg_port *lp; uint32_t p; if (sc->sc_bkt_count == 0 && sc->sc_bkt > 0) sc->sc_bkt_count = sc->sc_bkt; if (sc->sc_bkt > 0) { atomic_subtract_int(&sc->sc_bkt_count, 1); if (atomic_cmpset_int(&sc->sc_bkt_count, 0, sc->sc_bkt)) p = atomic_fetchadd_32(&sc->sc_seq, 1); else p = sc->sc_seq; } else p = atomic_fetchadd_32(&sc->sc_seq, 1); p %= sc->sc_count; lp = SLIST_FIRST(&sc->sc_ports); while (p--) lp = SLIST_NEXT(lp, lp_entries); /* * Check the port's link state. This will return the next active * port if the link is down or the port is NULL. */ if ((lp = lagg_link_active(sc, lp)) == NULL) { m_freem(m); return (ENETDOWN); } /* Send mbuf */ return (lagg_enqueue(lp->lp_ifp, m)); } static struct mbuf * lagg_rr_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { struct ifnet *ifp = sc->sc_ifp; /* Just pass in the packet to our lagg device */ m->m_pkthdr.rcvif = ifp; return (m); } /* * Broadcast mode */ static int lagg_bcast_start(struct lagg_softc *sc, struct mbuf *m) { int active_ports = 0; int errors = 0; int ret; struct lagg_port *lp, *last = NULL; struct mbuf *m0; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (!LAGG_PORTACTIVE(lp)) continue; active_ports++; if (last != NULL) { m0 = m_copym(m, 0, M_COPYALL, M_NOWAIT); if (m0 == NULL) { ret = ENOBUFS; errors++; break; } ret = lagg_enqueue(last->lp_ifp, m0); if (ret != 0) errors++; } last = lp; } if (last == NULL) { m_freem(m); return (ENOENT); } if ((last = lagg_link_active(sc, last)) == NULL) { m_freem(m); return (ENETDOWN); } ret = lagg_enqueue(last->lp_ifp, m); if (ret != 0) errors++; if (errors == 0) return (ret); return (0); } static struct mbuf* lagg_bcast_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { struct ifnet *ifp = sc->sc_ifp; /* Just pass in the packet to our lagg device */ m->m_pkthdr.rcvif = ifp; return (m); } /* * Active failover */ static int lagg_fail_start(struct lagg_softc *sc, struct mbuf *m) { struct lagg_port *lp; /* Use the master port if active or the next available port */ if ((lp = lagg_link_active(sc, sc->sc_primary)) == NULL) { m_freem(m); return (ENETDOWN); } /* Send mbuf */ return (lagg_enqueue(lp->lp_ifp, m)); } static struct mbuf * lagg_fail_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { struct ifnet *ifp = sc->sc_ifp; struct lagg_port *tmp_tp; if (lp == sc->sc_primary || V_lagg_failover_rx_all) { m->m_pkthdr.rcvif = ifp; return (m); } if (!LAGG_PORTACTIVE(sc->sc_primary)) { tmp_tp = lagg_link_active(sc, sc->sc_primary); /* * If tmp_tp is null, we've received a packet when all * our links are down. Weird, but process it anyways. */ if ((tmp_tp == NULL || tmp_tp == lp)) { m->m_pkthdr.rcvif = ifp; return (m); } } m_freem(m); return (NULL); } /* * Loadbalancing */ static void lagg_lb_attach(struct lagg_softc *sc) { struct lagg_port *lp; struct lagg_lb *lb; lb = malloc(sizeof(struct lagg_lb), M_DEVBUF, M_WAITOK | M_ZERO); lb->lb_key = m_ether_tcpip_hash_init(); sc->sc_psc = lb; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lagg_lb_port_create(lp); } static void lagg_lb_detach(struct lagg_softc *sc) { struct lagg_lb *lb; lb = (struct lagg_lb *)sc->sc_psc; LAGG_WUNLOCK(sc); if (lb != NULL) free(lb, M_DEVBUF); } static int lagg_lb_porttable(struct lagg_softc *sc, struct lagg_port *lp) { struct lagg_lb *lb = (struct lagg_lb *)sc->sc_psc; struct lagg_port *lp_next; int i = 0; bzero(&lb->lb_ports, sizeof(lb->lb_ports)); SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { if (lp_next == lp) continue; if (i >= LAGG_MAX_PORTS) return (EINVAL); if (sc->sc_ifflags & IFF_DEBUG) printf("%s: port %s at index %d\n", sc->sc_ifname, lp_next->lp_ifp->if_xname, i); lb->lb_ports[i++] = lp_next; } return (0); } static int lagg_lb_port_create(struct lagg_port *lp) { struct lagg_softc *sc = lp->lp_softc; return (lagg_lb_porttable(sc, NULL)); } static void lagg_lb_port_destroy(struct lagg_port *lp) { struct lagg_softc *sc = lp->lp_softc; lagg_lb_porttable(sc, lp); } static int lagg_lb_start(struct lagg_softc *sc, struct mbuf *m) { struct lagg_lb *lb = (struct lagg_lb *)sc->sc_psc; struct lagg_port *lp = NULL; uint32_t p = 0; if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) p = m->m_pkthdr.flowid >> sc->flowid_shift; else p = m_ether_tcpip_hash(sc->sc_flags, m, lb->lb_key); p %= sc->sc_count; lp = lb->lb_ports[p]; /* * Check the port's link state. This will return the next active * port if the link is down or the port is NULL. */ if ((lp = lagg_link_active(sc, lp)) == NULL) { m_freem(m); return (ENETDOWN); } /* Send mbuf */ return (lagg_enqueue(lp->lp_ifp, m)); } static struct mbuf * lagg_lb_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { struct ifnet *ifp = sc->sc_ifp; /* Just pass in the packet to our lagg device */ m->m_pkthdr.rcvif = ifp; return (m); } /* * 802.3ad LACP */ static void lagg_lacp_attach(struct lagg_softc *sc) { struct lagg_port *lp; lacp_attach(sc); SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_create(lp); } static void lagg_lacp_detach(struct lagg_softc *sc) { struct lagg_port *lp; void *psc; SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_destroy(lp); psc = sc->sc_psc; sc->sc_psc = NULL; LAGG_WUNLOCK(sc); lacp_detach(psc); } static void lagg_lacp_lladdr(struct lagg_softc *sc) { struct lagg_port *lp; LAGG_SXLOCK_ASSERT(sc); /* purge all the lacp ports */ SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_destroy(lp); /* add them back in */ SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_create(lp); } static int lagg_lacp_start(struct lagg_softc *sc, struct mbuf *m) { struct lagg_port *lp; lp = lacp_select_tx_port(sc, m); if (lp == NULL) { m_freem(m); return (ENETDOWN); } /* Send mbuf */ return (lagg_enqueue(lp->lp_ifp, m)); } static struct mbuf * lagg_lacp_input(struct lagg_softc *sc, struct lagg_port *lp, struct mbuf *m) { struct ifnet *ifp = sc->sc_ifp; struct ether_header *eh; u_short etype; eh = mtod(m, struct ether_header *); etype = ntohs(eh->ether_type); /* Tap off LACP control messages */ if ((m->m_flags & M_VLANTAG) == 0 && etype == ETHERTYPE_SLOW) { m = lacp_input(lp, m); if (m == NULL) return (NULL); } /* * If the port is not collecting or not in the active aggregator then * free and return. */ if (lacp_iscollecting(lp) == 0 || lacp_isactive(lp) == 0) { m_freem(m); return (NULL); } m->m_pkthdr.rcvif = ifp; return (m); } Index: projects/runtime-coverage/sys/netinet/cc/cc_cdg.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_cdg.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_cdg.c (revision 321585) @@ -1,710 +1,699 @@ /*- * Copyright (c) 2009-2013 * Swinburne University of Technology, Melbourne, Australia * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes, made * possible in part by a gift from The Cisco University Research Program Fund, * a corporate advised fund of Silicon Valley Community Foundation. Development * and testing were further assisted by a grant from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * CAIA Delay-Gradient (CDG) congestion control algorithm * * An implemention of the delay-gradient congestion control algorithm proposed * in the following paper: * * D. A. Hayes and G. Armitage, "Revisiting TCP Congestion Control using Delay * Gradients", in IFIP Networking, Valencia, Spain, 9-13 May 2011. * * Developed as part of the NewTCP research project at Swinburne University of * Technology's Centre for Advanced Internet Architectures, Melbourne, * Australia. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CDG_VERSION "0.1" #define CAST_PTR_INT(X) (*((int*)(X))) /* Private delay-gradient induced congestion control signal. */ #define CC_CDG_DELAY 0x01000000 /* NewReno window deflation factor on loss (as a percentage). */ #define RENO_BETA 50 /* Queue states. */ #define CDG_Q_EMPTY 1 #define CDG_Q_RISING 2 #define CDG_Q_FALLING 3 #define CDG_Q_FULL 4 #define CDG_Q_UNKNOWN 9999 /* Number of bit shifts used in probexp lookup table. */ #define EXP_PREC 15 /* Largest gradient represented in probexp lookup table. */ #define MAXGRAD 5 /* * Delay Precision Enhance - number of bit shifts used for qtrend related * integer arithmetic precision. */ #define D_P_E 7 struct qdiff_sample { long qdiff; STAILQ_ENTRY(qdiff_sample) qdiff_lnk; }; struct cdg { long max_qtrend; long min_qtrend; STAILQ_HEAD(minrtts_head, qdiff_sample) qdiffmin_q; STAILQ_HEAD(maxrtts_head, qdiff_sample) qdiffmax_q; long window_incr; /* rttcount for window increase when in congestion avoidance */ long rtt_count; /* maximum measured rtt within an rtt period */ int maxrtt_in_rtt; /* maximum measured rtt within prev rtt period */ int maxrtt_in_prevrtt; /* minimum measured rtt within an rtt period */ int minrtt_in_rtt; /* minimum measured rtt within prev rtt period */ int minrtt_in_prevrtt; /* consecutive congestion episode counter */ uint32_t consec_cong_cnt; /* when tracking a new reno type loss window */ uint32_t shadow_w; /* maximum number of samples in the moving average queue */ int sample_q_size; /* number of samples in the moving average queue */ int num_samples; /* estimate of the queue state of the path */ int queue_state; }; /* * Lookup table for: * (1 - exp(-x)) << EXP_PREC, where x = [0,MAXGRAD] in 2^-7 increments * * Note: probexp[0] is set to 10 (not 0) as a safety for very low increase * gradients. */ static const int probexp[641] = { 10,255,508,759,1008,1255,1501,1744,1985,2225,2463,2698,2932,3165,3395,3624, 3850,4075,4299,4520,4740,4958,5175,5389,5602,5814,6024,6232,6438,6643,6846, 7048,7248,7447,7644,7839,8033,8226,8417,8606,8794,8981,9166,9350,9532,9713, 9892,10070,10247,10422,10596,10769,10940,11110,11278,11445,11611,11776,11939, 12101,12262,12422,12580,12737,12893,13048,13201,13354,13505,13655,13803,13951, 14097,14243,14387,14530,14672,14813,14952,15091,15229,15365,15500,15635,15768, 15900,16032,16162,16291,16419,16547,16673,16798,16922,17046,17168,17289,17410, 17529,17648,17766,17882,17998,18113,18227,18340,18453,18564,18675,18784,18893, 19001,19108,19215,19320,19425,19529,19632,19734,19835,19936,20036,20135,20233, 20331,20427,20523,20619,20713,20807,20900,20993,21084,21175,21265,21355,21444, 21532,21619,21706,21792,21878,21962,22046,22130,22213,22295,22376,22457,22537, 22617,22696,22774,22852,22929,23006,23082,23157,23232,23306,23380,23453,23525, 23597,23669,23739,23810,23879,23949,24017,24085,24153,24220,24286,24352,24418, 24483,24547,24611,24675,24738,24800,24862,24924,24985,25045,25106,25165,25224, 25283,25341,25399,25456,25513,25570,25626,25681,25737,25791,25846,25899,25953, 26006,26059,26111,26163,26214,26265,26316,26366,26416,26465,26514,26563,26611, 26659,26707,26754,26801,26847,26893,26939,26984,27029,27074,27118,27162,27206, 27249,27292,27335,27377,27419,27460,27502,27543,27583,27624,27664,27703,27743, 27782,27821,27859,27897,27935,27973,28010,28047,28084,28121,28157,28193,28228, 28263,28299,28333,28368,28402,28436,28470,28503,28536,28569,28602,28634,28667, 28699,28730,28762,28793,28824,28854,28885,28915,28945,28975,29004,29034,29063, 29092,29120,29149,29177,29205,29232,29260,29287,29314,29341,29368,29394,29421, 29447,29472,29498,29524,29549,29574,29599,29623,29648,29672,29696,29720,29744, 29767,29791,29814,29837,29860,29882,29905,29927,29949,29971,29993,30014,30036, 30057,30078,30099,30120,30141,30161,30181,30201,30221,30241,30261,30280,30300, 30319,30338,30357,30376,30394,30413,30431,30449,30467,30485,30503,30521,30538, 30555,30573,30590,30607,30624,30640,30657,30673,30690,30706,30722,30738,30753, 30769,30785,30800,30815,30831,30846,30861,30876,30890,30905,30919,30934,30948, 30962,30976,30990,31004,31018,31031,31045,31058,31072,31085,31098,31111,31124, 31137,31149,31162,31174,31187,31199,31211,31223,31235,31247,31259,31271,31283, 31294,31306,31317,31328,31339,31351,31362,31373,31383,31394,31405,31416,31426, 31436,31447,31457,31467,31477,31487,31497,31507,31517,31527,31537,31546,31556, 31565,31574,31584,31593,31602,31611,31620,31629,31638,31647,31655,31664,31673, 31681,31690,31698,31706,31715,31723,31731,31739,31747,31755,31763,31771,31778, 31786,31794,31801,31809,31816,31824,31831,31838,31846,31853,31860,31867,31874, 31881,31888,31895,31902,31908,31915,31922,31928,31935,31941,31948,31954,31960, 31967,31973,31979,31985,31991,31997,32003,32009,32015,32021,32027,32033,32038, 32044,32050,32055,32061,32066,32072,32077,32083,32088,32093,32098,32104,32109, 32114,32119,32124,32129,32134,32139,32144,32149,32154,32158,32163,32168,32173, 32177,32182,32186,32191,32195,32200,32204,32209,32213,32217,32222,32226,32230, 32234,32238,32242,32247,32251,32255,32259,32263,32267,32270,32274,32278,32282, 32286,32290,32293,32297,32301,32304,32308,32311,32315,32318,32322,32325,32329, 32332,32336,32339,32342,32346,32349,32352,32356,32359,32362,32365,32368,32371, 32374,32377,32381,32384,32387,32389,32392,32395,32398,32401,32404,32407,32410, 32412,32415,32418,32421,32423,32426,32429,32431,32434,32437,32439,32442,32444, 32447,32449,32452,32454,32457,32459,32461,32464,32466,32469,32471,32473,32476, 32478,32480,32482,32485,32487,32489,32491,32493,32495,32497,32500,32502,32504, 32506,32508,32510,32512,32514,32516,32518,32520,32522,32524,32526,32527,32529, 32531,32533,32535,32537,32538,32540,32542,32544,32545,32547}; static uma_zone_t qdiffsample_zone; static MALLOC_DEFINE(M_CDG, "cdg data", "Per connection data required for the CDG congestion control algorithm"); static int ertt_id; static VNET_DEFINE(uint32_t, cdg_alpha_inc); static VNET_DEFINE(uint32_t, cdg_beta_delay); static VNET_DEFINE(uint32_t, cdg_beta_loss); static VNET_DEFINE(uint32_t, cdg_smoothing_factor); static VNET_DEFINE(uint32_t, cdg_exp_backoff_scale); static VNET_DEFINE(uint32_t, cdg_consec_cong); static VNET_DEFINE(uint32_t, cdg_hold_backoff); #define V_cdg_alpha_inc VNET(cdg_alpha_inc) #define V_cdg_beta_delay VNET(cdg_beta_delay) #define V_cdg_beta_loss VNET(cdg_beta_loss) #define V_cdg_smoothing_factor VNET(cdg_smoothing_factor) #define V_cdg_exp_backoff_scale VNET(cdg_exp_backoff_scale) #define V_cdg_consec_cong VNET(cdg_consec_cong) #define V_cdg_hold_backoff VNET(cdg_hold_backoff) /* Function prototypes. */ static int cdg_mod_init(void); static int cdg_mod_destroy(void); static void cdg_conn_init(struct cc_var *ccv); static int cdg_cb_init(struct cc_var *ccv); static void cdg_cb_destroy(struct cc_var *ccv); static void cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type); static void cdg_ack_received(struct cc_var *ccv, uint16_t ack_type); struct cc_algo cdg_cc_algo = { .name = "cdg", .mod_init = cdg_mod_init, .ack_received = cdg_ack_received, .cb_destroy = cdg_cb_destroy, .cb_init = cdg_cb_init, .conn_init = cdg_conn_init, .cong_signal = cdg_cong_signal, .mod_destroy = cdg_mod_destroy }; /* Vnet created and being initialised. */ static void cdg_init_vnet(const void *unused __unused) { V_cdg_alpha_inc = 0; V_cdg_beta_delay = 70; V_cdg_beta_loss = 50; V_cdg_smoothing_factor = 8; V_cdg_exp_backoff_scale = 3; V_cdg_consec_cong = 5; V_cdg_hold_backoff = 5; } static int cdg_mod_init(void) { VNET_ITERATOR_DECL(v); ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) return (EINVAL); qdiffsample_zone = uma_zcreate("cdg_qdiffsample", sizeof(struct qdiff_sample), NULL, NULL, NULL, NULL, 0, 0); VNET_LIST_RLOCK(); VNET_FOREACH(v) { CURVNET_SET(v); cdg_init_vnet(NULL); CURVNET_RESTORE(); } VNET_LIST_RUNLOCK(); cdg_cc_algo.post_recovery = newreno_cc_algo.post_recovery; cdg_cc_algo.after_idle = newreno_cc_algo.after_idle; return (0); } static int cdg_mod_destroy(void) { uma_zdestroy(qdiffsample_zone); return (0); } static int cdg_cb_init(struct cc_var *ccv) { struct cdg *cdg_data; cdg_data = malloc(sizeof(struct cdg), M_CDG, M_NOWAIT); if (cdg_data == NULL) return (ENOMEM); cdg_data->shadow_w = 0; cdg_data->max_qtrend = 0; cdg_data->min_qtrend = 0; cdg_data->queue_state = CDG_Q_UNKNOWN; cdg_data->maxrtt_in_rtt = 0; cdg_data->maxrtt_in_prevrtt = 0; cdg_data->minrtt_in_rtt = INT_MAX; cdg_data->minrtt_in_prevrtt = 0; cdg_data->window_incr = 0; cdg_data->rtt_count = 0; cdg_data->consec_cong_cnt = 0; cdg_data->sample_q_size = V_cdg_smoothing_factor; cdg_data->num_samples = 0; STAILQ_INIT(&cdg_data->qdiffmin_q); STAILQ_INIT(&cdg_data->qdiffmax_q); ccv->cc_data = cdg_data; return (0); } static void cdg_conn_init(struct cc_var *ccv) { struct cdg *cdg_data = ccv->cc_data; /* * Initialise the shadow_cwnd in case we are competing with loss based * flows from the start */ cdg_data->shadow_w = CCV(ccv, snd_cwnd); } static void cdg_cb_destroy(struct cc_var *ccv) { struct cdg *cdg_data; struct qdiff_sample *qds, *qds_n; cdg_data = ccv->cc_data; qds = STAILQ_FIRST(&cdg_data->qdiffmin_q); while (qds != NULL) { qds_n = STAILQ_NEXT(qds, qdiff_lnk); uma_zfree(qdiffsample_zone,qds); qds = qds_n; } qds = STAILQ_FIRST(&cdg_data->qdiffmax_q); while (qds != NULL) { qds_n = STAILQ_NEXT(qds, qdiff_lnk); uma_zfree(qdiffsample_zone,qds); qds = qds_n; } free(ccv->cc_data, M_CDG); } static int cdg_beta_handler(SYSCTL_HANDLER_ARGS) { if (req->newptr != NULL && (CAST_PTR_INT(req->newptr) == 0 || CAST_PTR_INT(req->newptr) > 100)) return (EINVAL); return (sysctl_handle_int(oidp, arg1, arg2, req)); } static int cdg_exp_backoff_scale_handler(SYSCTL_HANDLER_ARGS) { if (req->newptr != NULL && CAST_PTR_INT(req->newptr) < 1) return (EINVAL); return (sysctl_handle_int(oidp, arg1, arg2, req)); } static inline uint32_t cdg_window_decrease(struct cc_var *ccv, unsigned long owin, unsigned int beta) { return ((ulmin(CCV(ccv, snd_wnd), owin) * beta) / 100); } /* * Window increase function * This window increase function is independent of the initial window size * to ensure small window flows are not discriminated against (i.e. fairness). * It increases at 1pkt/rtt like Reno for alpha_inc rtts, and then 2pkts/rtt for * the next alpha_inc rtts, etc. */ static void cdg_window_increase(struct cc_var *ccv, int new_measurement) { struct cdg *cdg_data; int incr, s_w_incr; cdg_data = ccv->cc_data; incr = s_w_incr = 0; if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) { /* Slow start. */ incr = CCV(ccv, t_maxseg); s_w_incr = incr; cdg_data->window_incr = cdg_data->rtt_count = 0; } else { /* Congestion avoidance. */ if (new_measurement) { s_w_incr = CCV(ccv, t_maxseg); if (V_cdg_alpha_inc == 0) { incr = CCV(ccv, t_maxseg); } else { if (++cdg_data->rtt_count >= V_cdg_alpha_inc) { cdg_data->window_incr++; cdg_data->rtt_count = 0; } incr = CCV(ccv, t_maxseg) * cdg_data->window_incr; } } } if (cdg_data->shadow_w > 0) cdg_data->shadow_w = ulmin(cdg_data->shadow_w + s_w_incr, TCP_MAXWIN << CCV(ccv, snd_scale)); CCV(ccv, snd_cwnd) = ulmin(CCV(ccv, snd_cwnd) + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } static void cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type) { struct cdg *cdg_data = ccv->cc_data; - uint32_t cwin; - u_int mss; - cwin = CCV(ccv, snd_cwnd); - mss = CCV(ccv, t_maxseg); - switch(signal_type) { case CC_CDG_DELAY: CCV(ccv, snd_ssthresh) = cdg_window_decrease(ccv, CCV(ccv, snd_cwnd), V_cdg_beta_delay); CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); CCV(ccv, snd_recover) = CCV(ccv, snd_max); cdg_data->window_incr = cdg_data->rtt_count = 0; ENTER_CONGRECOVERY(CCV(ccv, t_flags)); break; case CC_NDUPACK: /* * If already responding to congestion OR we have guessed no * queue in the path is full. */ if (IN_CONGRECOVERY(CCV(ccv, t_flags)) || cdg_data->queue_state < CDG_Q_FULL) { - CCV(ccv, snd_ssthresh) = cwin; + CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); CCV(ccv, snd_recover) = CCV(ccv, snd_max); } else { /* * Loss is likely to be congestion related. We have * inferred a queue full state, so have shadow window * react to loss as NewReno would. */ if (cdg_data->shadow_w > 0) cdg_data->shadow_w = cdg_window_decrease(ccv, cdg_data->shadow_w, RENO_BETA); CCV(ccv, snd_ssthresh) = max(cdg_data->shadow_w, - cdg_window_decrease(ccv, cwin, V_cdg_beta_loss)); - CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); + cdg_window_decrease(ccv, CCV(ccv, snd_cwnd), + V_cdg_beta_loss)); cdg_data->window_incr = cdg_data->rtt_count = 0; } ENTER_RECOVERY(CCV(ccv, t_flags)); - break; - case CC_RTO: - CCV(ccv, snd_ssthresh) = - max((CCV(ccv, snd_max) - CCV(ccv, snd_una)) / 2 / mss, 2) - * mss; - CCV(ccv, snd_cwnd) = mss; break; default: newreno_cc_algo.cong_signal(ccv, signal_type); break; } } /* * Using a negative exponential probabilistic backoff so that sources with * varying RTTs which share the same link will, on average, have the same * probability of backoff over time. * * Prob_backoff = 1 - exp(-qtrend / V_cdg_exp_backoff_scale), where * V_cdg_exp_backoff_scale is the average qtrend for the exponential backoff. */ static inline int prob_backoff(long qtrend) { int backoff, idx, p; backoff = (qtrend > ((MAXGRAD * V_cdg_exp_backoff_scale) << D_P_E)); if (!backoff) { if (V_cdg_exp_backoff_scale > 1) idx = (qtrend + V_cdg_exp_backoff_scale / 2) / V_cdg_exp_backoff_scale; else idx = qtrend; /* Backoff probability proportional to rate of queue growth. */ p = (INT_MAX / (1 << EXP_PREC)) * probexp[idx]; backoff = (random() < p); } return (backoff); } static inline void calc_moving_average(struct cdg *cdg_data, long qdiff_max, long qdiff_min) { struct qdiff_sample *qds; ++cdg_data->num_samples; if (cdg_data->num_samples > cdg_data->sample_q_size) { /* Minimum RTT. */ qds = STAILQ_FIRST(&cdg_data->qdiffmin_q); cdg_data->min_qtrend = cdg_data->min_qtrend + (qdiff_min - qds->qdiff) / cdg_data->sample_q_size; STAILQ_REMOVE_HEAD(&cdg_data->qdiffmin_q, qdiff_lnk); qds->qdiff = qdiff_min; STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds, qdiff_lnk); /* Maximum RTT. */ qds = STAILQ_FIRST(&cdg_data->qdiffmax_q); cdg_data->max_qtrend = cdg_data->max_qtrend + (qdiff_max - qds->qdiff) / cdg_data->sample_q_size; STAILQ_REMOVE_HEAD(&cdg_data->qdiffmax_q, qdiff_lnk); qds->qdiff = qdiff_max; STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds, qdiff_lnk); --cdg_data->num_samples; } else { qds = uma_zalloc(qdiffsample_zone, M_NOWAIT); if (qds != NULL) { cdg_data->min_qtrend = cdg_data->min_qtrend + qdiff_min / cdg_data->sample_q_size; qds->qdiff = qdiff_min; STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds, qdiff_lnk); } qds = uma_zalloc(qdiffsample_zone, M_NOWAIT); if (qds) { cdg_data->max_qtrend = cdg_data->max_qtrend + qdiff_max / cdg_data->sample_q_size; qds->qdiff = qdiff_max; STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds, qdiff_lnk); } } } static void cdg_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct cdg *cdg_data; struct ertt *e_t; long qdiff_max, qdiff_min; int congestion, new_measurement, slowstart; cdg_data = ccv->cc_data; e_t = (struct ertt *)khelp_get_osd(CCV(ccv, osd), ertt_id); new_measurement = e_t->flags & ERTT_NEW_MEASUREMENT; congestion = 0; cdg_data->maxrtt_in_rtt = imax(e_t->rtt, cdg_data->maxrtt_in_rtt); cdg_data->minrtt_in_rtt = imin(e_t->rtt, cdg_data->minrtt_in_rtt); if (new_measurement) { slowstart = (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)); /* * Update smoothed gradient measurements. Since we are only * using one measurement per RTT, use max or min rtt_in_rtt. * This is also less noisy than a sample RTT measurement. Max * RTT measurements can have trouble due to OS issues. */ if (cdg_data->maxrtt_in_prevrtt) { qdiff_max = ((long)(cdg_data->maxrtt_in_rtt - cdg_data->maxrtt_in_prevrtt) << D_P_E ); qdiff_min = ((long)(cdg_data->minrtt_in_rtt - cdg_data->minrtt_in_prevrtt) << D_P_E ); calc_moving_average(cdg_data, qdiff_max, qdiff_min); /* Probabilistic backoff with respect to gradient. */ if (slowstart && qdiff_min > 0) congestion = prob_backoff(qdiff_min); else if (cdg_data->min_qtrend > 0) congestion = prob_backoff(cdg_data->min_qtrend); else if (slowstart && qdiff_max > 0) congestion = prob_backoff(qdiff_max); else if (cdg_data->max_qtrend > 0) congestion = prob_backoff(cdg_data->max_qtrend); /* Update estimate of queue state. */ if (cdg_data->min_qtrend > 0 && cdg_data->max_qtrend <= 0) { cdg_data->queue_state = CDG_Q_FULL; } else if (cdg_data->min_qtrend >= 0 && cdg_data->max_qtrend < 0) { cdg_data->queue_state = CDG_Q_EMPTY; cdg_data->shadow_w = 0; } else if (cdg_data->min_qtrend > 0 && cdg_data->max_qtrend > 0) { cdg_data->queue_state = CDG_Q_RISING; } else if (cdg_data->min_qtrend < 0 && cdg_data->max_qtrend < 0) { cdg_data->queue_state = CDG_Q_FALLING; } if (cdg_data->min_qtrend < 0 || cdg_data->max_qtrend < 0) cdg_data->consec_cong_cnt = 0; } cdg_data->minrtt_in_prevrtt = cdg_data->minrtt_in_rtt; cdg_data->minrtt_in_rtt = INT_MAX; cdg_data->maxrtt_in_prevrtt = cdg_data->maxrtt_in_rtt; cdg_data->maxrtt_in_rtt = 0; e_t->flags &= ~ERTT_NEW_MEASUREMENT; } if (congestion) { cdg_data->consec_cong_cnt++; if (!IN_RECOVERY(CCV(ccv, t_flags))) { if (cdg_data->consec_cong_cnt <= V_cdg_consec_cong) cdg_cong_signal(ccv, CC_CDG_DELAY); else /* * We have been backing off but the queue is not * falling. Assume we are competing with * loss-based flows and don't back off for the * next V_cdg_hold_backoff RTT periods. */ if (cdg_data->consec_cong_cnt >= V_cdg_consec_cong + V_cdg_hold_backoff) cdg_data->consec_cong_cnt = 0; /* Won't see effect until 2nd RTT. */ cdg_data->maxrtt_in_prevrtt = 0; /* * Resync shadow window in case we are competing with a * loss based flow */ cdg_data->shadow_w = ulmax(CCV(ccv, snd_cwnd), cdg_data->shadow_w); } } else if (ack_type == CC_ACK) cdg_window_increase(ccv, new_measurement); } /* When a vnet is created and being initialised, init the per-stack CDG vars. */ VNET_SYSINIT(cdg_init_vnet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, cdg_init_vnet, NULL); SYSCTL_DECL(_net_inet_tcp_cc_cdg); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, cdg, CTLFLAG_RW, NULL, "CAIA delay-gradient congestion control related settings"); SYSCTL_STRING(_net_inet_tcp_cc_cdg, OID_AUTO, version, CTLFLAG_RD, CDG_VERSION, sizeof(CDG_VERSION) - 1, "Current algorithm/implementation version number"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, alpha_inc, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_alpha_inc), 0, "Increment the window increase factor alpha by 1 MSS segment every " "alpha_inc RTTs during congestion avoidance mode."); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_delay, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_beta_delay), 70, &cdg_beta_handler, "IU", "Delay-based window decrease factor as a percentage " "(on delay-based backoff, w = w * beta_delay / 100)"); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_loss, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_beta_loss), 50, &cdg_beta_handler, "IU", "Loss-based window decrease factor as a percentage " "(on loss-based backoff, w = w * beta_loss / 100)"); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, exp_backoff_scale, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_exp_backoff_scale), 2, &cdg_exp_backoff_scale_handler, "IU", "Scaling parameter for the probabilistic exponential backoff"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, smoothing_factor, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_smoothing_factor), 8, "Number of samples used for moving average smoothing (0 = no smoothing)"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_consec_cong, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_consec_cong), 5, "Number of consecutive delay-gradient based congestion episodes which will " "trigger loss based CC compatibility"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_hold_backoff, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_hold_backoff), 5, "Number of consecutive delay-gradient based congestion episodes to hold " "the window backoff for loss based CC compatibility"); DECLARE_CC_MODULE(cdg, &cdg_cc_algo); MODULE_DEPEND(cdg, ertt, 1, 1, 1); Index: projects/runtime-coverage/sys/netinet/cc/cc_chd.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_chd.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_chd.c (revision 321585) @@ -1,506 +1,498 @@ /*- * Copyright (c) 2009-2010 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2010-2011 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes and * Lawrence Stewart, made possible in part by a grant from the Cisco University * Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, Melbourne, Australia by * David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the CAIA-Hamilton delay based congestion control * algorithm, based on "Improved coexistence and loss tolerance for delay based * TCP congestion control" by D. A. Hayes and G. Armitage., in 35th Annual IEEE * Conference on Local Computer Networks (LCN 2010), Denver, Colorado, USA, * 11-14 October 2010. * * Originally released as part of the NewTCP research project at Swinburne * University of Technology's Centre for Advanced Internet Architectures, * Melbourne, Australia, which was made possible in part by a grant from the * Cisco University Research Program Fund at Community Foundation Silicon * Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CAST_PTR_INT(X) (*((int*)(X))) /* * Private signal type for rate based congestion signal. * See for appropriate bit-range to use for private signals. */ #define CC_CHD_DELAY 0x02000000 /* Largest possible number returned by random(). */ #define RANDOM_MAX INT_MAX static void chd_ack_received(struct cc_var *ccv, uint16_t ack_type); static void chd_cb_destroy(struct cc_var *ccv); static int chd_cb_init(struct cc_var *ccv); static void chd_cong_signal(struct cc_var *ccv, uint32_t signal_type); static void chd_conn_init(struct cc_var *ccv); static int chd_mod_init(void); struct chd { /* * Shadow window - keeps track of what the NewReno congestion window * would have been if delay-based cwnd backoffs had not been made. This * functionality aids coexistence with loss-based TCP flows which may be * sharing links along the path. */ unsigned long shadow_w; /* * Loss-based TCP compatibility flag - When set, it turns on the shadow * window functionality. */ int loss_compete; /* The maximum round trip time seen within a measured rtt period. */ int maxrtt_in_rtt; /* The previous qdly that caused cwnd to backoff. */ int prev_backoff_qdly; }; static int ertt_id; static VNET_DEFINE(uint32_t, chd_qmin) = 5; static VNET_DEFINE(uint32_t, chd_pmax) = 50; static VNET_DEFINE(uint32_t, chd_loss_fair) = 1; static VNET_DEFINE(uint32_t, chd_use_max) = 1; static VNET_DEFINE(uint32_t, chd_qthresh) = 20; #define V_chd_qthresh VNET(chd_qthresh) #define V_chd_qmin VNET(chd_qmin) #define V_chd_pmax VNET(chd_pmax) #define V_chd_loss_fair VNET(chd_loss_fair) #define V_chd_use_max VNET(chd_use_max) static MALLOC_DEFINE(M_CHD, "chd data", "Per connection data required for the CHD congestion control algorithm"); struct cc_algo chd_cc_algo = { .name = "chd", .ack_received = chd_ack_received, .cb_destroy = chd_cb_destroy, .cb_init = chd_cb_init, .cong_signal = chd_cong_signal, .conn_init = chd_conn_init, .mod_init = chd_mod_init }; static __inline void chd_window_decrease(struct cc_var *ccv) { unsigned long win; win = min(CCV(ccv, snd_wnd), CCV(ccv, snd_cwnd)) / CCV(ccv, t_maxseg); win -= max((win / 2), 1); CCV(ccv, snd_ssthresh) = max(win, 2) * CCV(ccv, t_maxseg); } /* * Probabilistic backoff function. Returns 1 if we should backoff or 0 * otherwise. The calculation of p is similar to the calculation of p in cc_hd. */ static __inline int should_backoff(int qdly, int maxqdly, struct chd *chd_data) { unsigned long p, rand; rand = random(); if (qdly < V_chd_qthresh) { chd_data->loss_compete = 0; p = (((RANDOM_MAX / 100) * V_chd_pmax) / (V_chd_qthresh - V_chd_qmin)) * (qdly - V_chd_qmin); } else { if (qdly > V_chd_qthresh) { p = (((RANDOM_MAX / 100) * V_chd_pmax) / (maxqdly - V_chd_qthresh)) * (maxqdly - qdly); if (V_chd_loss_fair && rand < p) chd_data->loss_compete = 1; } else { p = (RANDOM_MAX / 100) * V_chd_pmax; chd_data->loss_compete = 0; } } return (rand < p); } static __inline void chd_window_increase(struct cc_var *ccv, int new_measurement) { struct chd *chd_data; int incr; chd_data = ccv->cc_data; incr = 0; if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) { /* Adapted from NewReno slow start. */ if (V_tcp_do_rfc3465) { /* In slow-start with ABC enabled. */ if (CCV(ccv, snd_nxt) == CCV(ccv, snd_max)) { /* Not due to RTO. */ incr = min(ccv->bytes_this_ack, V_tcp_abc_l_var * CCV(ccv, t_maxseg)); } else { /* Due to RTO. */ incr = min(ccv->bytes_this_ack, CCV(ccv, t_maxseg)); } } else incr = CCV(ccv, t_maxseg); } else { /* Congestion avoidance. */ if (V_tcp_do_rfc3465) { if (ccv->flags & CCF_ABC_SENTAWND) { ccv->flags &= ~CCF_ABC_SENTAWND; incr = CCV(ccv, t_maxseg); } } else if (new_measurement) incr = CCV(ccv, t_maxseg); } if (chd_data->shadow_w > 0) { /* Track NewReno window. */ chd_data->shadow_w = min(chd_data->shadow_w + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } CCV(ccv,snd_cwnd) = min(CCV(ccv, snd_cwnd) + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } /* * All ACK signals are used for timing measurements to determine delay-based * congestion. However, window increases are only performed when * ack_type == CC_ACK. */ static void chd_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct chd *chd_data; struct ertt *e_t; int backoff, new_measurement, qdly, rtt; e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); chd_data = ccv->cc_data; new_measurement = e_t->flags & ERTT_NEW_MEASUREMENT; backoff = qdly = 0; chd_data->maxrtt_in_rtt = imax(e_t->rtt, chd_data->maxrtt_in_rtt); if (new_measurement) { /* * There is a new per RTT measurement, so check to see if there * is delay based congestion. */ rtt = V_chd_use_max ? chd_data->maxrtt_in_rtt : e_t->rtt; chd_data->maxrtt_in_rtt = 0; if (rtt && e_t->minrtt && !IN_RECOVERY(CCV(ccv, t_flags))) { qdly = rtt - e_t->minrtt; if (qdly > V_chd_qmin) { /* * Probabilistic delay based congestion * indication. */ backoff = should_backoff(qdly, e_t->maxrtt - e_t->minrtt, chd_data); } else chd_data->loss_compete = 0; } /* Reset per RTT measurement flag to start a new measurement. */ e_t->flags &= ~ERTT_NEW_MEASUREMENT; } if (backoff) { /* * Update shadow_w before delay based backoff. */ if (chd_data->loss_compete || qdly > chd_data->prev_backoff_qdly) { /* * Delay is higher than when we backed off previously, * so it is possible that this flow is competing with * loss based flows. */ chd_data->shadow_w = max(CCV(ccv, snd_cwnd), chd_data->shadow_w); } else { /* * Reset shadow_w, as it is probable that this flow is * not competing with loss based flows at the moment. */ chd_data->shadow_w = 0; } chd_data->prev_backoff_qdly = qdly; /* * Send delay-based congestion signal to the congestion signal * handler. */ chd_cong_signal(ccv, CC_CHD_DELAY); } else if (ack_type == CC_ACK) chd_window_increase(ccv, new_measurement); } static void chd_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_CHD); } static int chd_cb_init(struct cc_var *ccv) { struct chd *chd_data; chd_data = malloc(sizeof(struct chd), M_CHD, M_NOWAIT); if (chd_data == NULL) return (ENOMEM); chd_data->shadow_w = 0; ccv->cc_data = chd_data; return (0); } static void chd_cong_signal(struct cc_var *ccv, uint32_t signal_type) { struct ertt *e_t; struct chd *chd_data; int qdly; - u_int mss; e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); chd_data = ccv->cc_data; qdly = imax(e_t->rtt, chd_data->maxrtt_in_rtt) - e_t->minrtt; - mss = CCV(ccv, t_maxseg); switch(signal_type) { case CC_CHD_DELAY: chd_window_decrease(ccv); /* Set new ssthresh. */ CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); CCV(ccv, snd_recover) = CCV(ccv, snd_max); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); break; case CC_NDUPACK: /* Packet loss. */ /* * Only react to loss as a congestion signal if qdly > * V_chd_qthresh. If qdly is less than qthresh, presume that * this is a non congestion related loss. If qdly is greater * than qthresh, assume that we are competing with loss based * tcp flows and restore window from any unnecessary backoffs, * before the decrease. */ if (!IN_RECOVERY(CCV(ccv, t_flags)) && qdly > V_chd_qthresh) { if (chd_data->loss_compete) { CCV(ccv, snd_cwnd) = max(CCV(ccv, snd_cwnd), chd_data->shadow_w); } chd_window_decrease(ccv); } else { /* * This loss isn't congestion related, or already * recovering from congestion. */ CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); CCV(ccv, snd_recover) = CCV(ccv, snd_max); } if (chd_data->shadow_w > 0) { chd_data->shadow_w = max(chd_data->shadow_w / CCV(ccv, t_maxseg) / 2, 2) * CCV(ccv, t_maxseg); } ENTER_FASTRECOVERY(CCV(ccv, t_flags)); - break; - case CC_RTO: - CCV(ccv, snd_ssthresh) = - max((CCV(ccv, snd_max) - CCV(ccv, snd_una)) / 2 / mss, 2) - * mss; - CCV(ccv, snd_cwnd) = mss; break; default: newreno_cc_algo.cong_signal(ccv, signal_type); } } static void chd_conn_init(struct cc_var *ccv) { struct chd *chd_data; chd_data = ccv->cc_data; chd_data->prev_backoff_qdly = 0; chd_data->maxrtt_in_rtt = 0; chd_data->loss_compete = 0; /* * Initialise the shadow_cwnd to be equal to snd_cwnd in case we are * competing with loss based flows from the start. */ chd_data->shadow_w = CCV(ccv, snd_cwnd); } static int chd_mod_init(void) { ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) { printf("%s: h_ertt module not found\n", __func__); return (ENOENT); } chd_cc_algo.after_idle = newreno_cc_algo.after_idle; chd_cc_algo.post_recovery = newreno_cc_algo.post_recovery; return (0); } static int chd_loss_fair_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_loss_fair; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) > 1) error = EINVAL; else V_chd_loss_fair = new; } return (error); } static int chd_pmax_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_pmax; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) == 0 || CAST_PTR_INT(req->newptr) > 100) error = EINVAL; else V_chd_pmax = new; } return (error); } static int chd_qthresh_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_qthresh; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) <= V_chd_qmin) error = EINVAL; else V_chd_qthresh = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_chd); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, chd, CTLFLAG_RW, NULL, "CAIA Hamilton delay-based congestion control related settings"); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, loss_fair, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_loss_fair), 1, &chd_loss_fair_handler, "IU", "Flag to enable shadow window functionality."); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, pmax, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_pmax), 5, &chd_pmax_handler, "IU", "Per RTT maximum backoff probability as a percentage"); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, queue_threshold, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_qthresh), 20, &chd_qthresh_handler, "IU", "Queueing congestion threshold in ticks"); SYSCTL_UINT(_net_inet_tcp_cc_chd, OID_AUTO, queue_min, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(chd_qmin), 5, "Minimum queueing delay threshold in ticks"); SYSCTL_UINT(_net_inet_tcp_cc_chd, OID_AUTO, use_max, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(chd_use_max), 1, "Use the maximum RTT seen within the measurement period (RTT) " "as the basic delay measurement for the algorithm."); DECLARE_CC_MODULE(chd, &chd_cc_algo); MODULE_DEPEND(chd, ertt, 1, 1, 1); Index: projects/runtime-coverage/sys/netinet/cc/cc_cubic.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_cubic.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_cubic.c (revision 321585) @@ -1,420 +1,411 @@ /*- * Copyright (c) 2008-2010 Lawrence Stewart * Copyright (c) 2010 The FreeBSD Foundation * All rights reserved. * * This software was developed by Lawrence Stewart while studying at the Centre * for Advanced Internet Architectures, Swinburne University of Technology, made * possible in part by a grant from the Cisco University Research Program Fund * at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the CUBIC congestion control algorithm for FreeBSD, * based on the Internet Draft "draft-rhee-tcpm-cubic-02" by Rhee, Xu and Ha. * Originally released as part of the NewTCP research project at Swinburne * University of Technology's Centre for Advanced Internet Architectures, * Melbourne, Australia, which was made possible in part by a grant from the * Cisco University Research Program Fund at Community Foundation Silicon * Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void cubic_ack_received(struct cc_var *ccv, uint16_t type); static void cubic_cb_destroy(struct cc_var *ccv); static int cubic_cb_init(struct cc_var *ccv); static void cubic_cong_signal(struct cc_var *ccv, uint32_t type); static void cubic_conn_init(struct cc_var *ccv); static int cubic_mod_init(void); static void cubic_post_recovery(struct cc_var *ccv); static void cubic_record_rtt(struct cc_var *ccv); static void cubic_ssthresh_update(struct cc_var *ccv); struct cubic { /* Cubic K in fixed point form with CUBIC_SHIFT worth of precision. */ int64_t K; /* Sum of RTT samples across an epoch in ticks. */ int64_t sum_rtt_ticks; /* cwnd at the most recent congestion event. */ unsigned long max_cwnd; /* cwnd at the previous congestion event. */ unsigned long prev_max_cwnd; /* Number of congestion events. */ uint32_t num_cong_events; /* Minimum observed rtt in ticks. */ int min_rtt_ticks; /* Mean observed rtt between congestion epochs. */ int mean_rtt_ticks; /* ACKs since last congestion event. */ int epoch_ack_count; /* Time of last congestion event in ticks. */ int t_last_cong; }; static MALLOC_DEFINE(M_CUBIC, "cubic data", "Per connection data required for the CUBIC congestion control algorithm"); struct cc_algo cubic_cc_algo = { .name = "cubic", .ack_received = cubic_ack_received, .cb_destroy = cubic_cb_destroy, .cb_init = cubic_cb_init, .cong_signal = cubic_cong_signal, .conn_init = cubic_conn_init, .mod_init = cubic_mod_init, .post_recovery = cubic_post_recovery, }; static void cubic_ack_received(struct cc_var *ccv, uint16_t type) { struct cubic *cubic_data; unsigned long w_tf, w_cubic_next; int ticks_since_cong; cubic_data = ccv->cc_data; cubic_record_rtt(ccv); /* * Regular ACK and we're not in cong/fast recovery and we're cwnd * limited and we're either not doing ABC or are slow starting or are * doing ABC and we've sent a cwnd's worth of bytes. */ if (type == CC_ACK && !IN_RECOVERY(CCV(ccv, t_flags)) && (ccv->flags & CCF_CWND_LIMITED) && (!V_tcp_do_rfc3465 || CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh) || (V_tcp_do_rfc3465 && ccv->flags & CCF_ABC_SENTAWND))) { /* Use the logic in NewReno ack_received() for slow start. */ if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh) || cubic_data->min_rtt_ticks == TCPTV_SRTTBASE) newreno_cc_algo.ack_received(ccv, type); else { ticks_since_cong = ticks - cubic_data->t_last_cong; /* * The mean RTT is used to best reflect the equations in * the I-D. Using min_rtt in the tf_cwnd calculation * causes w_tf to grow much faster than it should if the * RTT is dominated by network buffering rather than * propagation delay. */ w_tf = tf_cwnd(ticks_since_cong, cubic_data->mean_rtt_ticks, cubic_data->max_cwnd, CCV(ccv, t_maxseg)); w_cubic_next = cubic_cwnd(ticks_since_cong + cubic_data->mean_rtt_ticks, cubic_data->max_cwnd, CCV(ccv, t_maxseg), cubic_data->K); ccv->flags &= ~CCF_ABC_SENTAWND; if (w_cubic_next < w_tf) /* * TCP-friendly region, follow tf * cwnd growth. */ CCV(ccv, snd_cwnd) = w_tf; else if (CCV(ccv, snd_cwnd) < w_cubic_next) { /* * Concave or convex region, follow CUBIC * cwnd growth. */ if (V_tcp_do_rfc3465) CCV(ccv, snd_cwnd) = w_cubic_next; else CCV(ccv, snd_cwnd) += ((w_cubic_next - CCV(ccv, snd_cwnd)) * CCV(ccv, t_maxseg)) / CCV(ccv, snd_cwnd); } /* * If we're not in slow start and we're probing for a * new cwnd limit at the start of a connection * (happens when hostcache has a relevant entry), * keep updating our current estimate of the * max_cwnd. */ if (cubic_data->num_cong_events == 0 && cubic_data->max_cwnd < CCV(ccv, snd_cwnd)) cubic_data->max_cwnd = CCV(ccv, snd_cwnd); } } } static void cubic_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_CUBIC); } static int cubic_cb_init(struct cc_var *ccv) { struct cubic *cubic_data; cubic_data = malloc(sizeof(struct cubic), M_CUBIC, M_NOWAIT|M_ZERO); if (cubic_data == NULL) return (ENOMEM); /* Init some key variables with sensible defaults. */ cubic_data->t_last_cong = ticks; cubic_data->min_rtt_ticks = TCPTV_SRTTBASE; cubic_data->mean_rtt_ticks = 1; ccv->cc_data = cubic_data; return (0); } /* * Perform any necessary tasks before we enter congestion recovery. */ static void cubic_cong_signal(struct cc_var *ccv, uint32_t type) { struct cubic *cubic_data; - uint32_t cwin; - u_int mss; cubic_data = ccv->cc_data; - cwin = CCV(ccv, snd_cwnd); - mss = CCV(ccv, t_maxseg); switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { cubic_ssthresh_update(ccv); cubic_data->num_cong_events++; cubic_data->prev_max_cwnd = cubic_data->max_cwnd; - cubic_data->max_cwnd = cwin; - CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); + cubic_data->max_cwnd = CCV(ccv, snd_cwnd); } ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { cubic_ssthresh_update(ccv); cubic_data->num_cong_events++; cubic_data->prev_max_cwnd = cubic_data->max_cwnd; - cubic_data->max_cwnd = cwin; + cubic_data->max_cwnd = CCV(ccv, snd_cwnd); cubic_data->t_last_cong = ticks; CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } break; case CC_RTO: /* * Grab the current time and record it so we know when the * most recent congestion event was. Only record it when the * timeout has fired more than once, as there is a reasonable * chance the first one is a false alarm and may not indicate * congestion. */ - if (CCV(ccv, t_rxtshift) >= 2) { + if (CCV(ccv, t_rxtshift) >= 2) cubic_data->num_cong_events++; cubic_data->t_last_cong = ticks; - cubic_ssthresh_update(ccv); - cubic_data->max_cwnd = cwin; - CCV(ccv, snd_cwnd) = mss; - } break; } } static void cubic_conn_init(struct cc_var *ccv) { struct cubic *cubic_data; cubic_data = ccv->cc_data; /* * Ensure we have a sane initial value for max_cwnd recorded. Without * this here bad things happen when entries from the TCP hostcache * get used. */ cubic_data->max_cwnd = CCV(ccv, snd_cwnd); } static int cubic_mod_init(void) { cubic_cc_algo.after_idle = newreno_cc_algo.after_idle; return (0); } /* * Perform any necessary tasks before we exit congestion recovery. */ static void cubic_post_recovery(struct cc_var *ccv) { struct cubic *cubic_data; int pipe; cubic_data = ccv->cc_data; pipe = 0; /* Fast convergence heuristic. */ if (cubic_data->max_cwnd < cubic_data->prev_max_cwnd) cubic_data->max_cwnd = (cubic_data->max_cwnd * CUBIC_FC_FACTOR) >> CUBIC_SHIFT; if (IN_FASTRECOVERY(CCV(ccv, t_flags))) { /* * If inflight data is less than ssthresh, set cwnd * conservatively to avoid a burst of data, as suggested in * the NewReno RFC. Otherwise, use the CUBIC method. * * XXXLAS: Find a way to do this without needing curack */ if (V_tcp_do_rfc6675_pipe) pipe = tcp_compute_pipe(ccv->ccvc.tcp); else pipe = CCV(ccv, snd_max) - ccv->curack; if (pipe < CCV(ccv, snd_ssthresh)) CCV(ccv, snd_cwnd) = pipe + CCV(ccv, t_maxseg); else /* Update cwnd based on beta and adjusted max_cwnd. */ CCV(ccv, snd_cwnd) = max(1, ((CUBIC_BETA * cubic_data->max_cwnd) >> CUBIC_SHIFT)); } cubic_data->t_last_cong = ticks; /* Calculate the average RTT between congestion epochs. */ if (cubic_data->epoch_ack_count > 0 && cubic_data->sum_rtt_ticks >= cubic_data->epoch_ack_count) { cubic_data->mean_rtt_ticks = (int)(cubic_data->sum_rtt_ticks / cubic_data->epoch_ack_count); } cubic_data->epoch_ack_count = 0; cubic_data->sum_rtt_ticks = 0; cubic_data->K = cubic_k(cubic_data->max_cwnd / CCV(ccv, t_maxseg)); } /* * Record the min RTT and sum samples for the epoch average RTT calculation. */ static void cubic_record_rtt(struct cc_var *ccv) { struct cubic *cubic_data; int t_srtt_ticks; /* Ignore srtt until a min number of samples have been taken. */ if (CCV(ccv, t_rttupdated) >= CUBIC_MIN_RTT_SAMPLES) { cubic_data = ccv->cc_data; t_srtt_ticks = CCV(ccv, t_srtt) / TCP_RTT_SCALE; /* * Record the current SRTT as our minrtt if it's the smallest * we've seen or minrtt is currently equal to its initialised * value. * * XXXLAS: Should there be some hysteresis for minrtt? */ if ((t_srtt_ticks < cubic_data->min_rtt_ticks || cubic_data->min_rtt_ticks == TCPTV_SRTTBASE)) { cubic_data->min_rtt_ticks = max(1, t_srtt_ticks); /* * If the connection is within its first congestion * epoch, ensure we prime mean_rtt_ticks with a * reasonable value until the epoch average RTT is * calculated in cubic_post_recovery(). */ if (cubic_data->min_rtt_ticks > cubic_data->mean_rtt_ticks) cubic_data->mean_rtt_ticks = cubic_data->min_rtt_ticks; } /* Sum samples for epoch average RTT calculation. */ cubic_data->sum_rtt_ticks += t_srtt_ticks; cubic_data->epoch_ack_count++; } } /* * Update the ssthresh in the event of congestion. */ static void cubic_ssthresh_update(struct cc_var *ccv) { struct cubic *cubic_data; cubic_data = ccv->cc_data; /* * On the first congestion event, set ssthresh to cwnd * 0.5, on * subsequent congestion events, set it to cwnd * beta. */ if (cubic_data->num_cong_events == 0) CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd) >> 1; else CCV(ccv, snd_ssthresh) = ((u_long)CCV(ccv, snd_cwnd) * CUBIC_BETA) >> CUBIC_SHIFT; } DECLARE_CC_MODULE(cubic, &cubic_cc_algo); Index: projects/runtime-coverage/sys/netinet/cc/cc_dctcp.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_dctcp.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_dctcp.c (revision 321585) @@ -1,475 +1,472 @@ /*- * Copyright (c) 2007-2008 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2014 Midori Kato * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the DCTCP algorithm for FreeBSD, based on * "Data Center TCP (DCTCP)" by M. Alizadeh, A. Greenberg, D. A. Maltz, * J. Padhye, P. Patel, B. Prabhakar, S. Sengupta, and M. Sridharan., * in ACM Conference on SIGCOMM 2010, New York, USA, * Originally released as the contribution of Microsoft Research project. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CAST_PTR_INT(X) (*((int*)(X))) #define MAX_ALPHA_VALUE 1024 static VNET_DEFINE(uint32_t, dctcp_alpha) = 0; #define V_dctcp_alpha VNET(dctcp_alpha) static VNET_DEFINE(uint32_t, dctcp_shift_g) = 4; #define V_dctcp_shift_g VNET(dctcp_shift_g) static VNET_DEFINE(uint32_t, dctcp_slowstart) = 0; #define V_dctcp_slowstart VNET(dctcp_slowstart) struct dctcp { int bytes_ecn; /* # of marked bytes during a RTT */ int bytes_total; /* # of acked bytes during a RTT */ int alpha; /* the fraction of marked bytes */ int ce_prev; /* CE state of the last segment */ int save_sndnxt; /* end sequence number of the current window */ int ece_curr; /* ECE flag in this segment */ int ece_prev; /* ECE flag in the last segment */ uint32_t num_cong_events; /* # of congestion events */ }; static MALLOC_DEFINE(M_dctcp, "dctcp data", "Per connection data required for the dctcp algorithm"); static void dctcp_ack_received(struct cc_var *ccv, uint16_t type); static void dctcp_after_idle(struct cc_var *ccv); static void dctcp_cb_destroy(struct cc_var *ccv); static int dctcp_cb_init(struct cc_var *ccv); static void dctcp_cong_signal(struct cc_var *ccv, uint32_t type); static void dctcp_conn_init(struct cc_var *ccv); static void dctcp_post_recovery(struct cc_var *ccv); static void dctcp_ecnpkt_handler(struct cc_var *ccv); static void dctcp_update_alpha(struct cc_var *ccv); struct cc_algo dctcp_cc_algo = { .name = "dctcp", .ack_received = dctcp_ack_received, .cb_destroy = dctcp_cb_destroy, .cb_init = dctcp_cb_init, .cong_signal = dctcp_cong_signal, .conn_init = dctcp_conn_init, .post_recovery = dctcp_post_recovery, .ecnpkt_handler = dctcp_ecnpkt_handler, .after_idle = dctcp_after_idle, }; static void dctcp_ack_received(struct cc_var *ccv, uint16_t type) { struct dctcp *dctcp_data; int bytes_acked = 0; dctcp_data = ccv->cc_data; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) { /* * DCTCP doesn't treat receipt of ECN marked packet as a * congestion event. Thus, DCTCP always executes the ACK * processing out of congestion recovery. */ if (IN_CONGRECOVERY(CCV(ccv, t_flags))) { EXIT_CONGRECOVERY(CCV(ccv, t_flags)); newreno_cc_algo.ack_received(ccv, type); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } else newreno_cc_algo.ack_received(ccv, type); if (type == CC_DUPACK) bytes_acked = CCV(ccv, t_maxseg); if (type == CC_ACK) bytes_acked = ccv->bytes_this_ack; /* Update total bytes. */ dctcp_data->bytes_total += bytes_acked; /* Update total marked bytes. */ if (dctcp_data->ece_curr) { if (!dctcp_data->ece_prev && bytes_acked > CCV(ccv, t_maxseg)) { dctcp_data->bytes_ecn += (bytes_acked - CCV(ccv, t_maxseg)); } else dctcp_data->bytes_ecn += bytes_acked; dctcp_data->ece_prev = 1; } else { if (dctcp_data->ece_prev && bytes_acked > CCV(ccv, t_maxseg)) dctcp_data->bytes_ecn += CCV(ccv, t_maxseg); dctcp_data->ece_prev = 0; } dctcp_data->ece_curr = 0; /* * Update the fraction of marked bytes at the end of * current window size. */ if ((IN_FASTRECOVERY(CCV(ccv, t_flags)) && SEQ_GEQ(ccv->curack, CCV(ccv, snd_recover))) || (!IN_FASTRECOVERY(CCV(ccv, t_flags)) && SEQ_GT(ccv->curack, dctcp_data->save_sndnxt))) dctcp_update_alpha(ccv); } else newreno_cc_algo.ack_received(ccv, type); } static void dctcp_after_idle(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = ccv->cc_data; /* Initialize internal parameters after idle time */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); dctcp_data->alpha = V_dctcp_alpha; dctcp_data->ece_curr = 0; dctcp_data->ece_prev = 0; dctcp_data->num_cong_events = 0; dctcp_cc_algo.after_idle = newreno_cc_algo.after_idle; } static void dctcp_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_dctcp); } static int dctcp_cb_init(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = malloc(sizeof(struct dctcp), M_dctcp, M_NOWAIT|M_ZERO); if (dctcp_data == NULL) return (ENOMEM); /* Initialize some key variables with sensible defaults. */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; /* * When alpha is set to 0 in the beginning, DCTCP sender transfers as * much data as possible until the value converges which may expand the * queueing delay at the switch. When alpha is set to 1, queueing delay * is kept small. * Throughput-sensitive applications should have alpha = 0 * Latency-sensitive applications should have alpha = 1 * * Note: DCTCP draft suggests initial alpha to be 1 but we've decided to * keep it 0 as default. */ dctcp_data->alpha = V_dctcp_alpha; dctcp_data->save_sndnxt = 0; dctcp_data->ce_prev = 0; dctcp_data->ece_curr = 0; dctcp_data->ece_prev = 0; dctcp_data->num_cong_events = 0; ccv->cc_data = dctcp_data; return (0); } /* * Perform any necessary tasks before we enter congestion recovery. */ static void dctcp_cong_signal(struct cc_var *ccv, uint32_t type) { struct dctcp *dctcp_data; - uint32_t cwin, ssthresh_on_loss; - u_int mss; + u_int win, mss; dctcp_data = ccv->cc_data; - cwin = CCV(ccv, snd_cwnd); + win = CCV(ccv, snd_cwnd); mss = CCV(ccv, t_maxseg); - ssthresh_on_loss = - max((CCV(ccv, snd_max) - CCV(ccv, snd_una)) / 2 / mss, 2) - * mss; switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; + CCV(ccv, snd_ssthresh) = mss * + max(win / 2 / mss, 2); dctcp_data->num_cong_events++; } else { /* cwnd has already updated as congestion * recovery. Reverse cwnd value using * snd_cwnd_prev and recalculate snd_ssthresh */ - cwin = CCV(ccv, snd_cwnd_prev); - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; + win = CCV(ccv, snd_cwnd_prev); + CCV(ccv, snd_ssthresh) = + max(win / 2 / mss, 2) * mss; } ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: /* * Save current snd_cwnd when the host encounters both * congestion recovery and fast recovery. */ - CCV(ccv, snd_cwnd_prev) = cwin; + CCV(ccv, snd_cwnd_prev) = win; if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { if (V_dctcp_slowstart && dctcp_data->num_cong_events++ == 0) { - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; + CCV(ccv, snd_ssthresh) = + mss * max(win / 2 / mss, 2); dctcp_data->alpha = MAX_ALPHA_VALUE; dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } else - CCV(ccv, snd_ssthresh) = max((cwin - ((cwin * + CCV(ccv, snd_ssthresh) = max((win - ((win * dctcp_data->alpha) >> 11)) / mss, 2) * mss; CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } dctcp_data->ece_curr = 1; break; case CC_RTO: if (CCV(ccv, t_flags) & TF_ECN_PERMIT) { CCV(ccv, t_flags) |= TF_ECN_SND_CWR; dctcp_update_alpha(ccv); dctcp_data->save_sndnxt += CCV(ccv, t_maxseg); dctcp_data->num_cong_events++; - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; - CCV(ccv, snd_cwnd) = mss; } break; } } static void dctcp_conn_init(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = ccv->cc_data; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } /* * Perform any necessary tasks before we exit congestion recovery. */ static void dctcp_post_recovery(struct cc_var *ccv) { dctcp_cc_algo.post_recovery = newreno_cc_algo.post_recovery; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) dctcp_update_alpha(ccv); } /* * Execute an additional ECN processing using ECN field in IP header and the CWR * bit in TCP header. * * delay_ack == 0 - Delayed ACK disabled * delay_ack == 1 - Delayed ACK enabled */ static void dctcp_ecnpkt_handler(struct cc_var *ccv) { struct dctcp *dctcp_data; uint32_t ccflag; int delay_ack; dctcp_data = ccv->cc_data; ccflag = ccv->flags; delay_ack = 1; /* * DCTCP responses an ACK immediately when the CE state * in between this segment and the last segment is not same. */ if (ccflag & CCF_IPHDR_CE) { if (!dctcp_data->ce_prev && (ccflag & CCF_DELACK)) delay_ack = 0; dctcp_data->ce_prev = 1; CCV(ccv, t_flags) |= TF_ECN_SND_ECE; } else { if (dctcp_data->ce_prev && (ccflag & CCF_DELACK)) delay_ack = 0; dctcp_data->ce_prev = 0; CCV(ccv, t_flags) &= ~TF_ECN_SND_ECE; } /* DCTCP sets delayed ack when this segment sets the CWR flag. */ if ((ccflag & CCF_DELACK) && (ccflag & CCF_TCPHDR_CWR)) delay_ack = 1; if (delay_ack == 0) ccv->flags |= CCF_ACKNOW; else ccv->flags &= ~CCF_ACKNOW; } /* * Update the fraction of marked bytes represented as 'alpha'. * Also initialize several internal parameters at the end of this function. */ static void dctcp_update_alpha(struct cc_var *ccv) { struct dctcp *dctcp_data; int alpha_prev; dctcp_data = ccv->cc_data; alpha_prev = dctcp_data->alpha; dctcp_data->bytes_total = max(dctcp_data->bytes_total, 1); /* * Update alpha: alpha = (1 - g) * alpha + g * F. * Here: * g is weight factor * recommaded to be set to 1/16 * small g = slow convergence between competitive DCTCP flows * large g = impacts low utilization of bandwidth at switches * F is fraction of marked segments in last RTT * updated every RTT * Alpha must be round to 0 - MAX_ALPHA_VALUE. */ dctcp_data->alpha = min(alpha_prev - (alpha_prev >> V_dctcp_shift_g) + (dctcp_data->bytes_ecn << (10 - V_dctcp_shift_g)) / dctcp_data->bytes_total, MAX_ALPHA_VALUE); /* Initialize internal parameters for next alpha calculation */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } static int dctcp_alpha_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_alpha; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) > 1) error = EINVAL; else { if (new > MAX_ALPHA_VALUE) V_dctcp_alpha = MAX_ALPHA_VALUE; else V_dctcp_alpha = new; } } return (error); } static int dctcp_shift_g_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_shift_g; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) > 1) error = EINVAL; else V_dctcp_shift_g = new; } return (error); } static int dctcp_slowstart_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_slowstart; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { if (CAST_PTR_INT(req->newptr) > 1) error = EINVAL; else V_dctcp_slowstart = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_dctcp); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, dctcp, CTLFLAG_RW, NULL, "dctcp congestion control related settings"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, alpha, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_alpha), 0, &dctcp_alpha_handler, "IU", "dctcp alpha parameter"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, shift_g, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_shift_g), 4, &dctcp_shift_g_handler, "IU", "dctcp shift parameter"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, slowstart, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_slowstart), 0, &dctcp_slowstart_handler, "IU", "half CWND reduction after the first slow start"); DECLARE_CC_MODULE(dctcp, &dctcp_cc_algo); Index: projects/runtime-coverage/sys/netinet/cc/cc_htcp.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_htcp.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_htcp.c (revision 321585) @@ -1,540 +1,527 @@ /*- * Copyright (c) 2007-2008 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2010 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by Lawrence Stewart and * James Healy, made possible in part by a grant from the Cisco University * Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the H-TCP congestion control algorithm for FreeBSD, * based on the Internet Draft "draft-leith-tcp-htcp-06.txt" by Leith and * Shorten. Originally released as part of the NewTCP research project at * Swinburne University of Technology's Centre for Advanced Internet * Architectures, Melbourne, Australia, which was made possible in part by a * grant from the Cisco University Research Program Fund at Community Foundation * Silicon Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Fixed point math shifts. */ #define HTCP_SHIFT 8 #define HTCP_ALPHA_INC_SHIFT 4 #define HTCP_INIT_ALPHA 1 #define HTCP_DELTA_L hz /* 1 sec in ticks. */ #define HTCP_MINBETA 128 /* 0.5 << HTCP_SHIFT. */ #define HTCP_MAXBETA 204 /* ~0.8 << HTCP_SHIFT. */ #define HTCP_MINROWE 26 /* ~0.1 << HTCP_SHIFT. */ #define HTCP_MAXROWE 512 /* 2 << HTCP_SHIFT. */ /* RTT_ref (ms) used in the calculation of alpha if RTT scaling is enabled. */ #define HTCP_RTT_REF 100 /* Don't trust SRTT until this many samples have been taken. */ #define HTCP_MIN_RTT_SAMPLES 8 /* * HTCP_CALC_ALPHA performs a fixed point math calculation to determine the * value of alpha, based on the function defined in the HTCP spec. * * i.e. 1 + 10(delta - delta_l) + ((delta - delta_l) / 2) ^ 2 * * "diff" is passed in to the macro as "delta - delta_l" and is expected to be * in units of ticks. * * The joyousnous of fixed point maths means our function implementation looks a * little funky... * * In order to maintain some precision in the calculations, a fixed point shift * HTCP_ALPHA_INC_SHIFT is used to ensure the integer divisions don't * truncate the results too badly. * * The "16" value is the "1" term in the alpha function shifted up by * HTCP_ALPHA_INC_SHIFT * * The "160" value is the "10" multiplier in the alpha function multiplied by * 2^HTCP_ALPHA_INC_SHIFT * * Specifying these as constants reduces the computations required. After * up-shifting all the terms in the function and performing the required * calculations, we down-shift the final result by HTCP_ALPHA_INC_SHIFT to * ensure it is back in the correct range. * * The "hz" terms are required as kernels can be configured to run with * different tick timers, which we have to adjust for in the alpha calculation * (which originally was defined in terms of seconds). * * We also have to be careful to constrain the value of diff such that it won't * overflow whilst performing the calculation. The middle term i.e. (160 * diff) * / hz is the limiting factor in the calculation. We must constrain diff to be * less than the max size of an int divided by the constant 160 figure * i.e. diff < INT_MAX / 160 * * NB: Changing HTCP_ALPHA_INC_SHIFT will require you to MANUALLY update the * constants used in this function! */ #define HTCP_CALC_ALPHA(diff) \ ((\ (16) + \ ((160 * (diff)) / hz) + \ (((diff) / hz) * (((diff) << HTCP_ALPHA_INC_SHIFT) / (4 * hz))) \ ) >> HTCP_ALPHA_INC_SHIFT) static void htcp_ack_received(struct cc_var *ccv, uint16_t type); static void htcp_cb_destroy(struct cc_var *ccv); static int htcp_cb_init(struct cc_var *ccv); static void htcp_cong_signal(struct cc_var *ccv, uint32_t type); static int htcp_mod_init(void); static void htcp_post_recovery(struct cc_var *ccv); static void htcp_recalc_alpha(struct cc_var *ccv); static void htcp_recalc_beta(struct cc_var *ccv); static void htcp_record_rtt(struct cc_var *ccv); static void htcp_ssthresh_update(struct cc_var *ccv); struct htcp { /* cwnd before entering cong recovery. */ unsigned long prev_cwnd; /* cwnd additive increase parameter. */ int alpha; /* cwnd multiplicative decrease parameter. */ int beta; /* Largest rtt seen for the flow. */ int maxrtt; /* Shortest rtt seen for the flow. */ int minrtt; /* Time of last congestion event in ticks. */ int t_last_cong; }; static int htcp_rtt_ref; /* * The maximum number of ticks the value of diff can reach in * htcp_recalc_alpha() before alpha will stop increasing due to overflow. * See comment above HTCP_CALC_ALPHA for more info. */ static int htcp_max_diff = INT_MAX / ((1 << HTCP_ALPHA_INC_SHIFT) * 10); /* Per-netstack vars. */ static VNET_DEFINE(u_int, htcp_adaptive_backoff) = 0; static VNET_DEFINE(u_int, htcp_rtt_scaling) = 0; #define V_htcp_adaptive_backoff VNET(htcp_adaptive_backoff) #define V_htcp_rtt_scaling VNET(htcp_rtt_scaling) static MALLOC_DEFINE(M_HTCP, "htcp data", "Per connection data required for the HTCP congestion control algorithm"); struct cc_algo htcp_cc_algo = { .name = "htcp", .ack_received = htcp_ack_received, .cb_destroy = htcp_cb_destroy, .cb_init = htcp_cb_init, .cong_signal = htcp_cong_signal, .mod_init = htcp_mod_init, .post_recovery = htcp_post_recovery, }; static void htcp_ack_received(struct cc_var *ccv, uint16_t type) { struct htcp *htcp_data; htcp_data = ccv->cc_data; htcp_record_rtt(ccv); /* * Regular ACK and we're not in cong/fast recovery and we're cwnd * limited and we're either not doing ABC or are slow starting or are * doing ABC and we've sent a cwnd's worth of bytes. */ if (type == CC_ACK && !IN_RECOVERY(CCV(ccv, t_flags)) && (ccv->flags & CCF_CWND_LIMITED) && (!V_tcp_do_rfc3465 || CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh) || (V_tcp_do_rfc3465 && ccv->flags & CCF_ABC_SENTAWND))) { htcp_recalc_beta(ccv); htcp_recalc_alpha(ccv); /* * Use the logic in NewReno ack_received() for slow start and * for the first HTCP_DELTA_L ticks after either the flow starts * or a congestion event (when alpha equals 1). */ if (htcp_data->alpha == 1 || CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) newreno_cc_algo.ack_received(ccv, type); else { if (V_tcp_do_rfc3465) { /* Increment cwnd by alpha segments. */ CCV(ccv, snd_cwnd) += htcp_data->alpha * CCV(ccv, t_maxseg); ccv->flags &= ~CCF_ABC_SENTAWND; } else /* * Increment cwnd by alpha/cwnd segments to * approximate an increase of alpha segments * per RTT. */ CCV(ccv, snd_cwnd) += (((htcp_data->alpha << HTCP_SHIFT) / (CCV(ccv, snd_cwnd) / CCV(ccv, t_maxseg))) * CCV(ccv, t_maxseg)) >> HTCP_SHIFT; } } } static void htcp_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_HTCP); } static int htcp_cb_init(struct cc_var *ccv) { struct htcp *htcp_data; htcp_data = malloc(sizeof(struct htcp), M_HTCP, M_NOWAIT); if (htcp_data == NULL) return (ENOMEM); /* Init some key variables with sensible defaults. */ htcp_data->alpha = HTCP_INIT_ALPHA; htcp_data->beta = HTCP_MINBETA; htcp_data->maxrtt = TCPTV_SRTTBASE; htcp_data->minrtt = TCPTV_SRTTBASE; htcp_data->prev_cwnd = 0; htcp_data->t_last_cong = ticks; ccv->cc_data = htcp_data; return (0); } /* * Perform any necessary tasks before we enter congestion recovery. */ static void htcp_cong_signal(struct cc_var *ccv, uint32_t type) { struct htcp *htcp_data; - uint32_t cwin; - u_int mss; htcp_data = ccv->cc_data; - cwin = CCV(ccv, snd_cwnd); - mss = CCV(ccv, t_maxseg); switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { /* * Apply hysteresis to maxrtt to ensure * reductions in the RTT are reflected in our * measurements. */ htcp_data->maxrtt = (htcp_data->minrtt + (htcp_data->maxrtt - htcp_data->minrtt) * 95) / 100; htcp_ssthresh_update(ccv); - CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); htcp_data->t_last_cong = ticks; - htcp_data->prev_cwnd = cwin; + htcp_data->prev_cwnd = CCV(ccv, snd_cwnd); } ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { /* * Apply hysteresis to maxrtt to ensure reductions in * the RTT are reflected in our measurements. */ htcp_data->maxrtt = (htcp_data->minrtt + (htcp_data->maxrtt - htcp_data->minrtt) * 95) / 100; htcp_ssthresh_update(ccv); CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); htcp_data->t_last_cong = ticks; - htcp_data->prev_cwnd = cwin; + htcp_data->prev_cwnd = CCV(ccv, snd_cwnd); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } break; case CC_RTO: /* * Grab the current time and record it so we know when the * most recent congestion event was. Only record it when the * timeout has fired more than once, as there is a reasonable * chance the first one is a false alarm and may not indicate * congestion. */ if (CCV(ccv, t_rxtshift) >= 2) htcp_data->t_last_cong = ticks; - CCV(ccv, snd_ssthresh) = - max((CCV(ccv, snd_max) - CCV(ccv, snd_una)) / 2 / mss, 2) - * mss; - CCV(ccv, snd_cwnd) = mss; break; } } static int htcp_mod_init(void) { htcp_cc_algo.after_idle = newreno_cc_algo.after_idle; /* * HTCP_RTT_REF is defined in ms, and t_srtt in the tcpcb is stored in * units of TCP_RTT_SCALE*hz. Scale HTCP_RTT_REF to be in the same units * as t_srtt. */ htcp_rtt_ref = (HTCP_RTT_REF * TCP_RTT_SCALE * hz) / 1000; return (0); } /* * Perform any necessary tasks before we exit congestion recovery. */ static void htcp_post_recovery(struct cc_var *ccv) { int pipe; struct htcp *htcp_data; pipe = 0; htcp_data = ccv->cc_data; if (IN_FASTRECOVERY(CCV(ccv, t_flags))) { /* * If inflight data is less than ssthresh, set cwnd * conservatively to avoid a burst of data, as suggested in the * NewReno RFC. Otherwise, use the HTCP method. * * XXXLAS: Find a way to do this without needing curack */ if (V_tcp_do_rfc6675_pipe) pipe = tcp_compute_pipe(ccv->ccvc.tcp); else pipe = CCV(ccv, snd_max) - ccv->curack; if (pipe < CCV(ccv, snd_ssthresh)) CCV(ccv, snd_cwnd) = pipe + CCV(ccv, t_maxseg); else CCV(ccv, snd_cwnd) = max(1, ((htcp_data->beta * htcp_data->prev_cwnd / CCV(ccv, t_maxseg)) >> HTCP_SHIFT)) * CCV(ccv, t_maxseg); } } static void htcp_recalc_alpha(struct cc_var *ccv) { struct htcp *htcp_data; int alpha, diff, now; htcp_data = ccv->cc_data; now = ticks; /* * If ticks has wrapped around (will happen approximately once every 49 * days on a machine with the default kern.hz=1000) and a flow straddles * the wrap point, our alpha calcs will be completely wrong. We cut our * losses and restart alpha from scratch by setting t_last_cong = now - * HTCP_DELTA_L. * * This does not deflate our cwnd at all. It simply slows the rate cwnd * is growing by until alpha regains the value it held prior to taking * this drastic measure. */ if (now < htcp_data->t_last_cong) htcp_data->t_last_cong = now - HTCP_DELTA_L; diff = now - htcp_data->t_last_cong - HTCP_DELTA_L; /* Cap alpha if the value of diff would overflow HTCP_CALC_ALPHA(). */ if (diff < htcp_max_diff) { /* * If it has been more than HTCP_DELTA_L ticks since congestion, * increase alpha according to the function defined in the spec. */ if (diff > 0) { alpha = HTCP_CALC_ALPHA(diff); /* * Adaptive backoff fairness adjustment: * 2 * (1 - beta) * alpha_raw */ if (V_htcp_adaptive_backoff) alpha = max(1, (2 * ((1 << HTCP_SHIFT) - htcp_data->beta) * alpha) >> HTCP_SHIFT); /* * RTT scaling: (RTT / RTT_ref) * alpha * alpha will be the raw value from HTCP_CALC_ALPHA() if * adaptive backoff is off, or the adjusted value if * adaptive backoff is on. */ if (V_htcp_rtt_scaling) alpha = max(1, (min(max(HTCP_MINROWE, (CCV(ccv, t_srtt) << HTCP_SHIFT) / htcp_rtt_ref), HTCP_MAXROWE) * alpha) >> HTCP_SHIFT); } else alpha = 1; htcp_data->alpha = alpha; } } static void htcp_recalc_beta(struct cc_var *ccv) { struct htcp *htcp_data; htcp_data = ccv->cc_data; /* * TCPTV_SRTTBASE is the initialised value of each connection's SRTT, so * we only calc beta if the connection's SRTT has been changed from its * initial value. beta is bounded to ensure it is always between * HTCP_MINBETA and HTCP_MAXBETA. */ if (V_htcp_adaptive_backoff && htcp_data->minrtt != TCPTV_SRTTBASE && htcp_data->maxrtt != TCPTV_SRTTBASE) htcp_data->beta = min(max(HTCP_MINBETA, (htcp_data->minrtt << HTCP_SHIFT) / htcp_data->maxrtt), HTCP_MAXBETA); else htcp_data->beta = HTCP_MINBETA; } /* * Record the minimum and maximum RTT seen for the connection. These are used in * the calculation of beta if adaptive backoff is enabled. */ static void htcp_record_rtt(struct cc_var *ccv) { struct htcp *htcp_data; htcp_data = ccv->cc_data; /* XXXLAS: Should there be some hysteresis for minrtt? */ /* * Record the current SRTT as our minrtt if it's the smallest we've seen * or minrtt is currently equal to its initialised value. Ignore SRTT * until a min number of samples have been taken. */ if ((CCV(ccv, t_srtt) < htcp_data->minrtt || htcp_data->minrtt == TCPTV_SRTTBASE) && (CCV(ccv, t_rttupdated) >= HTCP_MIN_RTT_SAMPLES)) htcp_data->minrtt = CCV(ccv, t_srtt); /* * Record the current SRTT as our maxrtt if it's the largest we've * seen. Ignore SRTT until a min number of samples have been taken. */ if (CCV(ccv, t_srtt) > htcp_data->maxrtt && CCV(ccv, t_rttupdated) >= HTCP_MIN_RTT_SAMPLES) htcp_data->maxrtt = CCV(ccv, t_srtt); } /* * Update the ssthresh in the event of congestion. */ static void htcp_ssthresh_update(struct cc_var *ccv) { struct htcp *htcp_data; htcp_data = ccv->cc_data; /* * On the first congestion event, set ssthresh to cwnd * 0.5, on * subsequent congestion events, set it to cwnd * beta. */ if (CCV(ccv, snd_ssthresh) == TCP_MAXWIN << TCP_MAX_WINSHIFT) CCV(ccv, snd_ssthresh) = ((u_long)CCV(ccv, snd_cwnd) * HTCP_MINBETA) >> HTCP_SHIFT; else { htcp_recalc_beta(ccv); CCV(ccv, snd_ssthresh) = ((u_long)CCV(ccv, snd_cwnd) * htcp_data->beta) >> HTCP_SHIFT; } - - /* Align ssthresh to MSS boundary */ - CCV(ccv, snd_ssthresh) = (CCV(ccv, snd_ssthresh) / CCV(ccv, t_maxseg)) - * CCV(ccv, t_maxseg); } SYSCTL_DECL(_net_inet_tcp_cc_htcp); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, htcp, CTLFLAG_RW, NULL, "H-TCP related settings"); SYSCTL_UINT(_net_inet_tcp_cc_htcp, OID_AUTO, adaptive_backoff, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(htcp_adaptive_backoff), 0, "enable H-TCP adaptive backoff"); SYSCTL_UINT(_net_inet_tcp_cc_htcp, OID_AUTO, rtt_scaling, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(htcp_rtt_scaling), 0, "enable H-TCP RTT scaling"); DECLARE_CC_MODULE(htcp, &htcp_cc_algo); Index: projects/runtime-coverage/sys/netinet/cc/cc_newreno.c =================================================================== --- projects/runtime-coverage/sys/netinet/cc/cc_newreno.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/cc/cc_newreno.c (revision 321585) @@ -1,256 +1,244 @@ /*- * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995 * The Regents of the University of California. * Copyright (c) 2007-2008,2010 * Swinburne University of Technology, Melbourne, Australia. * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2010 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by Lawrence Stewart, James * Healy and David Hayes, made possible in part by a grant from the Cisco * University Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This software was first released in 2007 by James Healy and Lawrence Stewart * whilst working on the NewTCP research project at Swinburne University of * Technology's Centre for Advanced Internet Architectures, Melbourne, * Australia, which was made possible in part by a grant from the Cisco * University Research Program Fund at Community Foundation Silicon Valley. * More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void newreno_ack_received(struct cc_var *ccv, uint16_t type); static void newreno_after_idle(struct cc_var *ccv); static void newreno_cong_signal(struct cc_var *ccv, uint32_t type); static void newreno_post_recovery(struct cc_var *ccv); struct cc_algo newreno_cc_algo = { .name = "newreno", .ack_received = newreno_ack_received, .after_idle = newreno_after_idle, .cong_signal = newreno_cong_signal, .post_recovery = newreno_post_recovery, }; static void newreno_ack_received(struct cc_var *ccv, uint16_t type) { if (type == CC_ACK && !IN_RECOVERY(CCV(ccv, t_flags)) && (ccv->flags & CCF_CWND_LIMITED)) { u_int cw = CCV(ccv, snd_cwnd); u_int incr = CCV(ccv, t_maxseg); /* * Regular in-order ACK, open the congestion window. * Method depends on which congestion control state we're * in (slow start or cong avoid) and if ABC (RFC 3465) is * enabled. * * slow start: cwnd <= ssthresh * cong avoid: cwnd > ssthresh * * slow start and ABC (RFC 3465): * Grow cwnd exponentially by the amount of data * ACKed capping the max increment per ACK to * (abc_l_var * maxseg) bytes. * * slow start without ABC (RFC 5681): * Grow cwnd exponentially by maxseg per ACK. * * cong avoid and ABC (RFC 3465): * Grow cwnd linearly by maxseg per RTT for each * cwnd worth of ACKed data. * * cong avoid without ABC (RFC 5681): * Grow cwnd linearly by approximately maxseg per RTT using * maxseg^2 / cwnd per ACK as the increment. * If cwnd > maxseg^2, fix the cwnd increment at 1 byte to * avoid capping cwnd. */ if (cw > CCV(ccv, snd_ssthresh)) { if (V_tcp_do_rfc3465) { if (ccv->flags & CCF_ABC_SENTAWND) ccv->flags &= ~CCF_ABC_SENTAWND; else incr = 0; } else incr = max((incr * incr / cw), 1); } else if (V_tcp_do_rfc3465) { /* * In slow-start with ABC enabled and no RTO in sight? * (Must not use abc_l_var > 1 if slow starting after * an RTO. On RTO, snd_nxt = snd_una, so the * snd_nxt == snd_max check is sufficient to * handle this). * * XXXLAS: Find a way to signal SS after RTO that * doesn't rely on tcpcb vars. */ if (CCV(ccv, snd_nxt) == CCV(ccv, snd_max)) incr = min(ccv->bytes_this_ack, ccv->nsegs * V_tcp_abc_l_var * CCV(ccv, t_maxseg)); else incr = min(ccv->bytes_this_ack, CCV(ccv, t_maxseg)); } /* ABC is on by default, so incr equals 0 frequently. */ if (incr > 0) CCV(ccv, snd_cwnd) = min(cw + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } } static void newreno_after_idle(struct cc_var *ccv) { int rw; /* * If we've been idle for more than one retransmit timeout the old * congestion window is no longer current and we have to reduce it to * the restart window before we can transmit again. * * The restart window is the initial window or the last CWND, whichever * is smaller. * * This is done to prevent us from flooding the path with a full CWND at * wirespeed, overloading router and switch buffers along the way. * * See RFC5681 Section 4.1. "Restarting Idle Connections". */ if (V_tcp_do_rfc3390) rw = min(4 * CCV(ccv, t_maxseg), max(2 * CCV(ccv, t_maxseg), 4380)); else rw = CCV(ccv, t_maxseg) * 2; CCV(ccv, snd_cwnd) = min(rw, CCV(ccv, snd_cwnd)); } /* * Perform any necessary tasks before we enter congestion recovery. */ static void newreno_cong_signal(struct cc_var *ccv, uint32_t type) { - uint32_t cwin, ssthresh_on_loss; - u_int mss; + u_int win; - cwin = CCV(ccv, snd_cwnd); - mss = CCV(ccv, t_maxseg); - ssthresh_on_loss = - max((CCV(ccv, snd_max) - CCV(ccv, snd_una)) / 2 / mss, 2) - * mss; - /* Catch algos which mistakenly leak private signal types. */ KASSERT((type & CC_SIGPRIVMASK) == 0, ("%s: congestion signal type 0x%08x is private\n", __func__, type)); - cwin = max(cwin / 2 / mss, 2) * mss; + win = max(CCV(ccv, snd_cwnd) / 2 / CCV(ccv, t_maxseg), 2) * + CCV(ccv, t_maxseg); switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { - if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; - CCV(ccv, snd_cwnd) = cwin; - } + if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) + CCV(ccv, snd_ssthresh) = win; ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; - CCV(ccv, snd_cwnd) = cwin; + CCV(ccv, snd_ssthresh) = win; + CCV(ccv, snd_cwnd) = win; ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } - break; - case CC_RTO: - CCV(ccv, snd_ssthresh) = ssthresh_on_loss; - CCV(ccv, snd_cwnd) = mss; break; } } /* * Perform any necessary tasks before we exit congestion recovery. */ static void newreno_post_recovery(struct cc_var *ccv) { int pipe; pipe = 0; if (IN_FASTRECOVERY(CCV(ccv, t_flags))) { /* * Fast recovery will conclude after returning from this * function. Window inflation should have left us with * approximately snd_ssthresh outstanding data. But in case we * would be inclined to send a burst, better to do it via the * slow start mechanism. * * XXXLAS: Find a way to do this without needing curack */ if (V_tcp_do_rfc6675_pipe) pipe = tcp_compute_pipe(ccv->ccvc.tcp); else pipe = CCV(ccv, snd_max) - ccv->curack; if (pipe < CCV(ccv, snd_ssthresh)) CCV(ccv, snd_cwnd) = pipe + CCV(ccv, t_maxseg); else CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); } } DECLARE_CC_MODULE(newreno, &newreno_cc_algo); Index: projects/runtime-coverage/sys/netinet/sctp_indata.c =================================================================== --- projects/runtime-coverage/sys/netinet/sctp_indata.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/sctp_indata.c (revision 321585) @@ -1,5731 +1,5730 @@ /*- * Copyright (c) 2001-2007, by Cisco Systems, Inc. All rights reserved. * Copyright (c) 2008-2012, by Randall Stewart. All rights reserved. * Copyright (c) 2008-2012, by Michael Tuexen. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * a) Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * b) Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the distribution. * * c) Neither the name of Cisco Systems, Inc. nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * NOTES: On the outbound side of things I need to check the sack timer to * see if I should generate a sack into the chunk queue (if I have data to * send that is and will be sending it .. for bundling. * * The callback in sctp_usrreq.c will get called when the socket is read from. * This will cause sctp_service_queues() to get called on the top entry in * the list. */ static uint32_t sctp_add_chk_to_control(struct sctp_queued_to_read *control, struct sctp_stream_in *strm, struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_tmit_chunk *chk, int lock_held); void sctp_set_rwnd(struct sctp_tcb *stcb, struct sctp_association *asoc) { asoc->my_rwnd = sctp_calc_rwnd(stcb, asoc); } /* Calculate what the rwnd would be */ uint32_t sctp_calc_rwnd(struct sctp_tcb *stcb, struct sctp_association *asoc) { uint32_t calc = 0; /* * This is really set wrong with respect to a 1-2-m socket. Since * the sb_cc is the count that everyone as put up. When we re-write * sctp_soreceive then we will fix this so that ONLY this * associations data is taken into account. */ if (stcb->sctp_socket == NULL) { return (calc); } if (stcb->asoc.sb_cc == 0 && asoc->size_on_reasm_queue == 0 && asoc->size_on_all_streams == 0) { /* Full rwnd granted */ KASSERT(asoc->cnt_on_reasm_queue == 0, ("cnt_on_reasm_queue is %u", asoc->cnt_on_reasm_queue)); KASSERT(asoc->cnt_on_all_streams == 0, ("cnt_on_all_streams is %u", asoc->cnt_on_all_streams)); calc = max(SCTP_SB_LIMIT_RCV(stcb->sctp_socket), SCTP_MINIMAL_RWND); return (calc); } /* get actual space */ calc = (uint32_t)sctp_sbspace(&stcb->asoc, &stcb->sctp_socket->so_rcv); /* * take out what has NOT been put on socket queue and we yet hold * for putting up. */ calc = sctp_sbspace_sub(calc, (uint32_t)(asoc->size_on_reasm_queue + asoc->cnt_on_reasm_queue * MSIZE)); calc = sctp_sbspace_sub(calc, (uint32_t)(asoc->size_on_all_streams + asoc->cnt_on_all_streams * MSIZE)); if (calc == 0) { /* out of space */ return (calc); } /* what is the overhead of all these rwnd's */ calc = sctp_sbspace_sub(calc, stcb->asoc.my_rwnd_control_len); /* * If the window gets too small due to ctrl-stuff, reduce it to 1, * even it is 0. SWS engaged */ if (calc < stcb->asoc.my_rwnd_control_len) { calc = 1; } return (calc); } /* * Build out our readq entry based on the incoming packet. */ struct sctp_queued_to_read * sctp_build_readq_entry(struct sctp_tcb *stcb, struct sctp_nets *net, uint32_t tsn, uint32_t ppid, uint32_t context, uint16_t sid, uint32_t mid, uint8_t flags, struct mbuf *dm) { struct sctp_queued_to_read *read_queue_e = NULL; sctp_alloc_a_readq(stcb, read_queue_e); if (read_queue_e == NULL) { goto failed_build; } memset(read_queue_e, 0, sizeof(struct sctp_queued_to_read)); read_queue_e->sinfo_stream = sid; read_queue_e->sinfo_flags = (flags << 8); read_queue_e->sinfo_ppid = ppid; read_queue_e->sinfo_context = context; read_queue_e->sinfo_tsn = tsn; read_queue_e->sinfo_cumtsn = tsn; read_queue_e->sinfo_assoc_id = sctp_get_associd(stcb); read_queue_e->mid = mid; read_queue_e->top_fsn = read_queue_e->fsn_included = 0xffffffff; TAILQ_INIT(&read_queue_e->reasm); read_queue_e->whoFrom = net; atomic_add_int(&net->ref_count, 1); read_queue_e->data = dm; read_queue_e->stcb = stcb; read_queue_e->port_from = stcb->rport; failed_build: return (read_queue_e); } struct mbuf * sctp_build_ctl_nchunk(struct sctp_inpcb *inp, struct sctp_sndrcvinfo *sinfo) { struct sctp_extrcvinfo *seinfo; struct sctp_sndrcvinfo *outinfo; struct sctp_rcvinfo *rcvinfo; struct sctp_nxtinfo *nxtinfo; struct cmsghdr *cmh; struct mbuf *ret; int len; int use_extended; int provide_nxt; if (sctp_is_feature_off(inp, SCTP_PCB_FLAGS_RECVDATAIOEVNT) && sctp_is_feature_off(inp, SCTP_PCB_FLAGS_RECVRCVINFO) && sctp_is_feature_off(inp, SCTP_PCB_FLAGS_RECVNXTINFO)) { /* user does not want any ancillary data */ return (NULL); } len = 0; if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_RECVRCVINFO)) { len += CMSG_SPACE(sizeof(struct sctp_rcvinfo)); } seinfo = (struct sctp_extrcvinfo *)sinfo; if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_RECVNXTINFO) && (seinfo->serinfo_next_flags & SCTP_NEXT_MSG_AVAIL)) { provide_nxt = 1; len += CMSG_SPACE(sizeof(struct sctp_nxtinfo)); } else { provide_nxt = 0; } if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_RECVDATAIOEVNT)) { if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_EXT_RCVINFO)) { use_extended = 1; len += CMSG_SPACE(sizeof(struct sctp_extrcvinfo)); } else { use_extended = 0; len += CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); } } else { use_extended = 0; } ret = sctp_get_mbuf_for_msg(len, 0, M_NOWAIT, 1, MT_DATA); if (ret == NULL) { /* No space */ return (ret); } SCTP_BUF_LEN(ret) = 0; /* We need a CMSG header followed by the struct */ cmh = mtod(ret, struct cmsghdr *); /* * Make sure that there is no un-initialized padding between the * cmsg header and cmsg data and after the cmsg data. */ memset(cmh, 0, len); if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_RECVRCVINFO)) { cmh->cmsg_level = IPPROTO_SCTP; cmh->cmsg_len = CMSG_LEN(sizeof(struct sctp_rcvinfo)); cmh->cmsg_type = SCTP_RCVINFO; rcvinfo = (struct sctp_rcvinfo *)CMSG_DATA(cmh); rcvinfo->rcv_sid = sinfo->sinfo_stream; rcvinfo->rcv_ssn = sinfo->sinfo_ssn; rcvinfo->rcv_flags = sinfo->sinfo_flags; rcvinfo->rcv_ppid = sinfo->sinfo_ppid; rcvinfo->rcv_tsn = sinfo->sinfo_tsn; rcvinfo->rcv_cumtsn = sinfo->sinfo_cumtsn; rcvinfo->rcv_context = sinfo->sinfo_context; rcvinfo->rcv_assoc_id = sinfo->sinfo_assoc_id; cmh = (struct cmsghdr *)((caddr_t)cmh + CMSG_SPACE(sizeof(struct sctp_rcvinfo))); SCTP_BUF_LEN(ret) += CMSG_SPACE(sizeof(struct sctp_rcvinfo)); } if (provide_nxt) { cmh->cmsg_level = IPPROTO_SCTP; cmh->cmsg_len = CMSG_LEN(sizeof(struct sctp_nxtinfo)); cmh->cmsg_type = SCTP_NXTINFO; nxtinfo = (struct sctp_nxtinfo *)CMSG_DATA(cmh); nxtinfo->nxt_sid = seinfo->serinfo_next_stream; nxtinfo->nxt_flags = 0; if (seinfo->serinfo_next_flags & SCTP_NEXT_MSG_IS_UNORDERED) { nxtinfo->nxt_flags |= SCTP_UNORDERED; } if (seinfo->serinfo_next_flags & SCTP_NEXT_MSG_IS_NOTIFICATION) { nxtinfo->nxt_flags |= SCTP_NOTIFICATION; } if (seinfo->serinfo_next_flags & SCTP_NEXT_MSG_ISCOMPLETE) { nxtinfo->nxt_flags |= SCTP_COMPLETE; } nxtinfo->nxt_ppid = seinfo->serinfo_next_ppid; nxtinfo->nxt_length = seinfo->serinfo_next_length; nxtinfo->nxt_assoc_id = seinfo->serinfo_next_aid; cmh = (struct cmsghdr *)((caddr_t)cmh + CMSG_SPACE(sizeof(struct sctp_nxtinfo))); SCTP_BUF_LEN(ret) += CMSG_SPACE(sizeof(struct sctp_nxtinfo)); } if (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_RECVDATAIOEVNT)) { cmh->cmsg_level = IPPROTO_SCTP; outinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmh); if (use_extended) { cmh->cmsg_len = CMSG_LEN(sizeof(struct sctp_extrcvinfo)); cmh->cmsg_type = SCTP_EXTRCV; memcpy(outinfo, sinfo, sizeof(struct sctp_extrcvinfo)); SCTP_BUF_LEN(ret) += CMSG_SPACE(sizeof(struct sctp_extrcvinfo)); } else { cmh->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); cmh->cmsg_type = SCTP_SNDRCV; *outinfo = *sinfo; SCTP_BUF_LEN(ret) += CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); } } return (ret); } static void sctp_mark_non_revokable(struct sctp_association *asoc, uint32_t tsn) { uint32_t gap, i, cumackp1; int fnd = 0; int in_r = 0, in_nr = 0; if (SCTP_BASE_SYSCTL(sctp_do_drain) == 0) { return; } cumackp1 = asoc->cumulative_tsn + 1; if (SCTP_TSN_GT(cumackp1, tsn)) { /* * this tsn is behind the cum ack and thus we don't need to * worry about it being moved from one to the other. */ return; } SCTP_CALC_TSN_TO_GAP(gap, tsn, asoc->mapping_array_base_tsn); in_r = SCTP_IS_TSN_PRESENT(asoc->mapping_array, gap); in_nr = SCTP_IS_TSN_PRESENT(asoc->nr_mapping_array, gap); if ((in_r == 0) && (in_nr == 0)) { #ifdef INVARIANTS panic("Things are really messed up now"); #else SCTP_PRINTF("gap:%x tsn:%x\n", gap, tsn); sctp_print_mapping_array(asoc); #endif } if (in_nr == 0) SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, gap); if (in_r) SCTP_UNSET_TSN_PRESENT(asoc->mapping_array, gap); if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_nr_map)) { asoc->highest_tsn_inside_nr_map = tsn; } if (tsn == asoc->highest_tsn_inside_map) { /* We must back down to see what the new highest is */ for (i = tsn - 1; SCTP_TSN_GE(i, asoc->mapping_array_base_tsn); i--) { SCTP_CALC_TSN_TO_GAP(gap, i, asoc->mapping_array_base_tsn); if (SCTP_IS_TSN_PRESENT(asoc->mapping_array, gap)) { asoc->highest_tsn_inside_map = i; fnd = 1; break; } } if (!fnd) { asoc->highest_tsn_inside_map = asoc->mapping_array_base_tsn - 1; } } } static int sctp_place_control_in_stream(struct sctp_stream_in *strm, struct sctp_association *asoc, struct sctp_queued_to_read *control) { struct sctp_queued_to_read *at; struct sctp_readhead *q; uint8_t flags, unordered; flags = (control->sinfo_flags >> 8); unordered = flags & SCTP_DATA_UNORDERED; if (unordered) { q = &strm->uno_inqueue; if (asoc->idata_supported == 0) { if (!TAILQ_EMPTY(q)) { /* * Only one stream can be here in old style * -- abort */ return (-1); } TAILQ_INSERT_TAIL(q, control, next_instrm); control->on_strm_q = SCTP_ON_UNORDERED; return (0); } } else { q = &strm->inqueue; } if ((flags & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { control->end_added = 1; control->first_frag_seen = 1; control->last_frag_seen = 1; } if (TAILQ_EMPTY(q)) { /* Empty queue */ TAILQ_INSERT_HEAD(q, control, next_instrm); if (unordered) { control->on_strm_q = SCTP_ON_UNORDERED; } else { control->on_strm_q = SCTP_ON_ORDERED; } return (0); } else { TAILQ_FOREACH(at, q, next_instrm) { if (SCTP_MID_GT(asoc->idata_supported, at->mid, control->mid)) { /* * one in queue is bigger than the new one, * insert before this one */ TAILQ_INSERT_BEFORE(at, control, next_instrm); if (unordered) { control->on_strm_q = SCTP_ON_UNORDERED; } else { control->on_strm_q = SCTP_ON_ORDERED; } break; } else if (SCTP_MID_EQ(asoc->idata_supported, at->mid, control->mid)) { /* * Gak, He sent me a duplicate msg id * number?? return -1 to abort. */ return (-1); } else { if (TAILQ_NEXT(at, next_instrm) == NULL) { /* * We are at the end, insert it * after this one */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del(control, at, SCTP_STR_LOG_FROM_INSERT_TL); } TAILQ_INSERT_AFTER(q, at, control, next_instrm); if (unordered) { control->on_strm_q = SCTP_ON_UNORDERED; } else { control->on_strm_q = SCTP_ON_ORDERED; } break; } } } } return (0); } static void sctp_abort_in_reasm(struct sctp_tcb *stcb, struct sctp_queued_to_read *control, struct sctp_tmit_chunk *chk, int *abort_flag, int opspot) { char msg[SCTP_DIAG_INFO_LEN]; struct mbuf *oper; if (stcb->asoc.idata_supported) { snprintf(msg, sizeof(msg), "Reass %x,CF:%x,TSN=%8.8x,SID=%4.4x,FSN=%8.8x,MID:%8.8x", opspot, control->fsn_included, chk->rec.data.tsn, chk->rec.data.sid, chk->rec.data.fsn, chk->rec.data.mid); } else { snprintf(msg, sizeof(msg), "Reass %x,CI:%x,TSN=%8.8x,SID=%4.4x,FSN=%4.4x,SSN:%4.4x", opspot, control->fsn_included, chk->rec.data.tsn, chk->rec.data.sid, chk->rec.data.fsn, (uint16_t)chk->rec.data.mid); } oper = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); sctp_m_freem(chk->data); chk->data = NULL; sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_1; sctp_abort_an_association(stcb->sctp_ep, stcb, oper, SCTP_SO_NOT_LOCKED); *abort_flag = 1; } static void sctp_clean_up_control(struct sctp_tcb *stcb, struct sctp_queued_to_read *control) { /* * The control could not be placed and must be cleaned. */ struct sctp_tmit_chunk *chk, *nchk; TAILQ_FOREACH_SAFE(chk, &control->reasm, sctp_next, nchk) { TAILQ_REMOVE(&control->reasm, chk, sctp_next); if (chk->data) sctp_m_freem(chk->data); chk->data = NULL; sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); } sctp_free_a_readq(stcb, control); } /* * Queue the chunk either right into the socket buffer if it is the next one * to go OR put it in the correct place in the delivery queue. If we do * append to the so_buf, keep doing so until we are out of order as * long as the control's entered are non-fragmented. */ static void sctp_queue_data_to_stream(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_queued_to_read *control, int *abort_flag, int *need_reasm) { /* * FIX-ME maybe? What happens when the ssn wraps? If we are getting * all the data in one stream this could happen quite rapidly. One * could use the TSN to keep track of things, but this scheme breaks * down in the other type of stream usage that could occur. Send a * single msg to stream 0, send 4Billion messages to stream 1, now * send a message to stream 0. You have a situation where the TSN * has wrapped but not in the stream. Is this worth worrying about * or should we just change our queue sort at the bottom to be by * TSN. * * Could it also be legal for a peer to send ssn 1 with TSN 2 and * ssn 2 with TSN 1? If the peer is doing some sort of funky TSN/SSN * assignment this could happen... and I don't see how this would be * a violation. So for now I am undecided an will leave the sort by * SSN alone. Maybe a hybred approach is the answer * */ struct sctp_queued_to_read *at; int queue_needed; uint32_t nxt_todel; struct mbuf *op_err; struct sctp_stream_in *strm; char msg[SCTP_DIAG_INFO_LEN]; strm = &asoc->strmin[control->sinfo_stream]; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del(control, NULL, SCTP_STR_LOG_FROM_INTO_STRD); } if (SCTP_MID_GT((asoc->idata_supported), strm->last_mid_delivered, control->mid)) { /* The incoming sseq is behind where we last delivered? */ SCTPDBG(SCTP_DEBUG_INDATA1, "Duplicate S-SEQ: %u delivered: %u from peer, Abort association\n", strm->last_mid_delivered, control->mid); /* * throw it in the stream so it gets cleaned up in * association destruction */ TAILQ_INSERT_HEAD(&strm->inqueue, control, next_instrm); if (asoc->idata_supported) { snprintf(msg, sizeof(msg), "Delivered MID=%8.8x, got TSN=%8.8x, SID=%4.4x, MID=%8.8x", strm->last_mid_delivered, control->sinfo_tsn, control->sinfo_stream, control->mid); } else { snprintf(msg, sizeof(msg), "Delivered SSN=%4.4x, got TSN=%8.8x, SID=%4.4x, SSN=%4.4x", (uint16_t)strm->last_mid_delivered, control->sinfo_tsn, control->sinfo_stream, (uint16_t)control->mid); } op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_2; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; return; } queue_needed = 1; asoc->size_on_all_streams += control->length; sctp_ucount_incr(asoc->cnt_on_all_streams); nxt_todel = strm->last_mid_delivered + 1; if (SCTP_MID_EQ(asoc->idata_supported, nxt_todel, control->mid)) { #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) struct socket *so; so = SCTP_INP_SO(stcb->sctp_ep); atomic_add_int(&stcb->asoc.refcnt, 1); SCTP_TCB_UNLOCK(stcb); SCTP_SOCKET_LOCK(so, 1); SCTP_TCB_LOCK(stcb); atomic_subtract_int(&stcb->asoc.refcnt, 1); if (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE) { SCTP_SOCKET_UNLOCK(so, 1); return; } #endif /* can be delivered right away? */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del(control, NULL, SCTP_STR_LOG_FROM_IMMED_DEL); } /* EY it wont be queued if it could be delivered directly */ queue_needed = 0; if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); strm->last_mid_delivered++; sctp_mark_non_revokable(asoc, control->sinfo_tsn); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_NOT_HELD, SCTP_SO_LOCKED); TAILQ_FOREACH_SAFE(control, &strm->inqueue, next_instrm, at) { /* all delivered */ nxt_todel = strm->last_mid_delivered + 1; if (SCTP_MID_EQ(asoc->idata_supported, nxt_todel, control->mid) && (((control->sinfo_flags >> 8) & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG)) { if (control->on_strm_q == SCTP_ON_ORDERED) { TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); #ifdef INVARIANTS } else { panic("Huh control: %p is on_strm_q: %d", control, control->on_strm_q); #endif } control->on_strm_q = 0; strm->last_mid_delivered++; /* * We ignore the return of deliver_data here * since we always can hold the chunk on the * d-queue. And we have a finite number that * can be delivered from the strq. */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del(control, NULL, SCTP_STR_LOG_FROM_IMMED_DEL); } sctp_mark_non_revokable(asoc, control->sinfo_tsn); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_NOT_HELD, SCTP_SO_LOCKED); continue; } else if (SCTP_MID_EQ(asoc->idata_supported, nxt_todel, control->mid)) { *need_reasm = 1; } break; } #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) SCTP_SOCKET_UNLOCK(so, 1); #endif } if (queue_needed) { /* * Ok, we did not deliver this guy, find the correct place * to put it on the queue. */ if (sctp_place_control_in_stream(strm, asoc, control)) { snprintf(msg, sizeof(msg), "Queue to str MID: %u duplicate", control->mid); sctp_clean_up_control(stcb, control); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_3; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; } } } static void sctp_setup_tail_pointer(struct sctp_queued_to_read *control) { struct mbuf *m, *prev = NULL; struct sctp_tcb *stcb; stcb = control->stcb; control->held_length = 0; control->length = 0; m = control->data; while (m) { if (SCTP_BUF_LEN(m) == 0) { /* Skip mbufs with NO length */ if (prev == NULL) { /* First one */ control->data = sctp_m_free(m); m = control->data; } else { SCTP_BUF_NEXT(prev) = sctp_m_free(m); m = SCTP_BUF_NEXT(prev); } if (m == NULL) { control->tail_mbuf = prev; } continue; } prev = m; atomic_add_int(&control->length, SCTP_BUF_LEN(m)); if (control->on_read_q) { /* * On read queue so we must increment the SB stuff, * we assume caller has done any locks of SB. */ sctp_sballoc(stcb, &stcb->sctp_socket->so_rcv, m); } m = SCTP_BUF_NEXT(m); } if (prev) { control->tail_mbuf = prev; } } static void sctp_add_to_tail_pointer(struct sctp_queued_to_read *control, struct mbuf *m, uint32_t *added) { struct mbuf *prev = NULL; struct sctp_tcb *stcb; stcb = control->stcb; if (stcb == NULL) { #ifdef INVARIANTS panic("Control broken"); #else return; #endif } if (control->tail_mbuf == NULL) { /* TSNH */ control->data = m; sctp_setup_tail_pointer(control); return; } control->tail_mbuf->m_next = m; while (m) { if (SCTP_BUF_LEN(m) == 0) { /* Skip mbufs with NO length */ if (prev == NULL) { /* First one */ control->tail_mbuf->m_next = sctp_m_free(m); m = control->tail_mbuf->m_next; } else { SCTP_BUF_NEXT(prev) = sctp_m_free(m); m = SCTP_BUF_NEXT(prev); } if (m == NULL) { control->tail_mbuf = prev; } continue; } prev = m; if (control->on_read_q) { /* * On read queue so we must increment the SB stuff, * we assume caller has done any locks of SB. */ sctp_sballoc(stcb, &stcb->sctp_socket->so_rcv, m); } *added += SCTP_BUF_LEN(m); atomic_add_int(&control->length, SCTP_BUF_LEN(m)); m = SCTP_BUF_NEXT(m); } if (prev) { control->tail_mbuf = prev; } } static void sctp_build_readq_entry_from_ctl(struct sctp_queued_to_read *nc, struct sctp_queued_to_read *control) { memset(nc, 0, sizeof(struct sctp_queued_to_read)); nc->sinfo_stream = control->sinfo_stream; nc->mid = control->mid; TAILQ_INIT(&nc->reasm); nc->top_fsn = control->top_fsn; nc->mid = control->mid; nc->sinfo_flags = control->sinfo_flags; nc->sinfo_ppid = control->sinfo_ppid; nc->sinfo_context = control->sinfo_context; nc->fsn_included = 0xffffffff; nc->sinfo_tsn = control->sinfo_tsn; nc->sinfo_cumtsn = control->sinfo_cumtsn; nc->sinfo_assoc_id = control->sinfo_assoc_id; nc->whoFrom = control->whoFrom; atomic_add_int(&nc->whoFrom->ref_count, 1); nc->stcb = control->stcb; nc->port_from = control->port_from; } static void sctp_reset_a_control(struct sctp_queued_to_read *control, struct sctp_inpcb *inp, uint32_t tsn) { control->fsn_included = tsn; if (control->on_read_q) { /* * We have to purge it from there, hopefully this will work * :-) */ TAILQ_REMOVE(&inp->read_queue, control, next); control->on_read_q = 0; } } static int sctp_handle_old_unordered_data(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_stream_in *strm, struct sctp_queued_to_read *control, uint32_t pd_point, int inp_read_lock_held) { /* * Special handling for the old un-ordered data chunk. All the * chunks/TSN's go to mid 0. So we have to do the old style watching * to see if we have it all. If you return one, no other control * entries on the un-ordered queue will be looked at. In theory * there should be no others entries in reality, unless the guy is * sending both unordered NDATA and unordered DATA... */ struct sctp_tmit_chunk *chk, *lchk, *tchk; uint32_t fsn; struct sctp_queued_to_read *nc; int cnt_added; if (control->first_frag_seen == 0) { /* Nothing we can do, we have not seen the first piece yet */ return (1); } /* Collapse any we can */ cnt_added = 0; restart: fsn = control->fsn_included + 1; /* Now what can we add? */ TAILQ_FOREACH_SAFE(chk, &control->reasm, sctp_next, lchk) { if (chk->rec.data.fsn == fsn) { /* Ok lets add it */ sctp_alloc_a_readq(stcb, nc); if (nc == NULL) { break; } memset(nc, 0, sizeof(struct sctp_queued_to_read)); TAILQ_REMOVE(&control->reasm, chk, sctp_next); sctp_add_chk_to_control(control, strm, stcb, asoc, chk, SCTP_READ_LOCK_NOT_HELD); fsn++; cnt_added++; chk = NULL; if (control->end_added) { /* We are done */ if (!TAILQ_EMPTY(&control->reasm)) { /* * Ok we have to move anything left * on the control queue to a new * control. */ sctp_build_readq_entry_from_ctl(nc, control); tchk = TAILQ_FIRST(&control->reasm); if (tchk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) { TAILQ_REMOVE(&control->reasm, tchk, sctp_next); if (asoc->size_on_reasm_queue >= tchk->send_size) { asoc->size_on_reasm_queue -= tchk->send_size; } else { #ifdef INVARIANTS panic("size_on_reasm_queue = %u smaller than chunk length %u", asoc->size_on_reasm_queue, tchk->send_size); #else asoc->size_on_reasm_queue = 0; #endif } sctp_ucount_decr(asoc->cnt_on_reasm_queue); nc->first_frag_seen = 1; nc->fsn_included = tchk->rec.data.fsn; nc->data = tchk->data; nc->sinfo_ppid = tchk->rec.data.ppid; nc->sinfo_tsn = tchk->rec.data.tsn; sctp_mark_non_revokable(asoc, tchk->rec.data.tsn); tchk->data = NULL; sctp_free_a_chunk(stcb, tchk, SCTP_SO_NOT_LOCKED); sctp_setup_tail_pointer(nc); tchk = TAILQ_FIRST(&control->reasm); } /* Spin the rest onto the queue */ while (tchk) { TAILQ_REMOVE(&control->reasm, tchk, sctp_next); TAILQ_INSERT_TAIL(&nc->reasm, tchk, sctp_next); tchk = TAILQ_FIRST(&control->reasm); } /* * Now lets add it to the queue * after removing control */ TAILQ_INSERT_TAIL(&strm->uno_inqueue, nc, next_instrm); nc->on_strm_q = SCTP_ON_UNORDERED; if (control->on_strm_q) { TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); control->on_strm_q = 0; } } if (control->pdapi_started) { strm->pd_api_started = 0; control->pdapi_started = 0; } if (control->on_strm_q) { TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); control->on_strm_q = 0; SCTP_STAT_INCR_COUNTER64(sctps_reasmusrmsgs); } if (control->on_read_q == 0) { sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); } sctp_wakeup_the_read_socket(stcb->sctp_ep, stcb, SCTP_SO_NOT_LOCKED); if ((nc->first_frag_seen) && !TAILQ_EMPTY(&nc->reasm)) { /* * Switch to the new guy and * continue */ control = nc; goto restart; } else { if (nc->on_strm_q == 0) { sctp_free_a_readq(stcb, nc); } } return (1); } else { sctp_free_a_readq(stcb, nc); } } else { /* Can't add more */ break; } } if ((control->length > pd_point) && (strm->pd_api_started == 0)) { strm->pd_api_started = 1; control->pdapi_started = 1; sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); sctp_wakeup_the_read_socket(stcb->sctp_ep, stcb, SCTP_SO_NOT_LOCKED); return (0); } else { return (1); } } static void sctp_inject_old_unordered_data(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_queued_to_read *control, struct sctp_tmit_chunk *chk, int *abort_flag) { struct sctp_tmit_chunk *at; int inserted; /* * Here we need to place the chunk into the control structure sorted * in the correct order. */ if (chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) { /* Its the very first one. */ SCTPDBG(SCTP_DEBUG_XXX, "chunk is a first fsn: %u becomes fsn_included\n", chk->rec.data.fsn); if (control->first_frag_seen) { /* * In old un-ordered we can reassembly on one * control multiple messages. As long as the next * FIRST is greater then the old first (TSN i.e. FSN * wise) */ struct mbuf *tdata; uint32_t tmp; if (SCTP_TSN_GT(chk->rec.data.fsn, control->fsn_included)) { /* * Easy way the start of a new guy beyond * the lowest */ goto place_chunk; } if ((chk->rec.data.fsn == control->fsn_included) || (control->pdapi_started)) { /* * Ok this should not happen, if it does we * started the pd-api on the higher TSN * (since the equals part is a TSN failure * it must be that). * * We are completly hosed in that case since * I have no way to recover. This really * will only happen if we can get more TSN's * higher before the pd-api-point. */ sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_4); return; } /* * Ok we have two firsts and the one we just got is * smaller than the one we previously placed.. yuck! * We must swap them out. */ /* swap the mbufs */ tdata = control->data; control->data = chk->data; chk->data = tdata; /* Save the lengths */ chk->send_size = control->length; /* Recompute length of control and tail pointer */ sctp_setup_tail_pointer(control); /* Fix the FSN included */ tmp = control->fsn_included; control->fsn_included = chk->rec.data.fsn; chk->rec.data.fsn = tmp; /* Fix the TSN included */ tmp = control->sinfo_tsn; control->sinfo_tsn = chk->rec.data.tsn; chk->rec.data.tsn = tmp; /* Fix the PPID included */ tmp = control->sinfo_ppid; control->sinfo_ppid = chk->rec.data.ppid; chk->rec.data.ppid = tmp; /* Fix tail pointer */ goto place_chunk; } control->first_frag_seen = 1; control->fsn_included = chk->rec.data.fsn; control->top_fsn = chk->rec.data.fsn; control->sinfo_tsn = chk->rec.data.tsn; control->sinfo_ppid = chk->rec.data.ppid; control->data = chk->data; sctp_mark_non_revokable(asoc, chk->rec.data.tsn); chk->data = NULL; sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); sctp_setup_tail_pointer(control); return; } place_chunk: inserted = 0; TAILQ_FOREACH(at, &control->reasm, sctp_next) { if (SCTP_TSN_GT(at->rec.data.fsn, chk->rec.data.fsn)) { /* * This one in queue is bigger than the new one, * insert the new one before at. */ asoc->size_on_reasm_queue += chk->send_size; sctp_ucount_incr(asoc->cnt_on_reasm_queue); inserted = 1; TAILQ_INSERT_BEFORE(at, chk, sctp_next); break; } else if (at->rec.data.fsn == chk->rec.data.fsn) { /* * They sent a duplicate fsn number. This really * should not happen since the FSN is a TSN and it * should have been dropped earlier. */ sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_5); return; } } if (inserted == 0) { /* Its at the end */ asoc->size_on_reasm_queue += chk->send_size; sctp_ucount_incr(asoc->cnt_on_reasm_queue); control->top_fsn = chk->rec.data.fsn; TAILQ_INSERT_TAIL(&control->reasm, chk, sctp_next); } } static int sctp_deliver_reasm_check(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_stream_in *strm, int inp_read_lock_held) { /* * Given a stream, strm, see if any of the SSN's on it that are * fragmented are ready to deliver. If so go ahead and place them on * the read queue. In so placing if we have hit the end, then we * need to remove them from the stream's queue. */ struct sctp_queued_to_read *control, *nctl = NULL; uint32_t next_to_del; uint32_t pd_point; int ret = 0; if (stcb->sctp_socket) { pd_point = min(SCTP_SB_LIMIT_RCV(stcb->sctp_socket) >> SCTP_PARTIAL_DELIVERY_SHIFT, stcb->sctp_ep->partial_delivery_point); } else { pd_point = stcb->sctp_ep->partial_delivery_point; } control = TAILQ_FIRST(&strm->uno_inqueue); if ((control != NULL) && (asoc->idata_supported == 0)) { /* Special handling needed for "old" data format */ if (sctp_handle_old_unordered_data(stcb, asoc, strm, control, pd_point, inp_read_lock_held)) { goto done_un; } } if (strm->pd_api_started) { /* Can't add more */ return (0); } while (control) { SCTPDBG(SCTP_DEBUG_XXX, "Looking at control: %p e(%d) ssn: %u top_fsn: %u inc_fsn: %u -uo\n", control, control->end_added, control->mid, control->top_fsn, control->fsn_included); nctl = TAILQ_NEXT(control, next_instrm); if (control->end_added) { /* We just put the last bit on */ if (control->on_strm_q) { #ifdef INVARIANTS if (control->on_strm_q != SCTP_ON_UNORDERED) { panic("Huh control: %p on_q: %d -- not unordered?", control, control->on_strm_q); } #endif SCTP_STAT_INCR_COUNTER64(sctps_reasmusrmsgs); TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); control->on_strm_q = 0; } if (control->on_read_q == 0) { sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); } } else { /* Can we do a PD-API for this un-ordered guy? */ if ((control->length >= pd_point) && (strm->pd_api_started == 0)) { strm->pd_api_started = 1; control->pdapi_started = 1; sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); break; } } control = nctl; } done_un: control = TAILQ_FIRST(&strm->inqueue); if (strm->pd_api_started) { /* Can't add more */ return (0); } if (control == NULL) { return (ret); } if (SCTP_MID_EQ(asoc->idata_supported, strm->last_mid_delivered, control->mid)) { /* * Ok the guy at the top was being partially delivered * completed, so we remove it. Note the pd_api flag was * taken off when the chunk was merged on in * sctp_queue_data_for_reasm below. */ nctl = TAILQ_NEXT(control, next_instrm); SCTPDBG(SCTP_DEBUG_XXX, "Looking at control: %p e(%d) ssn: %u top_fsn: %u inc_fsn: %u (lastdel: %u)- o\n", control, control->end_added, control->mid, control->top_fsn, control->fsn_included, strm->last_mid_delivered); if (control->end_added) { if (control->on_strm_q) { #ifdef INVARIANTS if (control->on_strm_q != SCTP_ON_ORDERED) { panic("Huh control: %p on_q: %d -- not ordered?", control, control->on_strm_q); } #endif SCTP_STAT_INCR_COUNTER64(sctps_reasmusrmsgs); TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); control->on_strm_q = 0; } if (strm->pd_api_started && control->pdapi_started) { control->pdapi_started = 0; strm->pd_api_started = 0; } if (control->on_read_q == 0) { sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); } control = nctl; } } if (strm->pd_api_started) { /* * Can't add more must have gotten an un-ordered above being * partially delivered. */ return (0); } deliver_more: next_to_del = strm->last_mid_delivered + 1; if (control) { SCTPDBG(SCTP_DEBUG_XXX, "Looking at control: %p e(%d) ssn: %u top_fsn: %u inc_fsn: %u (nxtdel: %u)- o\n", control, control->end_added, control->mid, control->top_fsn, control->fsn_included, next_to_del); nctl = TAILQ_NEXT(control, next_instrm); if (SCTP_MID_EQ(asoc->idata_supported, control->mid, next_to_del) && (control->first_frag_seen)) { int done; /* Ok we can deliver it onto the stream. */ if (control->end_added) { /* We are done with it afterwards */ if (control->on_strm_q) { #ifdef INVARIANTS if (control->on_strm_q != SCTP_ON_ORDERED) { panic("Huh control: %p on_q: %d -- not ordered?", control, control->on_strm_q); } #endif SCTP_STAT_INCR_COUNTER64(sctps_reasmusrmsgs); TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); control->on_strm_q = 0; } ret++; } if (((control->sinfo_flags >> 8) & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { /* * A singleton now slipping through - mark * it non-revokable too */ sctp_mark_non_revokable(asoc, control->sinfo_tsn); } else if (control->end_added == 0) { /* * Check if we can defer adding until its * all there */ if ((control->length < pd_point) || (strm->pd_api_started)) { /* * Don't need it or cannot add more * (one being delivered that way) */ goto out; } } done = (control->end_added) && (control->last_frag_seen); if (control->on_read_q == 0) { sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, inp_read_lock_held, SCTP_SO_NOT_LOCKED); } strm->last_mid_delivered = next_to_del; if (done) { control = nctl; goto deliver_more; } else { /* We are now doing PD API */ strm->pd_api_started = 1; control->pdapi_started = 1; } } } out: return (ret); } uint32_t sctp_add_chk_to_control(struct sctp_queued_to_read *control, struct sctp_stream_in *strm, struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_tmit_chunk *chk, int hold_rlock) { /* * Given a control and a chunk, merge the data from the chk onto the * control and free up the chunk resources. */ uint32_t added = 0; int i_locked = 0; if (control->on_read_q && (hold_rlock == 0)) { /* * Its being pd-api'd so we must do some locks. */ SCTP_INP_READ_LOCK(stcb->sctp_ep); i_locked = 1; } if (control->data == NULL) { control->data = chk->data; sctp_setup_tail_pointer(control); } else { sctp_add_to_tail_pointer(control, chk->data, &added); } control->fsn_included = chk->rec.data.fsn; asoc->size_on_reasm_queue -= chk->send_size; sctp_ucount_decr(asoc->cnt_on_reasm_queue); sctp_mark_non_revokable(asoc, chk->rec.data.tsn); chk->data = NULL; if (chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) { control->first_frag_seen = 1; control->sinfo_tsn = chk->rec.data.tsn; control->sinfo_ppid = chk->rec.data.ppid; } if (chk->rec.data.rcv_flags & SCTP_DATA_LAST_FRAG) { /* Its complete */ if ((control->on_strm_q) && (control->on_read_q)) { if (control->pdapi_started) { control->pdapi_started = 0; strm->pd_api_started = 0; } if (control->on_strm_q == SCTP_ON_UNORDERED) { /* Unordered */ TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); control->on_strm_q = 0; } else if (control->on_strm_q == SCTP_ON_ORDERED) { /* Ordered */ TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); control->on_strm_q = 0; #ifdef INVARIANTS } else if (control->on_strm_q) { panic("Unknown state on ctrl: %p on_strm_q: %d", control, control->on_strm_q); #endif } } control->end_added = 1; control->last_frag_seen = 1; } if (i_locked) { SCTP_INP_READ_UNLOCK(stcb->sctp_ep); } sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); return (added); } /* * Dump onto the re-assembly queue, in its proper place. After dumping on the * queue, see if anthing can be delivered. If so pull it off (or as much as * we can. If we run out of space then we must dump what we can and set the * appropriate flag to say we queued what we could. */ static void sctp_queue_data_for_reasm(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_queued_to_read *control, struct sctp_tmit_chunk *chk, int created_control, int *abort_flag, uint32_t tsn) { uint32_t next_fsn; struct sctp_tmit_chunk *at, *nat; struct sctp_stream_in *strm; int do_wakeup, unordered; uint32_t lenadded; strm = &asoc->strmin[control->sinfo_stream]; /* * For old un-ordered data chunks. */ if ((control->sinfo_flags >> 8) & SCTP_DATA_UNORDERED) { unordered = 1; } else { unordered = 0; } /* Must be added to the stream-in queue */ if (created_control) { if (unordered == 0) { sctp_ucount_incr(asoc->cnt_on_all_streams); } if (sctp_place_control_in_stream(strm, asoc, control)) { /* Duplicate SSN? */ sctp_clean_up_control(stcb, control); sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_6); return; } if ((tsn == (asoc->cumulative_tsn + 1) && (asoc->idata_supported == 0))) { /* * Ok we created this control and now lets validate * that its legal i.e. there is a B bit set, if not * and we have up to the cum-ack then its invalid. */ if ((chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) == 0) { sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_7); return; } } } if ((asoc->idata_supported == 0) && (unordered == 1)) { sctp_inject_old_unordered_data(stcb, asoc, control, chk, abort_flag); return; } /* * Ok we must queue the chunk into the reasembly portion: o if its * the first it goes to the control mbuf. o if its not first but the * next in sequence it goes to the control, and each succeeding one * in order also goes. o if its not in order we place it on the list * in its place. */ if (chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) { /* Its the very first one. */ SCTPDBG(SCTP_DEBUG_XXX, "chunk is a first fsn: %u becomes fsn_included\n", chk->rec.data.fsn); if (control->first_frag_seen) { /* * Error on senders part, they either sent us two * data chunks with FIRST, or they sent two * un-ordered chunks that were fragmented at the * same time in the same stream. */ sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_8); return; } control->first_frag_seen = 1; control->sinfo_ppid = chk->rec.data.ppid; control->sinfo_tsn = chk->rec.data.tsn; control->fsn_included = chk->rec.data.fsn; control->data = chk->data; sctp_mark_non_revokable(asoc, chk->rec.data.tsn); chk->data = NULL; sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); sctp_setup_tail_pointer(control); asoc->size_on_all_streams += control->length; } else { /* Place the chunk in our list */ int inserted = 0; if (control->last_frag_seen == 0) { /* Still willing to raise highest FSN seen */ if (SCTP_TSN_GT(chk->rec.data.fsn, control->top_fsn)) { SCTPDBG(SCTP_DEBUG_XXX, "We have a new top_fsn: %u\n", chk->rec.data.fsn); control->top_fsn = chk->rec.data.fsn; } if (chk->rec.data.rcv_flags & SCTP_DATA_LAST_FRAG) { SCTPDBG(SCTP_DEBUG_XXX, "The last fsn is now in place fsn: %u\n", chk->rec.data.fsn); control->last_frag_seen = 1; } if (asoc->idata_supported || control->first_frag_seen) { /* * For IDATA we always check since we know * that the first fragment is 0. For old * DATA we have to receive the first before * we know the first FSN (which is the TSN). */ if (SCTP_TSN_GE(control->fsn_included, chk->rec.data.fsn)) { /* * We have already delivered up to * this so its a dup */ sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_9); return; } } } else { if (chk->rec.data.rcv_flags & SCTP_DATA_LAST_FRAG) { /* Second last? huh? */ SCTPDBG(SCTP_DEBUG_XXX, "Duplicate last fsn: %u (top: %u) -- abort\n", chk->rec.data.fsn, control->top_fsn); sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_10); return; } if (asoc->idata_supported || control->first_frag_seen) { /* * For IDATA we always check since we know * that the first fragment is 0. For old * DATA we have to receive the first before * we know the first FSN (which is the TSN). */ if (SCTP_TSN_GE(control->fsn_included, chk->rec.data.fsn)) { /* * We have already delivered up to * this so its a dup */ SCTPDBG(SCTP_DEBUG_XXX, "New fsn: %u is already seen in included_fsn: %u -- abort\n", chk->rec.data.fsn, control->fsn_included); sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_11); return; } } /* * validate not beyond top FSN if we have seen last * one */ if (SCTP_TSN_GT(chk->rec.data.fsn, control->top_fsn)) { SCTPDBG(SCTP_DEBUG_XXX, "New fsn: %u is beyond or at top_fsn: %u -- abort\n", chk->rec.data.fsn, control->top_fsn); sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_12); return; } } /* * If we reach here, we need to place the new chunk in the * reassembly for this control. */ SCTPDBG(SCTP_DEBUG_XXX, "chunk is a not first fsn: %u needs to be inserted\n", chk->rec.data.fsn); TAILQ_FOREACH(at, &control->reasm, sctp_next) { if (SCTP_TSN_GT(at->rec.data.fsn, chk->rec.data.fsn)) { /* * This one in queue is bigger than the new * one, insert the new one before at. */ SCTPDBG(SCTP_DEBUG_XXX, "Insert it before fsn: %u\n", at->rec.data.fsn); asoc->size_on_reasm_queue += chk->send_size; sctp_ucount_incr(asoc->cnt_on_reasm_queue); TAILQ_INSERT_BEFORE(at, chk, sctp_next); inserted = 1; break; } else if (at->rec.data.fsn == chk->rec.data.fsn) { /* * Gak, He sent me a duplicate str seq * number */ /* * foo bar, I guess I will just free this * new guy, should we abort too? FIX ME * MAYBE? Or it COULD be that the SSN's have * wrapped. Maybe I should compare to TSN * somehow... sigh for now just blow away * the chunk! */ SCTPDBG(SCTP_DEBUG_XXX, "Duplicate to fsn: %u -- abort\n", at->rec.data.fsn); sctp_abort_in_reasm(stcb, control, chk, abort_flag, SCTP_FROM_SCTP_INDATA + SCTP_LOC_13); return; } } if (inserted == 0) { /* Goes on the end */ SCTPDBG(SCTP_DEBUG_XXX, "Inserting at tail of list fsn: %u\n", chk->rec.data.fsn); asoc->size_on_reasm_queue += chk->send_size; sctp_ucount_incr(asoc->cnt_on_reasm_queue); TAILQ_INSERT_TAIL(&control->reasm, chk, sctp_next); } } /* * Ok lets see if we can suck any up into the control structure that * are in seq if it makes sense. */ do_wakeup = 0; /* * If the first fragment has not been seen there is no sense in * looking. */ if (control->first_frag_seen) { next_fsn = control->fsn_included + 1; TAILQ_FOREACH_SAFE(at, &control->reasm, sctp_next, nat) { if (at->rec.data.fsn == next_fsn) { /* We can add this one now to the control */ SCTPDBG(SCTP_DEBUG_XXX, "Adding more to control: %p at: %p fsn: %u next_fsn: %u included: %u\n", control, at, at->rec.data.fsn, next_fsn, control->fsn_included); TAILQ_REMOVE(&control->reasm, at, sctp_next); lenadded = sctp_add_chk_to_control(control, strm, stcb, asoc, at, SCTP_READ_LOCK_NOT_HELD); asoc->size_on_all_streams += lenadded; if (control->on_read_q) { do_wakeup = 1; } next_fsn++; if (control->end_added && control->pdapi_started) { if (strm->pd_api_started) { strm->pd_api_started = 0; control->pdapi_started = 0; } if (control->on_read_q == 0) { sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, control->end_added, SCTP_READ_LOCK_NOT_HELD, SCTP_SO_NOT_LOCKED); do_wakeup = 1; } break; } } else { break; } } } if (do_wakeup) { /* Need to wakeup the reader */ sctp_wakeup_the_read_socket(stcb->sctp_ep, stcb, SCTP_SO_NOT_LOCKED); } } static struct sctp_queued_to_read * sctp_find_reasm_entry(struct sctp_stream_in *strm, uint32_t mid, int ordered, int idata_supported) { struct sctp_queued_to_read *control; if (ordered) { TAILQ_FOREACH(control, &strm->inqueue, next_instrm) { if (SCTP_MID_EQ(idata_supported, control->mid, mid)) { break; } } } else { if (idata_supported) { TAILQ_FOREACH(control, &strm->uno_inqueue, next_instrm) { if (SCTP_MID_EQ(idata_supported, control->mid, mid)) { break; } } } else { control = TAILQ_FIRST(&strm->uno_inqueue); } } return (control); } static int sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc, struct mbuf **m, int offset, int chk_length, struct sctp_nets *net, uint32_t *high_tsn, int *abort_flag, int *break_flag, int last_chunk, uint8_t chk_type) { /* Process a data chunk */ /* struct sctp_tmit_chunk *chk; */ struct sctp_tmit_chunk *chk; uint32_t tsn, fsn, gap, mid; struct mbuf *dmbuf; int the_len; int need_reasm_check = 0; uint16_t sid; struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; struct sctp_queued_to_read *control, *ncontrol; uint32_t ppid; uint8_t chk_flags; struct sctp_stream_reset_list *liste; int ordered; size_t clen; int created_control = 0; if (chk_type == SCTP_IDATA) { struct sctp_idata_chunk *chunk, chunk_buf; chunk = (struct sctp_idata_chunk *)sctp_m_getptr(*m, offset, sizeof(struct sctp_idata_chunk), (uint8_t *)&chunk_buf); chk_flags = chunk->ch.chunk_flags; clen = sizeof(struct sctp_idata_chunk); tsn = ntohl(chunk->dp.tsn); sid = ntohs(chunk->dp.sid); mid = ntohl(chunk->dp.mid); if (chk_flags & SCTP_DATA_FIRST_FRAG) { fsn = 0; ppid = chunk->dp.ppid_fsn.ppid; } else { fsn = ntohl(chunk->dp.ppid_fsn.fsn); ppid = 0xffffffff; /* Use as an invalid value. */ } } else { struct sctp_data_chunk *chunk, chunk_buf; chunk = (struct sctp_data_chunk *)sctp_m_getptr(*m, offset, sizeof(struct sctp_data_chunk), (uint8_t *)&chunk_buf); chk_flags = chunk->ch.chunk_flags; clen = sizeof(struct sctp_data_chunk); tsn = ntohl(chunk->dp.tsn); sid = ntohs(chunk->dp.sid); mid = (uint32_t)(ntohs(chunk->dp.ssn)); fsn = tsn; ppid = chunk->dp.ppid; } if ((size_t)chk_length == clen) { /* * Need to send an abort since we had a empty data chunk. */ op_err = sctp_generate_no_user_data_cause(tsn); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_14; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; return (0); } if ((chk_flags & SCTP_DATA_SACK_IMMEDIATELY) == SCTP_DATA_SACK_IMMEDIATELY) { asoc->send_sack = 1; } ordered = ((chk_flags & SCTP_DATA_UNORDERED) == 0); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(tsn, asoc->cumulative_tsn, asoc->highest_tsn_inside_map, SCTP_MAP_TSN_ENTERS); } if (stcb == NULL) { return (0); } SCTP_LTRACE_CHK(stcb->sctp_ep, stcb, chk_type, tsn); if (SCTP_TSN_GE(asoc->cumulative_tsn, tsn)) { /* It is a duplicate */ SCTP_STAT_INCR(sctps_recvdupdata); if (asoc->numduptsns < SCTP_MAX_DUP_TSNS) { /* Record a dup for the next outbound sack */ asoc->dup_tsns[asoc->numduptsns] = tsn; asoc->numduptsns++; } asoc->send_sack = 1; return (0); } /* Calculate the number of TSN's between the base and this TSN */ SCTP_CALC_TSN_TO_GAP(gap, tsn, asoc->mapping_array_base_tsn); if (gap >= (SCTP_MAPPING_ARRAY << 3)) { /* Can't hold the bit in the mapping at max array, toss it */ return (0); } if (gap >= (uint32_t)(asoc->mapping_array_size << 3)) { SCTP_TCB_LOCK_ASSERT(stcb); if (sctp_expand_mapping_array(asoc, gap)) { /* Can't expand, drop it */ return (0); } } if (SCTP_TSN_GT(tsn, *high_tsn)) { *high_tsn = tsn; } /* See if we have received this one already */ if (SCTP_IS_TSN_PRESENT(asoc->mapping_array, gap) || SCTP_IS_TSN_PRESENT(asoc->nr_mapping_array, gap)) { SCTP_STAT_INCR(sctps_recvdupdata); if (asoc->numduptsns < SCTP_MAX_DUP_TSNS) { /* Record a dup for the next outbound sack */ asoc->dup_tsns[asoc->numduptsns] = tsn; asoc->numduptsns++; } asoc->send_sack = 1; return (0); } /* * Check to see about the GONE flag, duplicates would cause a sack * to be sent up above */ if (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE) || (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_SOCKET_ALLGONE) || (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET))) { /* * wait a minute, this guy is gone, there is no longer a * receiver. Send peer an ABORT! */ op_err = sctp_generate_cause(SCTP_CAUSE_OUT_OF_RESC, ""); sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; return (0); } /* * Now before going further we see if there is room. If NOT then we * MAY let one through only IF this TSN is the one we are waiting * for on a partial delivery API. */ /* Is the stream valid? */ if (sid >= asoc->streamincnt) { struct sctp_error_invalid_stream *cause; op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_error_invalid_stream), 0, M_NOWAIT, 1, MT_DATA); if (op_err != NULL) { /* add some space up front so prepend will work well */ SCTP_BUF_RESV_UF(op_err, sizeof(struct sctp_chunkhdr)); cause = mtod(op_err, struct sctp_error_invalid_stream *); /* * Error causes are just param's and this one has * two back to back phdr, one with the error type * and size, the other with the streamid and a rsvd */ SCTP_BUF_LEN(op_err) = sizeof(struct sctp_error_invalid_stream); cause->cause.code = htons(SCTP_CAUSE_INVALID_STREAM); cause->cause.length = htons(sizeof(struct sctp_error_invalid_stream)); cause->stream_id = htons(sid); cause->reserved = htons(0); sctp_queue_op_err(stcb, op_err); } SCTP_STAT_INCR(sctps_badsid); SCTP_TCB_LOCK_ASSERT(stcb); SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, gap); if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_nr_map)) { asoc->highest_tsn_inside_nr_map = tsn; } if (tsn == (asoc->cumulative_tsn + 1)) { /* Update cum-ack */ asoc->cumulative_tsn = tsn; } return (0); } /* * If its a fragmented message, lets see if we can find the control * on the reassembly queues. */ if ((chk_type == SCTP_IDATA) && ((chk_flags & SCTP_DATA_FIRST_FRAG) == 0) && (fsn == 0)) { /* * The first *must* be fsn 0, and other (middle/end) pieces * can *not* be fsn 0. XXX: This can happen in case of a * wrap around. Ignore is for now. */ snprintf(msg, sizeof(msg), "FSN zero for MID=%8.8x, but flags=%2.2x", mid, chk_flags); goto err_out; } control = sctp_find_reasm_entry(&asoc->strmin[sid], mid, ordered, asoc->idata_supported); SCTPDBG(SCTP_DEBUG_XXX, "chunk_flags:0x%x look for control on queues %p\n", chk_flags, control); if ((chk_flags & SCTP_DATA_NOT_FRAG) != SCTP_DATA_NOT_FRAG) { /* See if we can find the re-assembly entity */ if (control != NULL) { /* We found something, does it belong? */ if (ordered && (mid != control->mid)) { snprintf(msg, sizeof(msg), "Reassembly problem (MID=%8.8x)", mid); err_out: op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_15; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; return (0); } if (ordered && ((control->sinfo_flags >> 8) & SCTP_DATA_UNORDERED)) { /* * We can't have a switched order with an * unordered chunk */ snprintf(msg, sizeof(msg), "All fragments of a user message must be ordered or unordered (TSN=%8.8x)", tsn); goto err_out; } if (!ordered && (((control->sinfo_flags >> 8) & SCTP_DATA_UNORDERED) == 0)) { /* * We can't have a switched unordered with a * ordered chunk */ snprintf(msg, sizeof(msg), "All fragments of a user message must be ordered or unordered (TSN=%8.8x)", tsn); goto err_out; } } } else { /* * Its a complete segment. Lets validate we don't have a * re-assembly going on with the same Stream/Seq (for * ordered) or in the same Stream for unordered. */ if (control != NULL) { if (ordered || asoc->idata_supported) { SCTPDBG(SCTP_DEBUG_XXX, "chunk_flags: 0x%x dup detected on MID: %u\n", chk_flags, mid); snprintf(msg, sizeof(msg), "Duplicate MID=%8.8x detected.", mid); goto err_out; } else { if ((tsn == control->fsn_included + 1) && (control->end_added == 0)) { snprintf(msg, sizeof(msg), "Illegal message sequence, missing end for MID: %8.8x", control->fsn_included); goto err_out; } else { control = NULL; } } } } /* now do the tests */ if (((asoc->cnt_on_all_streams + asoc->cnt_on_reasm_queue + asoc->cnt_msg_on_sb) >= SCTP_BASE_SYSCTL(sctp_max_chunks_on_queue)) || (((int)asoc->my_rwnd) <= 0)) { /* * When we have NO room in the rwnd we check to make sure * the reader is doing its job... */ if (stcb->sctp_socket->so_rcv.sb_cc) { /* some to read, wake-up */ #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) struct socket *so; so = SCTP_INP_SO(stcb->sctp_ep); atomic_add_int(&stcb->asoc.refcnt, 1); SCTP_TCB_UNLOCK(stcb); SCTP_SOCKET_LOCK(so, 1); SCTP_TCB_LOCK(stcb); atomic_subtract_int(&stcb->asoc.refcnt, 1); if (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET) { /* assoc was freed while we were unlocked */ SCTP_SOCKET_UNLOCK(so, 1); return (0); } #endif sctp_sorwakeup(stcb->sctp_ep, stcb->sctp_socket); #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) SCTP_SOCKET_UNLOCK(so, 1); #endif } /* now is it in the mapping array of what we have accepted? */ if (chk_type == SCTP_DATA) { if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_map) && SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_nr_map)) { /* Nope not in the valid range dump it */ dump_packet: sctp_set_rwnd(stcb, asoc); if ((asoc->cnt_on_all_streams + asoc->cnt_on_reasm_queue + asoc->cnt_msg_on_sb) >= SCTP_BASE_SYSCTL(sctp_max_chunks_on_queue)) { SCTP_STAT_INCR(sctps_datadropchklmt); } else { SCTP_STAT_INCR(sctps_datadroprwnd); } *break_flag = 1; return (0); } } else { if (control == NULL) { goto dump_packet; } if (SCTP_TSN_GT(fsn, control->top_fsn)) { goto dump_packet; } } } #ifdef SCTP_ASOCLOG_OF_TSNS SCTP_TCB_LOCK_ASSERT(stcb); if (asoc->tsn_in_at >= SCTP_TSN_LOG_SIZE) { asoc->tsn_in_at = 0; asoc->tsn_in_wrapped = 1; } asoc->in_tsnlog[asoc->tsn_in_at].tsn = tsn; asoc->in_tsnlog[asoc->tsn_in_at].strm = sid; asoc->in_tsnlog[asoc->tsn_in_at].seq = mid; asoc->in_tsnlog[asoc->tsn_in_at].sz = chk_length; asoc->in_tsnlog[asoc->tsn_in_at].flgs = chunk_flags; asoc->in_tsnlog[asoc->tsn_in_at].stcb = (void *)stcb; asoc->in_tsnlog[asoc->tsn_in_at].in_pos = asoc->tsn_in_at; asoc->in_tsnlog[asoc->tsn_in_at].in_out = 1; asoc->tsn_in_at++; #endif /* * Before we continue lets validate that we are not being fooled by * an evil attacker. We can only have Nk chunks based on our TSN * spread allowed by the mapping array N * 8 bits, so there is no * way our stream sequence numbers could have wrapped. We of course * only validate the FIRST fragment so the bit must be set. */ if ((chk_flags & SCTP_DATA_FIRST_FRAG) && (TAILQ_EMPTY(&asoc->resetHead)) && (chk_flags & SCTP_DATA_UNORDERED) == 0 && SCTP_MID_GE(asoc->idata_supported, asoc->strmin[sid].last_mid_delivered, mid)) { /* The incoming sseq is behind where we last delivered? */ SCTPDBG(SCTP_DEBUG_INDATA1, "EVIL/Broken-Dup S-SEQ: %u delivered: %u from peer, Abort!\n", mid, asoc->strmin[sid].last_mid_delivered); if (asoc->idata_supported) { snprintf(msg, sizeof(msg), "Delivered MID=%8.8x, got TSN=%8.8x, SID=%4.4x, MID=%8.8x", asoc->strmin[sid].last_mid_delivered, tsn, sid, mid); } else { snprintf(msg, sizeof(msg), "Delivered SSN=%4.4x, got TSN=%8.8x, SID=%4.4x, SSN=%4.4x", (uint16_t)asoc->strmin[sid].last_mid_delivered, tsn, sid, (uint16_t)mid); } op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_16; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); *abort_flag = 1; return (0); } if (chk_type == SCTP_IDATA) { the_len = (chk_length - sizeof(struct sctp_idata_chunk)); } else { the_len = (chk_length - sizeof(struct sctp_data_chunk)); } if (last_chunk == 0) { if (chk_type == SCTP_IDATA) { dmbuf = SCTP_M_COPYM(*m, (offset + sizeof(struct sctp_idata_chunk)), the_len, M_NOWAIT); } else { dmbuf = SCTP_M_COPYM(*m, (offset + sizeof(struct sctp_data_chunk)), the_len, M_NOWAIT); } #ifdef SCTP_MBUF_LOGGING if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MBUF_LOGGING_ENABLE) { sctp_log_mbc(dmbuf, SCTP_MBUF_ICOPY); } #endif } else { /* We can steal the last chunk */ int l_len; dmbuf = *m; /* lop off the top part */ if (chk_type == SCTP_IDATA) { m_adj(dmbuf, (offset + sizeof(struct sctp_idata_chunk))); } else { m_adj(dmbuf, (offset + sizeof(struct sctp_data_chunk))); } if (SCTP_BUF_NEXT(dmbuf) == NULL) { l_len = SCTP_BUF_LEN(dmbuf); } else { /* * need to count up the size hopefully does not hit * this to often :-0 */ struct mbuf *lat; l_len = 0; for (lat = dmbuf; lat; lat = SCTP_BUF_NEXT(lat)) { l_len += SCTP_BUF_LEN(lat); } } if (l_len > the_len) { /* Trim the end round bytes off too */ m_adj(dmbuf, -(l_len - the_len)); } } if (dmbuf == NULL) { SCTP_STAT_INCR(sctps_nomem); return (0); } /* * Now no matter what, we need a control, get one if we don't have * one (we may have gotten it above when we found the message was * fragmented */ if (control == NULL) { sctp_alloc_a_readq(stcb, control); sctp_build_readq_entry_mac(control, stcb, asoc->context, net, tsn, ppid, sid, chk_flags, NULL, fsn, mid); if (control == NULL) { SCTP_STAT_INCR(sctps_nomem); return (0); } if ((chk_flags & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { struct mbuf *mm; control->data = dmbuf; - mm = control->data; for (mm = control->data; mm; mm = mm->m_next) { control->length += SCTP_BUF_LEN(mm); } control->tail_mbuf = NULL; control->end_added = 1; control->last_frag_seen = 1; control->first_frag_seen = 1; control->fsn_included = fsn; control->top_fsn = fsn; } created_control = 1; } SCTPDBG(SCTP_DEBUG_XXX, "chunk_flags: 0x%x ordered: %d MID: %u control: %p\n", chk_flags, ordered, mid, control); if ((chk_flags & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG && TAILQ_EMPTY(&asoc->resetHead) && ((ordered == 0) || (SCTP_MID_EQ(asoc->idata_supported, asoc->strmin[sid].last_mid_delivered + 1, mid) && TAILQ_EMPTY(&asoc->strmin[sid].inqueue)))) { /* Candidate for express delivery */ /* * Its not fragmented, No PD-API is up, Nothing in the * delivery queue, Its un-ordered OR ordered and the next to * deliver AND nothing else is stuck on the stream queue, * And there is room for it in the socket buffer. Lets just * stuff it up the buffer.... */ SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, gap); if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_nr_map)) { asoc->highest_tsn_inside_nr_map = tsn; } SCTPDBG(SCTP_DEBUG_XXX, "Injecting control: %p to be read (MID: %u)\n", control, mid); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_NOT_HELD, SCTP_SO_NOT_LOCKED); if ((chk_flags & SCTP_DATA_UNORDERED) == 0) { /* for ordered, bump what we delivered */ asoc->strmin[sid].last_mid_delivered++; } SCTP_STAT_INCR(sctps_recvexpress); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del_alt(stcb, tsn, mid, sid, SCTP_STR_LOG_FROM_EXPRS_DEL); } control = NULL; goto finish_express_del; } /* Now will we need a chunk too? */ if ((chk_flags & SCTP_DATA_NOT_FRAG) != SCTP_DATA_NOT_FRAG) { sctp_alloc_a_chunk(stcb, chk); if (chk == NULL) { /* No memory so we drop the chunk */ SCTP_STAT_INCR(sctps_nomem); if (last_chunk == 0) { /* we copied it, free the copy */ sctp_m_freem(dmbuf); } return (0); } chk->rec.data.tsn = tsn; chk->no_fr_allowed = 0; chk->rec.data.fsn = fsn; chk->rec.data.mid = mid; chk->rec.data.sid = sid; chk->rec.data.ppid = ppid; chk->rec.data.context = stcb->asoc.context; chk->rec.data.doing_fast_retransmit = 0; chk->rec.data.rcv_flags = chk_flags; chk->asoc = asoc; chk->send_size = the_len; chk->whoTo = net; SCTPDBG(SCTP_DEBUG_XXX, "Building ck: %p for control: %p to be read (MID: %u)\n", chk, control, mid); atomic_add_int(&net->ref_count, 1); chk->data = dmbuf; } /* Set the appropriate TSN mark */ if (SCTP_BASE_SYSCTL(sctp_do_drain) == 0) { SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, gap); if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_nr_map)) { asoc->highest_tsn_inside_nr_map = tsn; } } else { SCTP_SET_TSN_PRESENT(asoc->mapping_array, gap); if (SCTP_TSN_GT(tsn, asoc->highest_tsn_inside_map)) { asoc->highest_tsn_inside_map = tsn; } } /* Now is it complete (i.e. not fragmented)? */ if ((chk_flags & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { /* * Special check for when streams are resetting. We could be * more smart about this and check the actual stream to see * if it is not being reset.. that way we would not create a * HOLB when amongst streams being reset and those not being * reset. * */ if (((liste = TAILQ_FIRST(&asoc->resetHead)) != NULL) && SCTP_TSN_GT(tsn, liste->tsn)) { /* * yep its past where we need to reset... go ahead * and queue it. */ if (TAILQ_EMPTY(&asoc->pending_reply_queue)) { /* first one on */ TAILQ_INSERT_TAIL(&asoc->pending_reply_queue, control, next); } else { struct sctp_queued_to_read *lcontrol, *nlcontrol; unsigned char inserted = 0; TAILQ_FOREACH_SAFE(lcontrol, &asoc->pending_reply_queue, next, nlcontrol) { if (SCTP_TSN_GT(control->sinfo_tsn, lcontrol->sinfo_tsn)) { continue; } else { /* found it */ TAILQ_INSERT_BEFORE(lcontrol, control, next); inserted = 1; break; } } if (inserted == 0) { /* * must be put at end, use prevP * (all setup from loop) to setup * nextP. */ TAILQ_INSERT_TAIL(&asoc->pending_reply_queue, control, next); } } goto finish_express_del; } if (chk_flags & SCTP_DATA_UNORDERED) { /* queue directly into socket buffer */ SCTPDBG(SCTP_DEBUG_XXX, "Unordered data to be read control: %p MID: %u\n", control, mid); sctp_mark_non_revokable(asoc, control->sinfo_tsn); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_NOT_HELD, SCTP_SO_NOT_LOCKED); } else { SCTPDBG(SCTP_DEBUG_XXX, "Queue control: %p for reordering MID: %u\n", control, mid); sctp_queue_data_to_stream(stcb, asoc, control, abort_flag, &need_reasm_check); if (*abort_flag) { if (last_chunk) { *m = NULL; } return (0); } } goto finish_express_del; } /* If we reach here its a reassembly */ need_reasm_check = 1; SCTPDBG(SCTP_DEBUG_XXX, "Queue data to stream for reasm control: %p MID: %u\n", control, mid); sctp_queue_data_for_reasm(stcb, asoc, control, chk, created_control, abort_flag, tsn); if (*abort_flag) { /* * the assoc is now gone and chk was put onto the reasm * queue, which has all been freed. */ if (last_chunk) { *m = NULL; } return (0); } finish_express_del: /* Here we tidy up things */ if (tsn == (asoc->cumulative_tsn + 1)) { /* Update cum-ack */ asoc->cumulative_tsn = tsn; } if (last_chunk) { *m = NULL; } if (ordered) { SCTP_STAT_INCR_COUNTER64(sctps_inorderchunks); } else { SCTP_STAT_INCR_COUNTER64(sctps_inunorderchunks); } SCTP_STAT_INCR(sctps_recvdata); /* Set it present please */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_STR_LOGGING_ENABLE) { sctp_log_strm_del_alt(stcb, tsn, mid, sid, SCTP_STR_LOG_FROM_MARK_TSN); } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(asoc->mapping_array_base_tsn, asoc->cumulative_tsn, asoc->highest_tsn_inside_map, SCTP_MAP_PREPARE_SLIDE); } if (need_reasm_check) { (void)sctp_deliver_reasm_check(stcb, asoc, &asoc->strmin[sid], SCTP_READ_LOCK_NOT_HELD); need_reasm_check = 0; } /* check the special flag for stream resets */ if (((liste = TAILQ_FIRST(&asoc->resetHead)) != NULL) && SCTP_TSN_GE(asoc->cumulative_tsn, liste->tsn)) { /* * we have finished working through the backlogged TSN's now * time to reset streams. 1: call reset function. 2: free * pending_reply space 3: distribute any chunks in * pending_reply_queue. */ sctp_reset_in_stream(stcb, liste->number_entries, liste->list_of_streams); TAILQ_REMOVE(&asoc->resetHead, liste, next_resp); sctp_send_deferred_reset_response(stcb, liste, SCTP_STREAM_RESET_RESULT_PERFORMED); SCTP_FREE(liste, SCTP_M_STRESET); /* sa_ignore FREED_MEMORY */ liste = TAILQ_FIRST(&asoc->resetHead); if (TAILQ_EMPTY(&asoc->resetHead)) { /* All can be removed */ TAILQ_FOREACH_SAFE(control, &asoc->pending_reply_queue, next, ncontrol) { TAILQ_REMOVE(&asoc->pending_reply_queue, control, next); sctp_queue_data_to_stream(stcb, asoc, control, abort_flag, &need_reasm_check); if (*abort_flag) { return (0); } if (need_reasm_check) { (void)sctp_deliver_reasm_check(stcb, asoc, &asoc->strmin[control->sinfo_stream], SCTP_READ_LOCK_NOT_HELD); need_reasm_check = 0; } } } else { TAILQ_FOREACH_SAFE(control, &asoc->pending_reply_queue, next, ncontrol) { if (SCTP_TSN_GT(control->sinfo_tsn, liste->tsn)) { break; } /* * if control->sinfo_tsn is <= liste->tsn we * can process it which is the NOT of * control->sinfo_tsn > liste->tsn */ TAILQ_REMOVE(&asoc->pending_reply_queue, control, next); sctp_queue_data_to_stream(stcb, asoc, control, abort_flag, &need_reasm_check); if (*abort_flag) { return (0); } if (need_reasm_check) { (void)sctp_deliver_reasm_check(stcb, asoc, &asoc->strmin[control->sinfo_stream], SCTP_READ_LOCK_NOT_HELD); need_reasm_check = 0; } } } } return (1); } static const int8_t sctp_map_lookup_tab[256] = { 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 8 }; void sctp_slide_mapping_arrays(struct sctp_tcb *stcb) { /* * Now we also need to check the mapping array in a couple of ways. * 1) Did we move the cum-ack point? * * When you first glance at this you might think that all entries * that make up the position of the cum-ack would be in the * nr-mapping array only.. i.e. things up to the cum-ack are always * deliverable. Thats true with one exception, when its a fragmented * message we may not deliver the data until some threshold (or all * of it) is in place. So we must OR the nr_mapping_array and * mapping_array to get a true picture of the cum-ack. */ struct sctp_association *asoc; int at; uint8_t val; int slide_from, slide_end, lgap, distance; uint32_t old_cumack, old_base, old_highest, highest_tsn; asoc = &stcb->asoc; old_cumack = asoc->cumulative_tsn; old_base = asoc->mapping_array_base_tsn; old_highest = asoc->highest_tsn_inside_map; /* * We could probably improve this a small bit by calculating the * offset of the current cum-ack as the starting point. */ at = 0; for (slide_from = 0; slide_from < stcb->asoc.mapping_array_size; slide_from++) { val = asoc->nr_mapping_array[slide_from] | asoc->mapping_array[slide_from]; if (val == 0xff) { at += 8; } else { /* there is a 0 bit */ at += sctp_map_lookup_tab[val]; break; } } asoc->cumulative_tsn = asoc->mapping_array_base_tsn + (at - 1); if (SCTP_TSN_GT(asoc->cumulative_tsn, asoc->highest_tsn_inside_map) && SCTP_TSN_GT(asoc->cumulative_tsn, asoc->highest_tsn_inside_nr_map)) { #ifdef INVARIANTS panic("huh, cumack 0x%x greater than high-tsn 0x%x in map", asoc->cumulative_tsn, asoc->highest_tsn_inside_map); #else SCTP_PRINTF("huh, cumack 0x%x greater than high-tsn 0x%x in map - should panic?\n", asoc->cumulative_tsn, asoc->highest_tsn_inside_map); sctp_print_mapping_array(asoc); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(0, 6, asoc->highest_tsn_inside_map, SCTP_MAP_SLIDE_RESULT); } asoc->highest_tsn_inside_map = asoc->cumulative_tsn; asoc->highest_tsn_inside_nr_map = asoc->cumulative_tsn; #endif } if (SCTP_TSN_GT(asoc->highest_tsn_inside_nr_map, asoc->highest_tsn_inside_map)) { highest_tsn = asoc->highest_tsn_inside_nr_map; } else { highest_tsn = asoc->highest_tsn_inside_map; } if ((asoc->cumulative_tsn == highest_tsn) && (at >= 8)) { /* The complete array was completed by a single FR */ /* highest becomes the cum-ack */ int clr; #ifdef INVARIANTS unsigned int i; #endif /* clear the array */ clr = ((at + 7) >> 3); if (clr > asoc->mapping_array_size) { clr = asoc->mapping_array_size; } memset(asoc->mapping_array, 0, clr); memset(asoc->nr_mapping_array, 0, clr); #ifdef INVARIANTS for (i = 0; i < asoc->mapping_array_size; i++) { if ((asoc->mapping_array[i]) || (asoc->nr_mapping_array[i])) { SCTP_PRINTF("Error Mapping array's not clean at clear\n"); sctp_print_mapping_array(asoc); } } #endif asoc->mapping_array_base_tsn = asoc->cumulative_tsn + 1; asoc->highest_tsn_inside_nr_map = asoc->highest_tsn_inside_map = asoc->cumulative_tsn; } else if (at >= 8) { /* we can slide the mapping array down */ /* slide_from holds where we hit the first NON 0xff byte */ /* * now calculate the ceiling of the move using our highest * TSN value */ SCTP_CALC_TSN_TO_GAP(lgap, highest_tsn, asoc->mapping_array_base_tsn); slide_end = (lgap >> 3); if (slide_end < slide_from) { sctp_print_mapping_array(asoc); #ifdef INVARIANTS panic("impossible slide"); #else SCTP_PRINTF("impossible slide lgap: %x slide_end: %x slide_from: %x? at: %d\n", lgap, slide_end, slide_from, at); return; #endif } if (slide_end > asoc->mapping_array_size) { #ifdef INVARIANTS panic("would overrun buffer"); #else SCTP_PRINTF("Gak, would have overrun map end: %d slide_end: %d\n", asoc->mapping_array_size, slide_end); slide_end = asoc->mapping_array_size; #endif } distance = (slide_end - slide_from) + 1; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(old_base, old_cumack, old_highest, SCTP_MAP_PREPARE_SLIDE); sctp_log_map((uint32_t)slide_from, (uint32_t)slide_end, (uint32_t)lgap, SCTP_MAP_SLIDE_FROM); } if (distance + slide_from > asoc->mapping_array_size || distance < 0) { /* * Here we do NOT slide forward the array so that * hopefully when more data comes in to fill it up * we will be able to slide it forward. Really I * don't think this should happen :-0 */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map((uint32_t)distance, (uint32_t)slide_from, (uint32_t)asoc->mapping_array_size, SCTP_MAP_SLIDE_NONE); } } else { int ii; for (ii = 0; ii < distance; ii++) { asoc->mapping_array[ii] = asoc->mapping_array[slide_from + ii]; asoc->nr_mapping_array[ii] = asoc->nr_mapping_array[slide_from + ii]; } for (ii = distance; ii < asoc->mapping_array_size; ii++) { asoc->mapping_array[ii] = 0; asoc->nr_mapping_array[ii] = 0; } if (asoc->highest_tsn_inside_map + 1 == asoc->mapping_array_base_tsn) { asoc->highest_tsn_inside_map += (slide_from << 3); } if (asoc->highest_tsn_inside_nr_map + 1 == asoc->mapping_array_base_tsn) { asoc->highest_tsn_inside_nr_map += (slide_from << 3); } asoc->mapping_array_base_tsn += (slide_from << 3); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(asoc->mapping_array_base_tsn, asoc->cumulative_tsn, asoc->highest_tsn_inside_map, SCTP_MAP_SLIDE_RESULT); } } } } void sctp_sack_check(struct sctp_tcb *stcb, int was_a_gap) { struct sctp_association *asoc; uint32_t highest_tsn; int is_a_gap; sctp_slide_mapping_arrays(stcb); asoc = &stcb->asoc; if (SCTP_TSN_GT(asoc->highest_tsn_inside_nr_map, asoc->highest_tsn_inside_map)) { highest_tsn = asoc->highest_tsn_inside_nr_map; } else { highest_tsn = asoc->highest_tsn_inside_map; } /* Is there a gap now? */ is_a_gap = SCTP_TSN_GT(highest_tsn, stcb->asoc.cumulative_tsn); /* * Now we need to see if we need to queue a sack or just start the * timer (if allowed). */ if (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_SENT) { /* * Ok special case, in SHUTDOWN-SENT case. here we maker * sure SACK timer is off and instead send a SHUTDOWN and a * SACK */ if (SCTP_OS_TIMER_PENDING(&stcb->asoc.dack_timer.timer)) { sctp_timer_stop(SCTP_TIMER_TYPE_RECV, stcb->sctp_ep, stcb, NULL, SCTP_FROM_SCTP_INDATA + SCTP_LOC_17); } sctp_send_shutdown(stcb, ((stcb->asoc.alternate) ? stcb->asoc.alternate : stcb->asoc.primary_destination)); if (is_a_gap) { sctp_send_sack(stcb, SCTP_SO_NOT_LOCKED); } } else { /* * CMT DAC algorithm: increase number of packets received * since last ack */ stcb->asoc.cmt_dac_pkts_rcvd++; if ((stcb->asoc.send_sack == 1) || /* We need to send a * SACK */ ((was_a_gap) && (is_a_gap == 0)) || /* was a gap, but no * longer is one */ (stcb->asoc.numduptsns) || /* we have dup's */ (is_a_gap) || /* is still a gap */ (stcb->asoc.delayed_ack == 0) || /* Delayed sack disabled */ (stcb->asoc.data_pkts_seen >= stcb->asoc.sack_freq) /* hit limit of pkts */ ) { if ((stcb->asoc.sctp_cmt_on_off > 0) && (SCTP_BASE_SYSCTL(sctp_cmt_use_dac)) && (stcb->asoc.send_sack == 0) && (stcb->asoc.numduptsns == 0) && (stcb->asoc.delayed_ack) && (!SCTP_OS_TIMER_PENDING(&stcb->asoc.dack_timer.timer))) { /* * CMT DAC algorithm: With CMT, delay acks * even in the face of * * reordering. Therefore, if acks that do * not have to be sent because of the above * reasons, will be delayed. That is, acks * that would have been sent due to gap * reports will be delayed with DAC. Start * the delayed ack timer. */ sctp_timer_start(SCTP_TIMER_TYPE_RECV, stcb->sctp_ep, stcb, NULL); } else { /* * Ok we must build a SACK since the timer * is pending, we got our first packet OR * there are gaps or duplicates. */ (void)SCTP_OS_TIMER_STOP(&stcb->asoc.dack_timer.timer); sctp_send_sack(stcb, SCTP_SO_NOT_LOCKED); } } else { if (!SCTP_OS_TIMER_PENDING(&stcb->asoc.dack_timer.timer)) { sctp_timer_start(SCTP_TIMER_TYPE_RECV, stcb->sctp_ep, stcb, NULL); } } } } int sctp_process_data(struct mbuf **mm, int iphlen, int *offset, int length, struct sctp_inpcb *inp, struct sctp_tcb *stcb, struct sctp_nets *net, uint32_t *high_tsn) { struct sctp_chunkhdr *ch, chunk_buf; struct sctp_association *asoc; int num_chunks = 0; /* number of control chunks processed */ int stop_proc = 0; int chk_length, break_flag, last_chunk; int abort_flag = 0, was_a_gap; struct mbuf *m; uint32_t highest_tsn; /* set the rwnd */ sctp_set_rwnd(stcb, &stcb->asoc); m = *mm; SCTP_TCB_LOCK_ASSERT(stcb); asoc = &stcb->asoc; if (SCTP_TSN_GT(asoc->highest_tsn_inside_nr_map, asoc->highest_tsn_inside_map)) { highest_tsn = asoc->highest_tsn_inside_nr_map; } else { highest_tsn = asoc->highest_tsn_inside_map; } was_a_gap = SCTP_TSN_GT(highest_tsn, stcb->asoc.cumulative_tsn); /* * setup where we got the last DATA packet from for any SACK that * may need to go out. Don't bump the net. This is done ONLY when a * chunk is assigned. */ asoc->last_data_chunk_from = net; /*- * Now before we proceed we must figure out if this is a wasted * cluster... i.e. it is a small packet sent in and yet the driver * underneath allocated a full cluster for it. If so we must copy it * to a smaller mbuf and free up the cluster mbuf. This will help * with cluster starvation. Note for __Panda__ we don't do this * since it has clusters all the way down to 64 bytes. */ if (SCTP_BUF_LEN(m) < (long)MLEN && SCTP_BUF_NEXT(m) == NULL) { /* we only handle mbufs that are singletons.. not chains */ m = sctp_get_mbuf_for_msg(SCTP_BUF_LEN(m), 0, M_NOWAIT, 1, MT_DATA); if (m) { /* ok lets see if we can copy the data up */ caddr_t *from, *to; /* get the pointers and copy */ to = mtod(m, caddr_t *); from = mtod((*mm), caddr_t *); memcpy(to, from, SCTP_BUF_LEN((*mm))); /* copy the length and free up the old */ SCTP_BUF_LEN(m) = SCTP_BUF_LEN((*mm)); sctp_m_freem(*mm); /* success, back copy */ *mm = m; } else { /* We are in trouble in the mbuf world .. yikes */ m = *mm; } } /* get pointer to the first chunk header */ ch = (struct sctp_chunkhdr *)sctp_m_getptr(m, *offset, sizeof(struct sctp_chunkhdr), (uint8_t *)&chunk_buf); if (ch == NULL) { return (1); } /* * process all DATA chunks... */ *high_tsn = asoc->cumulative_tsn; break_flag = 0; asoc->data_pkts_seen++; while (stop_proc == 0) { /* validate chunk length */ chk_length = ntohs(ch->chunk_length); if (length - *offset < chk_length) { /* all done, mutulated chunk */ stop_proc = 1; continue; } if ((asoc->idata_supported == 1) && (ch->chunk_type == SCTP_DATA)) { struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; snprintf(msg, sizeof(msg), "%s", "I-DATA chunk received when DATA was negotiated"); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_18; sctp_abort_an_association(inp, stcb, op_err, SCTP_SO_NOT_LOCKED); return (2); } if ((asoc->idata_supported == 0) && (ch->chunk_type == SCTP_IDATA)) { struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; snprintf(msg, sizeof(msg), "%s", "DATA chunk received when I-DATA was negotiated"); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_19; sctp_abort_an_association(inp, stcb, op_err, SCTP_SO_NOT_LOCKED); return (2); } if ((ch->chunk_type == SCTP_DATA) || (ch->chunk_type == SCTP_IDATA)) { int clen; if (ch->chunk_type == SCTP_DATA) { clen = sizeof(struct sctp_data_chunk); } else { clen = sizeof(struct sctp_idata_chunk); } if (chk_length < clen) { /* * Need to send an abort since we had a * invalid data chunk. */ struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; snprintf(msg, sizeof(msg), "DATA chunk of length %d", chk_length); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_20; sctp_abort_an_association(inp, stcb, op_err, SCTP_SO_NOT_LOCKED); return (2); } #ifdef SCTP_AUDITING_ENABLED sctp_audit_log(0xB1, 0); #endif if (SCTP_SIZE32(chk_length) == (length - *offset)) { last_chunk = 1; } else { last_chunk = 0; } if (sctp_process_a_data_chunk(stcb, asoc, mm, *offset, chk_length, net, high_tsn, &abort_flag, &break_flag, last_chunk, ch->chunk_type)) { num_chunks++; } if (abort_flag) return (2); if (break_flag) { /* * Set because of out of rwnd space and no * drop rep space left. */ stop_proc = 1; continue; } } else { /* not a data chunk in the data region */ switch (ch->chunk_type) { case SCTP_INITIATION: case SCTP_INITIATION_ACK: case SCTP_SELECTIVE_ACK: case SCTP_NR_SELECTIVE_ACK: case SCTP_HEARTBEAT_REQUEST: case SCTP_HEARTBEAT_ACK: case SCTP_ABORT_ASSOCIATION: case SCTP_SHUTDOWN: case SCTP_SHUTDOWN_ACK: case SCTP_OPERATION_ERROR: case SCTP_COOKIE_ECHO: case SCTP_COOKIE_ACK: case SCTP_ECN_ECHO: case SCTP_ECN_CWR: case SCTP_SHUTDOWN_COMPLETE: case SCTP_AUTHENTICATION: case SCTP_ASCONF_ACK: case SCTP_PACKET_DROPPED: case SCTP_STREAM_RESET: case SCTP_FORWARD_CUM_TSN: case SCTP_ASCONF: { /* * Now, what do we do with KNOWN * chunks that are NOT in the right * place? * * For now, I do nothing but ignore * them. We may later want to add * sysctl stuff to switch out and do * either an ABORT() or possibly * process them. */ struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; snprintf(msg, sizeof(msg), "DATA chunk followed by chunk of type %2.2x", ch->chunk_type); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); sctp_abort_an_association(inp, stcb, op_err, SCTP_SO_NOT_LOCKED); return (2); } default: /* unknown chunk type, use bit rules */ if (ch->chunk_type & 0x40) { /* Add a error report to the queue */ struct mbuf *op_err; struct sctp_gen_error_cause *cause; op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_gen_error_cause), 0, M_NOWAIT, 1, MT_DATA); if (op_err != NULL) { cause = mtod(op_err, struct sctp_gen_error_cause *); cause->code = htons(SCTP_CAUSE_UNRECOG_CHUNK); cause->length = htons((uint16_t)(chk_length + sizeof(struct sctp_gen_error_cause))); SCTP_BUF_LEN(op_err) = sizeof(struct sctp_gen_error_cause); SCTP_BUF_NEXT(op_err) = SCTP_M_COPYM(m, *offset, chk_length, M_NOWAIT); if (SCTP_BUF_NEXT(op_err) != NULL) { sctp_queue_op_err(stcb, op_err); } else { sctp_m_freem(op_err); } } } if ((ch->chunk_type & 0x80) == 0) { /* discard the rest of this packet */ stop_proc = 1; } /* else skip this bad chunk and * continue... */ break; } /* switch of chunk type */ } *offset += SCTP_SIZE32(chk_length); if ((*offset >= length) || stop_proc) { /* no more data left in the mbuf chain */ stop_proc = 1; continue; } ch = (struct sctp_chunkhdr *)sctp_m_getptr(m, *offset, sizeof(struct sctp_chunkhdr), (uint8_t *)&chunk_buf); if (ch == NULL) { *offset = length; stop_proc = 1; continue; } } if (break_flag) { /* * we need to report rwnd overrun drops. */ sctp_send_packet_dropped(stcb, net, *mm, length, iphlen, 0); } if (num_chunks) { /* * Did we get data, if so update the time for auto-close and * give peer credit for being alive. */ SCTP_STAT_INCR(sctps_recvpktwithdata); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_THRESHOLD_LOGGING) { sctp_misc_ints(SCTP_THRESHOLD_CLEAR, stcb->asoc.overall_error_count, 0, SCTP_FROM_SCTP_INDATA, __LINE__); } stcb->asoc.overall_error_count = 0; (void)SCTP_GETTIME_TIMEVAL(&stcb->asoc.time_last_rcvd); } /* now service all of the reassm queue if needed */ if (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_SENT) { /* Assure that we ack right away */ stcb->asoc.send_sack = 1; } /* Start a sack timer or QUEUE a SACK for sending */ sctp_sack_check(stcb, was_a_gap); return (0); } static int sctp_process_segment_range(struct sctp_tcb *stcb, struct sctp_tmit_chunk **p_tp1, uint32_t last_tsn, uint16_t frag_strt, uint16_t frag_end, int nr_sacking, int *num_frs, uint32_t *biggest_newly_acked_tsn, uint32_t *this_sack_lowest_newack, int *rto_ok) { struct sctp_tmit_chunk *tp1; unsigned int theTSN; int j, wake_him = 0, circled = 0; /* Recover the tp1 we last saw */ tp1 = *p_tp1; if (tp1 == NULL) { tp1 = TAILQ_FIRST(&stcb->asoc.sent_queue); } for (j = frag_strt; j <= frag_end; j++) { theTSN = j + last_tsn; while (tp1) { if (tp1->rec.data.doing_fast_retransmit) (*num_frs) += 1; /*- * CMT: CUCv2 algorithm. For each TSN being * processed from the sent queue, track the * next expected pseudo-cumack, or * rtx_pseudo_cumack, if required. Separate * cumack trackers for first transmissions, * and retransmissions. */ if ((tp1->sent < SCTP_DATAGRAM_RESEND) && (tp1->whoTo->find_pseudo_cumack == 1) && (tp1->snd_count == 1)) { tp1->whoTo->pseudo_cumack = tp1->rec.data.tsn; tp1->whoTo->find_pseudo_cumack = 0; } if ((tp1->sent < SCTP_DATAGRAM_RESEND) && (tp1->whoTo->find_rtx_pseudo_cumack == 1) && (tp1->snd_count > 1)) { tp1->whoTo->rtx_pseudo_cumack = tp1->rec.data.tsn; tp1->whoTo->find_rtx_pseudo_cumack = 0; } if (tp1->rec.data.tsn == theTSN) { if (tp1->sent != SCTP_DATAGRAM_UNSENT) { /*- * must be held until * cum-ack passes */ if (tp1->sent < SCTP_DATAGRAM_RESEND) { /*- * If it is less than RESEND, it is * now no-longer in flight. * Higher values may already be set * via previous Gap Ack Blocks... * i.e. ACKED or RESEND. */ if (SCTP_TSN_GT(tp1->rec.data.tsn, *biggest_newly_acked_tsn)) { *biggest_newly_acked_tsn = tp1->rec.data.tsn; } /*- * CMT: SFR algo (and HTNA) - set * saw_newack to 1 for dest being * newly acked. update * this_sack_highest_newack if * appropriate. */ if (tp1->rec.data.chunk_was_revoked == 0) tp1->whoTo->saw_newack = 1; if (SCTP_TSN_GT(tp1->rec.data.tsn, tp1->whoTo->this_sack_highest_newack)) { tp1->whoTo->this_sack_highest_newack = tp1->rec.data.tsn; } /*- * CMT DAC algo: also update * this_sack_lowest_newack */ if (*this_sack_lowest_newack == 0) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(*this_sack_lowest_newack, last_tsn, tp1->rec.data.tsn, 0, 0, SCTP_LOG_TSN_ACKED); } *this_sack_lowest_newack = tp1->rec.data.tsn; } /*- * CMT: CUCv2 algorithm. If (rtx-)pseudo-cumack for corresp * dest is being acked, then we have a new (rtx-)pseudo-cumack. Set * new_(rtx_)pseudo_cumack to TRUE so that the cwnd for this dest can be * updated. Also trigger search for the next expected (rtx-)pseudo-cumack. * Separate pseudo_cumack trackers for first transmissions and * retransmissions. */ if (tp1->rec.data.tsn == tp1->whoTo->pseudo_cumack) { if (tp1->rec.data.chunk_was_revoked == 0) { tp1->whoTo->new_pseudo_cumack = 1; } tp1->whoTo->find_pseudo_cumack = 1; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) { sctp_log_cwnd(stcb, tp1->whoTo, tp1->rec.data.tsn, SCTP_CWND_LOG_FROM_SACK); } if (tp1->rec.data.tsn == tp1->whoTo->rtx_pseudo_cumack) { if (tp1->rec.data.chunk_was_revoked == 0) { tp1->whoTo->new_pseudo_cumack = 1; } tp1->whoTo->find_rtx_pseudo_cumack = 1; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(*biggest_newly_acked_tsn, last_tsn, tp1->rec.data.tsn, frag_strt, frag_end, SCTP_LOG_TSN_ACKED); } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_DOWN_GAP, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } sctp_flight_size_decrease(tp1); if (stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) { (*stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) (tp1->whoTo, tp1); } sctp_total_flight_decrease(stcb, tp1); tp1->whoTo->net_ack += tp1->send_size; if (tp1->snd_count < 2) { /*- * True non-retransmited chunk */ tp1->whoTo->net_ack2 += tp1->send_size; /*- * update RTO too ? */ if (tp1->do_rtt) { if (*rto_ok) { tp1->whoTo->RTO = sctp_calculate_rto(stcb, &stcb->asoc, tp1->whoTo, &tp1->sent_rcv_time, sctp_align_safe_nocopy, SCTP_RTT_FROM_DATA); *rto_ok = 0; } if (tp1->whoTo->rto_needed == 0) { tp1->whoTo->rto_needed = 1; } tp1->do_rtt = 0; } } } if (tp1->sent <= SCTP_DATAGRAM_RESEND) { if (SCTP_TSN_GT(tp1->rec.data.tsn, stcb->asoc.this_sack_highest_gap)) { stcb->asoc.this_sack_highest_gap = tp1->rec.data.tsn; } if (tp1->sent == SCTP_DATAGRAM_RESEND) { sctp_ucount_decr(stcb->asoc.sent_queue_retran_cnt); #ifdef SCTP_AUDITING_ENABLED sctp_audit_log(0xB2, (stcb->asoc.sent_queue_retran_cnt & 0x000000ff)); #endif } } /*- * All chunks NOT UNSENT fall through here and are marked * (leave PR-SCTP ones that are to skip alone though) */ if ((tp1->sent != SCTP_FORWARD_TSN_SKIP) && (tp1->sent != SCTP_DATAGRAM_NR_ACKED)) { tp1->sent = SCTP_DATAGRAM_MARKED; } if (tp1->rec.data.chunk_was_revoked) { /* deflate the cwnd */ tp1->whoTo->cwnd -= tp1->book_size; tp1->rec.data.chunk_was_revoked = 0; } /* NR Sack code here */ if (nr_sacking && (tp1->sent != SCTP_DATAGRAM_NR_ACKED)) { if (stcb->asoc.strmout[tp1->rec.data.sid].chunks_on_queues > 0) { stcb->asoc.strmout[tp1->rec.data.sid].chunks_on_queues--; #ifdef INVARIANTS } else { panic("No chunks on the queues for sid %u.", tp1->rec.data.sid); #endif } if ((stcb->asoc.strmout[tp1->rec.data.sid].chunks_on_queues == 0) && (stcb->asoc.strmout[tp1->rec.data.sid].state == SCTP_STREAM_RESET_PENDING) && TAILQ_EMPTY(&stcb->asoc.strmout[tp1->rec.data.sid].outqueue)) { stcb->asoc.trigger_reset = 1; } tp1->sent = SCTP_DATAGRAM_NR_ACKED; if (tp1->data) { /* * sa_ignore * NO_NULL_CHK */ sctp_free_bufspace(stcb, &stcb->asoc, tp1, 1); sctp_m_freem(tp1->data); tp1->data = NULL; } wake_him++; } } break; } /* if (tp1->tsn == theTSN) */ if (SCTP_TSN_GT(tp1->rec.data.tsn, theTSN)) { break; } tp1 = TAILQ_NEXT(tp1, sctp_next); if ((tp1 == NULL) && (circled == 0)) { circled++; tp1 = TAILQ_FIRST(&stcb->asoc.sent_queue); } } /* end while (tp1) */ if (tp1 == NULL) { circled = 0; tp1 = TAILQ_FIRST(&stcb->asoc.sent_queue); } /* In case the fragments were not in order we must reset */ } /* end for (j = fragStart */ *p_tp1 = tp1; return (wake_him); /* Return value only used for nr-sack */ } static int sctp_handle_segments(struct mbuf *m, int *offset, struct sctp_tcb *stcb, struct sctp_association *asoc, uint32_t last_tsn, uint32_t *biggest_tsn_acked, uint32_t *biggest_newly_acked_tsn, uint32_t *this_sack_lowest_newack, int num_seg, int num_nr_seg, int *rto_ok) { struct sctp_gap_ack_block *frag, block; struct sctp_tmit_chunk *tp1; int i; int num_frs = 0; int chunk_freed; int non_revocable; uint16_t frag_strt, frag_end, prev_frag_end; tp1 = TAILQ_FIRST(&asoc->sent_queue); prev_frag_end = 0; chunk_freed = 0; for (i = 0; i < (num_seg + num_nr_seg); i++) { if (i == num_seg) { prev_frag_end = 0; tp1 = TAILQ_FIRST(&asoc->sent_queue); } frag = (struct sctp_gap_ack_block *)sctp_m_getptr(m, *offset, sizeof(struct sctp_gap_ack_block), (uint8_t *)&block); *offset += sizeof(block); if (frag == NULL) { return (chunk_freed); } frag_strt = ntohs(frag->start); frag_end = ntohs(frag->end); if (frag_strt > frag_end) { /* This gap report is malformed, skip it. */ continue; } if (frag_strt <= prev_frag_end) { /* This gap report is not in order, so restart. */ tp1 = TAILQ_FIRST(&asoc->sent_queue); } if (SCTP_TSN_GT((last_tsn + frag_end), *biggest_tsn_acked)) { *biggest_tsn_acked = last_tsn + frag_end; } if (i < num_seg) { non_revocable = 0; } else { non_revocable = 1; } if (sctp_process_segment_range(stcb, &tp1, last_tsn, frag_strt, frag_end, non_revocable, &num_frs, biggest_newly_acked_tsn, this_sack_lowest_newack, rto_ok)) { chunk_freed = 1; } prev_frag_end = frag_end; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { if (num_frs) sctp_log_fr(*biggest_tsn_acked, *biggest_newly_acked_tsn, last_tsn, SCTP_FR_LOG_BIGGEST_TSNS); } return (chunk_freed); } static void sctp_check_for_revoked(struct sctp_tcb *stcb, struct sctp_association *asoc, uint32_t cumack, uint32_t biggest_tsn_acked) { struct sctp_tmit_chunk *tp1; TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (SCTP_TSN_GT(tp1->rec.data.tsn, cumack)) { /* * ok this guy is either ACK or MARKED. If it is * ACKED it has been previously acked but not this * time i.e. revoked. If it is MARKED it was ACK'ed * again. */ if (SCTP_TSN_GT(tp1->rec.data.tsn, biggest_tsn_acked)) { break; } if (tp1->sent == SCTP_DATAGRAM_ACKED) { /* it has been revoked */ tp1->sent = SCTP_DATAGRAM_SENT; tp1->rec.data.chunk_was_revoked = 1; /* * We must add this stuff back in to assure * timers and such get started. */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_UP_REVOKE, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } sctp_flight_size_increase(tp1); sctp_total_flight_increase(stcb, tp1); /* * We inflate the cwnd to compensate for our * artificial inflation of the flight_size. */ tp1->whoTo->cwnd += tp1->book_size; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(asoc->last_acked_seq, cumack, tp1->rec.data.tsn, 0, 0, SCTP_LOG_TSN_REVOKED); } } else if (tp1->sent == SCTP_DATAGRAM_MARKED) { /* it has been re-acked in this SACK */ tp1->sent = SCTP_DATAGRAM_ACKED; } } if (tp1->sent == SCTP_DATAGRAM_UNSENT) break; } } static void sctp_strike_gap_ack_chunks(struct sctp_tcb *stcb, struct sctp_association *asoc, uint32_t biggest_tsn_acked, uint32_t biggest_tsn_newly_acked, uint32_t this_sack_lowest_newack, int accum_moved) { struct sctp_tmit_chunk *tp1; int strike_flag = 0; struct timeval now; int tot_retrans = 0; uint32_t sending_seq; struct sctp_nets *net; int num_dests_sacked = 0; /* * select the sending_seq, this is either the next thing ready to be * sent but not transmitted, OR, the next seq we assign. */ tp1 = TAILQ_FIRST(&stcb->asoc.send_queue); if (tp1 == NULL) { sending_seq = asoc->sending_seq; } else { sending_seq = tp1->rec.data.tsn; } /* CMT DAC algo: finding out if SACK is a mixed SACK */ if ((asoc->sctp_cmt_on_off > 0) && SCTP_BASE_SYSCTL(sctp_cmt_use_dac)) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (net->saw_newack) num_dests_sacked++; } } if (stcb->asoc.prsctp_supported) { (void)SCTP_GETTIME_TIMEVAL(&now); } TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { strike_flag = 0; if (tp1->no_fr_allowed) { /* this one had a timeout or something */ continue; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { if (tp1->sent < SCTP_DATAGRAM_RESEND) sctp_log_fr(biggest_tsn_newly_acked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_CHECK_STRIKE); } if (SCTP_TSN_GT(tp1->rec.data.tsn, biggest_tsn_acked) || tp1->sent == SCTP_DATAGRAM_UNSENT) { /* done */ break; } if (stcb->asoc.prsctp_supported) { if ((PR_SCTP_TTL_ENABLED(tp1->flags)) && tp1->sent < SCTP_DATAGRAM_ACKED) { /* Is it expired? */ if (timevalcmp(&now, &tp1->rec.data.timetodrop, >)) { /* Yes so drop it */ if (tp1->data != NULL) { (void)sctp_release_pr_sctp_chunk(stcb, tp1, 1, SCTP_SO_NOT_LOCKED); } continue; } } } if (SCTP_TSN_GT(tp1->rec.data.tsn, asoc->this_sack_highest_gap)) { /* we are beyond the tsn in the sack */ break; } if (tp1->sent >= SCTP_DATAGRAM_RESEND) { /* either a RESEND, ACKED, or MARKED */ /* skip */ if (tp1->sent == SCTP_FORWARD_TSN_SKIP) { /* Continue strikin FWD-TSN chunks */ tp1->rec.data.fwd_tsn_cnt++; } continue; } /* * CMT : SFR algo (covers part of DAC and HTNA as well) */ if (tp1->whoTo && tp1->whoTo->saw_newack == 0) { /* * No new acks were receieved for data sent to this * dest. Therefore, according to the SFR algo for * CMT, no data sent to this dest can be marked for * FR using this SACK. */ continue; } else if (tp1->whoTo && SCTP_TSN_GT(tp1->rec.data.tsn, tp1->whoTo->this_sack_highest_newack)) { /* * CMT: New acks were receieved for data sent to * this dest. But no new acks were seen for data * sent after tp1. Therefore, according to the SFR * algo for CMT, tp1 cannot be marked for FR using * this SACK. This step covers part of the DAC algo * and the HTNA algo as well. */ continue; } /* * Here we check to see if we were have already done a FR * and if so we see if the biggest TSN we saw in the sack is * smaller than the recovery point. If so we don't strike * the tsn... otherwise we CAN strike the TSN. */ /* * @@@ JRI: Check for CMT if (accum_moved && * asoc->fast_retran_loss_recovery && (sctp_cmt_on_off == * 0)) { */ if (accum_moved && asoc->fast_retran_loss_recovery) { /* * Strike the TSN if in fast-recovery and cum-ack * moved. */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(biggest_tsn_newly_acked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } if (tp1->sent < SCTP_DATAGRAM_RESEND) { tp1->sent++; } if ((asoc->sctp_cmt_on_off > 0) && SCTP_BASE_SYSCTL(sctp_cmt_use_dac)) { /* * CMT DAC algorithm: If SACK flag is set to * 0, then lowest_newack test will not pass * because it would have been set to the * cumack earlier. If not already to be * rtx'd, If not a mixed sack and if tp1 is * not between two sacked TSNs, then mark by * one more. NOTE that we are marking by one * additional time since the SACK DAC flag * indicates that two packets have been * received after this missing TSN. */ if ((tp1->sent < SCTP_DATAGRAM_RESEND) && (num_dests_sacked == 1) && SCTP_TSN_GT(this_sack_lowest_newack, tp1->rec.data.tsn)) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(16 + num_dests_sacked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } tp1->sent++; } } } else if ((tp1->rec.data.doing_fast_retransmit) && (asoc->sctp_cmt_on_off == 0)) { /* * For those that have done a FR we must take * special consideration if we strike. I.e the * biggest_newly_acked must be higher than the * sending_seq at the time we did the FR. */ if ( #ifdef SCTP_FR_TO_ALTERNATE /* * If FR's go to new networks, then we must only do * this for singly homed asoc's. However if the FR's * go to the same network (Armando's work) then its * ok to FR multiple times. */ (asoc->numnets < 2) #else (1) #endif ) { if (SCTP_TSN_GE(biggest_tsn_newly_acked, tp1->rec.data.fast_retran_tsn)) { /* * Strike the TSN, since this ack is * beyond where things were when we * did a FR. */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(biggest_tsn_newly_acked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } if (tp1->sent < SCTP_DATAGRAM_RESEND) { tp1->sent++; } strike_flag = 1; if ((asoc->sctp_cmt_on_off > 0) && SCTP_BASE_SYSCTL(sctp_cmt_use_dac)) { /* * CMT DAC algorithm: If * SACK flag is set to 0, * then lowest_newack test * will not pass because it * would have been set to * the cumack earlier. If * not already to be rtx'd, * If not a mixed sack and * if tp1 is not between two * sacked TSNs, then mark by * one more. NOTE that we * are marking by one * additional time since the * SACK DAC flag indicates * that two packets have * been received after this * missing TSN. */ if ((tp1->sent < SCTP_DATAGRAM_RESEND) && (num_dests_sacked == 1) && SCTP_TSN_GT(this_sack_lowest_newack, tp1->rec.data.tsn)) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(32 + num_dests_sacked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } if (tp1->sent < SCTP_DATAGRAM_RESEND) { tp1->sent++; } } } } } /* * JRI: TODO: remove code for HTNA algo. CMT's SFR * algo covers HTNA. */ } else if (SCTP_TSN_GT(tp1->rec.data.tsn, biggest_tsn_newly_acked)) { /* * We don't strike these: This is the HTNA * algorithm i.e. we don't strike If our TSN is * larger than the Highest TSN Newly Acked. */ ; } else { /* Strike the TSN */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(biggest_tsn_newly_acked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } if (tp1->sent < SCTP_DATAGRAM_RESEND) { tp1->sent++; } if ((asoc->sctp_cmt_on_off > 0) && SCTP_BASE_SYSCTL(sctp_cmt_use_dac)) { /* * CMT DAC algorithm: If SACK flag is set to * 0, then lowest_newack test will not pass * because it would have been set to the * cumack earlier. If not already to be * rtx'd, If not a mixed sack and if tp1 is * not between two sacked TSNs, then mark by * one more. NOTE that we are marking by one * additional time since the SACK DAC flag * indicates that two packets have been * received after this missing TSN. */ if ((tp1->sent < SCTP_DATAGRAM_RESEND) && (num_dests_sacked == 1) && SCTP_TSN_GT(this_sack_lowest_newack, tp1->rec.data.tsn)) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(48 + num_dests_sacked, tp1->rec.data.tsn, tp1->sent, SCTP_FR_LOG_STRIKE_CHUNK); } tp1->sent++; } } } if (tp1->sent == SCTP_DATAGRAM_RESEND) { struct sctp_nets *alt; /* fix counts and things */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_DOWN_RSND, (tp1->whoTo ? (tp1->whoTo->flight_size) : 0), tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } if (tp1->whoTo) { tp1->whoTo->net_ack++; sctp_flight_size_decrease(tp1); if (stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) { (*stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) (tp1->whoTo, tp1); } } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_RWND_ENABLE) { sctp_log_rwnd(SCTP_INCREASE_PEER_RWND, asoc->peers_rwnd, tp1->send_size, SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)); } /* add back to the rwnd */ asoc->peers_rwnd += (tp1->send_size + SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)); /* remove from the total flight */ sctp_total_flight_decrease(stcb, tp1); if ((stcb->asoc.prsctp_supported) && (PR_SCTP_RTX_ENABLED(tp1->flags))) { /* * Has it been retransmitted tv_sec times? - * we store the retran count there. */ if (tp1->snd_count > tp1->rec.data.timetodrop.tv_sec) { /* Yes, so drop it */ if (tp1->data != NULL) { (void)sctp_release_pr_sctp_chunk(stcb, tp1, 1, SCTP_SO_NOT_LOCKED); } /* Make sure to flag we had a FR */ tp1->whoTo->net_ack++; continue; } } /* * SCTP_PRINTF("OK, we are now ready to FR this * guy\n"); */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) { sctp_log_fr(tp1->rec.data.tsn, tp1->snd_count, 0, SCTP_FR_MARKED); } if (strike_flag) { /* This is a subsequent FR */ SCTP_STAT_INCR(sctps_sendmultfastretrans); } sctp_ucount_incr(stcb->asoc.sent_queue_retran_cnt); if (asoc->sctp_cmt_on_off > 0) { /* * CMT: Using RTX_SSTHRESH policy for CMT. * If CMT is being used, then pick dest with * largest ssthresh for any retransmission. */ tp1->no_fr_allowed = 1; alt = tp1->whoTo; /* sa_ignore NO_NULL_CHK */ if (asoc->sctp_cmt_pf > 0) { /* * JRS 5/18/07 - If CMT PF is on, * use the PF version of * find_alt_net() */ alt = sctp_find_alternate_net(stcb, alt, 2); } else { /* * JRS 5/18/07 - If only CMT is on, * use the CMT version of * find_alt_net() */ /* sa_ignore NO_NULL_CHK */ alt = sctp_find_alternate_net(stcb, alt, 1); } if (alt == NULL) { alt = tp1->whoTo; } /* * CUCv2: If a different dest is picked for * the retransmission, then new * (rtx-)pseudo_cumack needs to be tracked * for orig dest. Let CUCv2 track new (rtx-) * pseudo-cumack always. */ if (tp1->whoTo) { tp1->whoTo->find_pseudo_cumack = 1; tp1->whoTo->find_rtx_pseudo_cumack = 1; } } else {/* CMT is OFF */ #ifdef SCTP_FR_TO_ALTERNATE /* Can we find an alternate? */ alt = sctp_find_alternate_net(stcb, tp1->whoTo, 0); #else /* * default behavior is to NOT retransmit * FR's to an alternate. Armando Caro's * paper details why. */ alt = tp1->whoTo; #endif } tp1->rec.data.doing_fast_retransmit = 1; tot_retrans++; /* mark the sending seq for possible subsequent FR's */ /* * SCTP_PRINTF("Marking TSN for FR new value %x\n", * (uint32_t)tpi->rec.data.tsn); */ if (TAILQ_EMPTY(&asoc->send_queue)) { /* * If the queue of send is empty then its * the next sequence number that will be * assigned so we subtract one from this to * get the one we last sent. */ tp1->rec.data.fast_retran_tsn = sending_seq; } else { /* * If there are chunks on the send queue * (unsent data that has made it from the * stream queues but not out the door, we * take the first one (which will have the * lowest TSN) and subtract one to get the * one we last sent. */ struct sctp_tmit_chunk *ttt; ttt = TAILQ_FIRST(&asoc->send_queue); tp1->rec.data.fast_retran_tsn = ttt->rec.data.tsn; } if (tp1->do_rtt) { /* * this guy had a RTO calculation pending on * it, cancel it */ if ((tp1->whoTo != NULL) && (tp1->whoTo->rto_needed == 0)) { tp1->whoTo->rto_needed = 1; } tp1->do_rtt = 0; } if (alt != tp1->whoTo) { /* yes, there is an alternate. */ sctp_free_remote_addr(tp1->whoTo); /* sa_ignore FREED_MEMORY */ tp1->whoTo = alt; atomic_add_int(&alt->ref_count, 1); } } } } struct sctp_tmit_chunk * sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb, struct sctp_association *asoc) { struct sctp_tmit_chunk *tp1, *tp2, *a_adv = NULL; struct timeval now; int now_filled = 0; if (asoc->prsctp_supported == 0) { return (NULL); } TAILQ_FOREACH_SAFE(tp1, &asoc->sent_queue, sctp_next, tp2) { if (tp1->sent != SCTP_FORWARD_TSN_SKIP && tp1->sent != SCTP_DATAGRAM_RESEND && tp1->sent != SCTP_DATAGRAM_NR_ACKED) { /* no chance to advance, out of here */ break; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) { if ((tp1->sent == SCTP_FORWARD_TSN_SKIP) || (tp1->sent == SCTP_DATAGRAM_NR_ACKED)) { sctp_misc_ints(SCTP_FWD_TSN_CHECK, asoc->advanced_peer_ack_point, tp1->rec.data.tsn, 0, 0); } } if (!PR_SCTP_ENABLED(tp1->flags)) { /* * We can't fwd-tsn past any that are reliable aka * retransmitted until the asoc fails. */ break; } if (!now_filled) { (void)SCTP_GETTIME_TIMEVAL(&now); now_filled = 1; } /* * now we got a chunk which is marked for another * retransmission to a PR-stream but has run out its chances * already maybe OR has been marked to skip now. Can we skip * it if its a resend? */ if (tp1->sent == SCTP_DATAGRAM_RESEND && (PR_SCTP_TTL_ENABLED(tp1->flags))) { /* * Now is this one marked for resend and its time is * now up? */ if (timevalcmp(&now, &tp1->rec.data.timetodrop, >)) { /* Yes so drop it */ if (tp1->data) { (void)sctp_release_pr_sctp_chunk(stcb, tp1, 1, SCTP_SO_NOT_LOCKED); } } else { /* * No, we are done when hit one for resend * whos time as not expired. */ break; } } /* * Ok now if this chunk is marked to drop it we can clean up * the chunk, advance our peer ack point and we can check * the next chunk. */ if ((tp1->sent == SCTP_FORWARD_TSN_SKIP) || (tp1->sent == SCTP_DATAGRAM_NR_ACKED)) { /* advance PeerAckPoint goes forward */ if (SCTP_TSN_GT(tp1->rec.data.tsn, asoc->advanced_peer_ack_point)) { asoc->advanced_peer_ack_point = tp1->rec.data.tsn; a_adv = tp1; } else if (tp1->rec.data.tsn == asoc->advanced_peer_ack_point) { /* No update but we do save the chk */ a_adv = tp1; } } else { /* * If it is still in RESEND we can advance no * further */ break; } } return (a_adv); } static int sctp_fs_audit(struct sctp_association *asoc) { struct sctp_tmit_chunk *chk; int inflight = 0, resend = 0, inbetween = 0, acked = 0, above = 0; int ret; #ifndef INVARIANTS int entry_flight, entry_cnt; #endif ret = 0; #ifndef INVARIANTS entry_flight = asoc->total_flight; entry_cnt = asoc->total_flight_count; #endif if (asoc->pr_sctp_cnt >= asoc->sent_queue_cnt) return (0); TAILQ_FOREACH(chk, &asoc->sent_queue, sctp_next) { if (chk->sent < SCTP_DATAGRAM_RESEND) { SCTP_PRINTF("Chk TSN: %u size: %d inflight cnt: %d\n", chk->rec.data.tsn, chk->send_size, chk->snd_count); inflight++; } else if (chk->sent == SCTP_DATAGRAM_RESEND) { resend++; } else if (chk->sent < SCTP_DATAGRAM_ACKED) { inbetween++; } else if (chk->sent > SCTP_DATAGRAM_ACKED) { above++; } else { acked++; } } if ((inflight > 0) || (inbetween > 0)) { #ifdef INVARIANTS panic("Flight size-express incorrect? \n"); #else SCTP_PRINTF("asoc->total_flight: %d cnt: %d\n", entry_flight, entry_cnt); SCTP_PRINTF("Flight size-express incorrect F: %d I: %d R: %d Ab: %d ACK: %d\n", inflight, inbetween, resend, above, acked); ret = 1; #endif } return (ret); } static void sctp_window_probe_recovery(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_tmit_chunk *tp1) { tp1->window_probe = 0; if ((tp1->sent >= SCTP_DATAGRAM_ACKED) || (tp1->data == NULL)) { /* TSN's skipped we do NOT move back. */ sctp_misc_ints(SCTP_FLIGHT_LOG_DWN_WP_FWD, tp1->whoTo ? tp1->whoTo->flight_size : 0, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); return; } /* First setup this by shrinking flight */ if (stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) { (*stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) (tp1->whoTo, tp1); } sctp_flight_size_decrease(tp1); sctp_total_flight_decrease(stcb, tp1); /* Now mark for resend */ tp1->sent = SCTP_DATAGRAM_RESEND; sctp_ucount_incr(asoc->sent_queue_retran_cnt); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_DOWN_WP, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } } void sctp_express_handle_sack(struct sctp_tcb *stcb, uint32_t cumack, uint32_t rwnd, int *abort_now, int ecne_seen) { struct sctp_nets *net; struct sctp_association *asoc; struct sctp_tmit_chunk *tp1, *tp2; uint32_t old_rwnd; int win_probe_recovery = 0; int win_probe_recovered = 0; int j, done_once = 0; int rto_ok = 1; uint32_t send_s; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_SACK_ARRIVALS_ENABLE) { sctp_misc_ints(SCTP_SACK_LOG_EXPRESS, cumack, rwnd, stcb->asoc.last_acked_seq, stcb->asoc.peers_rwnd); } SCTP_TCB_LOCK_ASSERT(stcb); #ifdef SCTP_ASOCLOG_OF_TSNS stcb->asoc.cumack_log[stcb->asoc.cumack_log_at] = cumack; stcb->asoc.cumack_log_at++; if (stcb->asoc.cumack_log_at > SCTP_TSN_LOG_SIZE) { stcb->asoc.cumack_log_at = 0; } #endif asoc = &stcb->asoc; old_rwnd = asoc->peers_rwnd; if (SCTP_TSN_GT(asoc->last_acked_seq, cumack)) { /* old ack */ return; } else if (asoc->last_acked_seq == cumack) { /* Window update sack */ asoc->peers_rwnd = sctp_sbspace_sub(rwnd, (uint32_t)(asoc->total_flight + (asoc->total_flight_count * SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)))); if (asoc->peers_rwnd < stcb->sctp_ep->sctp_ep.sctp_sws_sender) { /* SWS sender side engages */ asoc->peers_rwnd = 0; } if (asoc->peers_rwnd > old_rwnd) { goto again; } return; } /* First setup for CC stuff */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (SCTP_TSN_GT(cumack, net->cwr_window_tsn)) { /* Drag along the window_tsn for cwr's */ net->cwr_window_tsn = cumack; } net->prev_cwnd = net->cwnd; net->net_ack = 0; net->net_ack2 = 0; /* * CMT: Reset CUC and Fast recovery algo variables before * SACK processing */ net->new_pseudo_cumack = 0; net->will_exit_fast_recovery = 0; if (stcb->asoc.cc_functions.sctp_cwnd_prepare_net_for_sack) { (*stcb->asoc.cc_functions.sctp_cwnd_prepare_net_for_sack) (stcb, net); } } if (!TAILQ_EMPTY(&asoc->sent_queue)) { tp1 = TAILQ_LAST(&asoc->sent_queue, sctpchunk_listhead); send_s = tp1->rec.data.tsn + 1; } else { send_s = asoc->sending_seq; } if (SCTP_TSN_GE(cumack, send_s)) { struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; *abort_now = 1; /* XXX */ snprintf(msg, sizeof(msg), "Cum ack %8.8x greater or equal than TSN %8.8x", cumack, send_s); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_21; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); return; } asoc->this_sack_highest_gap = cumack; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_THRESHOLD_LOGGING) { sctp_misc_ints(SCTP_THRESHOLD_CLEAR, stcb->asoc.overall_error_count, 0, SCTP_FROM_SCTP_INDATA, __LINE__); } stcb->asoc.overall_error_count = 0; if (SCTP_TSN_GT(cumack, asoc->last_acked_seq)) { /* process the new consecutive TSN first */ TAILQ_FOREACH_SAFE(tp1, &asoc->sent_queue, sctp_next, tp2) { if (SCTP_TSN_GE(cumack, tp1->rec.data.tsn)) { if (tp1->sent == SCTP_DATAGRAM_UNSENT) { SCTP_PRINTF("Warning, an unsent is now acked?\n"); } if (tp1->sent < SCTP_DATAGRAM_ACKED) { /* * If it is less than ACKED, it is * now no-longer in flight. Higher * values may occur during marking */ if (tp1->sent < SCTP_DATAGRAM_RESEND) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_DOWN_CA, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } sctp_flight_size_decrease(tp1); if (stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) { (*stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) (tp1->whoTo, tp1); } /* sa_ignore NO_NULL_CHK */ sctp_total_flight_decrease(stcb, tp1); } tp1->whoTo->net_ack += tp1->send_size; if (tp1->snd_count < 2) { /* * True non-retransmited * chunk */ tp1->whoTo->net_ack2 += tp1->send_size; /* update RTO too? */ if (tp1->do_rtt) { if (rto_ok) { tp1->whoTo->RTO = /* * sa_ignore * NO_NULL_CHK */ sctp_calculate_rto(stcb, asoc, tp1->whoTo, &tp1->sent_rcv_time, sctp_align_safe_nocopy, SCTP_RTT_FROM_DATA); rto_ok = 0; } if (tp1->whoTo->rto_needed == 0) { tp1->whoTo->rto_needed = 1; } tp1->do_rtt = 0; } } /* * CMT: CUCv2 algorithm. From the * cumack'd TSNs, for each TSN being * acked for the first time, set the * following variables for the * corresp destination. * new_pseudo_cumack will trigger a * cwnd update. * find_(rtx_)pseudo_cumack will * trigger search for the next * expected (rtx-)pseudo-cumack. */ tp1->whoTo->new_pseudo_cumack = 1; tp1->whoTo->find_pseudo_cumack = 1; tp1->whoTo->find_rtx_pseudo_cumack = 1; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) { /* sa_ignore NO_NULL_CHK */ sctp_log_cwnd(stcb, tp1->whoTo, tp1->rec.data.tsn, SCTP_CWND_LOG_FROM_SACK); } } if (tp1->sent == SCTP_DATAGRAM_RESEND) { sctp_ucount_decr(asoc->sent_queue_retran_cnt); } if (tp1->rec.data.chunk_was_revoked) { /* deflate the cwnd */ tp1->whoTo->cwnd -= tp1->book_size; tp1->rec.data.chunk_was_revoked = 0; } if (tp1->sent != SCTP_DATAGRAM_NR_ACKED) { if (asoc->strmout[tp1->rec.data.sid].chunks_on_queues > 0) { asoc->strmout[tp1->rec.data.sid].chunks_on_queues--; #ifdef INVARIANTS } else { panic("No chunks on the queues for sid %u.", tp1->rec.data.sid); #endif } } if ((asoc->strmout[tp1->rec.data.sid].chunks_on_queues == 0) && (asoc->strmout[tp1->rec.data.sid].state == SCTP_STREAM_RESET_PENDING) && TAILQ_EMPTY(&asoc->strmout[tp1->rec.data.sid].outqueue)) { asoc->trigger_reset = 1; } TAILQ_REMOVE(&asoc->sent_queue, tp1, sctp_next); if (tp1->data) { /* sa_ignore NO_NULL_CHK */ sctp_free_bufspace(stcb, asoc, tp1, 1); sctp_m_freem(tp1->data); tp1->data = NULL; } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(asoc->last_acked_seq, cumack, tp1->rec.data.tsn, 0, 0, SCTP_LOG_FREE_SENT); } asoc->sent_queue_cnt--; sctp_free_a_chunk(stcb, tp1, SCTP_SO_NOT_LOCKED); } else { break; } } } /* sa_ignore NO_NULL_CHK */ if (stcb->sctp_socket) { #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) struct socket *so; #endif SOCKBUF_LOCK(&stcb->sctp_socket->so_snd); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_WAKE_LOGGING_ENABLE) { /* sa_ignore NO_NULL_CHK */ sctp_wakeup_log(stcb, 1, SCTP_WAKESND_FROM_SACK); } #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) so = SCTP_INP_SO(stcb->sctp_ep); atomic_add_int(&stcb->asoc.refcnt, 1); SCTP_TCB_UNLOCK(stcb); SCTP_SOCKET_LOCK(so, 1); SCTP_TCB_LOCK(stcb); atomic_subtract_int(&stcb->asoc.refcnt, 1); if (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET) { /* assoc was freed while we were unlocked */ SCTP_SOCKET_UNLOCK(so, 1); return; } #endif sctp_sowwakeup_locked(stcb->sctp_ep, stcb->sctp_socket); #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) SCTP_SOCKET_UNLOCK(so, 1); #endif } else { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_WAKE_LOGGING_ENABLE) { sctp_wakeup_log(stcb, 1, SCTP_NOWAKE_FROM_SACK); } } /* JRS - Use the congestion control given in the CC module */ if ((asoc->last_acked_seq != cumack) && (ecne_seen == 0)) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (net->net_ack2 > 0) { /* * Karn's rule applies to clearing error * count, this is optional. */ net->error_count = 0; if (!(net->dest_state & SCTP_ADDR_REACHABLE)) { /* addr came good */ net->dest_state |= SCTP_ADDR_REACHABLE; sctp_ulp_notify(SCTP_NOTIFY_INTERFACE_UP, stcb, 0, (void *)net, SCTP_SO_NOT_LOCKED); } if (net == stcb->asoc.primary_destination) { if (stcb->asoc.alternate) { /* * release the alternate, * primary is good */ sctp_free_remote_addr(stcb->asoc.alternate); stcb->asoc.alternate = NULL; } } if (net->dest_state & SCTP_ADDR_PF) { net->dest_state &= ~SCTP_ADDR_PF; sctp_timer_stop(SCTP_TIMER_TYPE_HEARTBEAT, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_22); sctp_timer_start(SCTP_TIMER_TYPE_HEARTBEAT, stcb->sctp_ep, stcb, net); asoc->cc_functions.sctp_cwnd_update_exit_pf(stcb, net); /* Done with this net */ net->net_ack = 0; } /* restore any doubled timers */ net->RTO = (net->lastsa >> SCTP_RTT_SHIFT) + net->lastsv; if (net->RTO < stcb->asoc.minrto) { net->RTO = stcb->asoc.minrto; } if (net->RTO > stcb->asoc.maxrto) { net->RTO = stcb->asoc.maxrto; } } } asoc->cc_functions.sctp_cwnd_update_after_sack(stcb, asoc, 1, 0, 0); } asoc->last_acked_seq = cumack; if (TAILQ_EMPTY(&asoc->sent_queue)) { /* nothing left in-flight */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { net->flight_size = 0; net->partial_bytes_acked = 0; } asoc->total_flight = 0; asoc->total_flight_count = 0; } /* RWND update */ asoc->peers_rwnd = sctp_sbspace_sub(rwnd, (uint32_t)(asoc->total_flight + (asoc->total_flight_count * SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)))); if (asoc->peers_rwnd < stcb->sctp_ep->sctp_ep.sctp_sws_sender) { /* SWS sender side engages */ asoc->peers_rwnd = 0; } if (asoc->peers_rwnd > old_rwnd) { win_probe_recovery = 1; } /* Now assure a timer where data is queued at */ again: j = 0; TAILQ_FOREACH(net, &asoc->nets, sctp_next) { int to_ticks; if (win_probe_recovery && (net->window_probe)) { win_probe_recovered = 1; /* * Find first chunk that was used with window probe * and clear the sent */ /* sa_ignore FREED_MEMORY */ TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (tp1->window_probe) { /* move back to data send queue */ sctp_window_probe_recovery(stcb, asoc, tp1); break; } } } if (net->RTO == 0) { to_ticks = MSEC_TO_TICKS(stcb->asoc.initial_rto); } else { to_ticks = MSEC_TO_TICKS(net->RTO); } if (net->flight_size) { j++; (void)SCTP_OS_TIMER_START(&net->rxt_timer.timer, to_ticks, sctp_timeout_handler, &net->rxt_timer); if (net->window_probe) { net->window_probe = 0; } } else { if (net->window_probe) { /* * In window probes we must assure a timer * is still running there */ net->window_probe = 0; if (!SCTP_OS_TIMER_PENDING(&net->rxt_timer.timer)) { SCTP_OS_TIMER_START(&net->rxt_timer.timer, to_ticks, sctp_timeout_handler, &net->rxt_timer); } } else if (SCTP_OS_TIMER_PENDING(&net->rxt_timer.timer)) { sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_23); } } } if ((j == 0) && (!TAILQ_EMPTY(&asoc->sent_queue)) && (asoc->sent_queue_retran_cnt == 0) && (win_probe_recovered == 0) && (done_once == 0)) { /* * huh, this should not happen unless all packets are * PR-SCTP and marked to skip of course. */ if (sctp_fs_audit(asoc)) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { net->flight_size = 0; } asoc->total_flight = 0; asoc->total_flight_count = 0; asoc->sent_queue_retran_cnt = 0; TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (tp1->sent < SCTP_DATAGRAM_RESEND) { sctp_flight_size_increase(tp1); sctp_total_flight_increase(stcb, tp1); } else if (tp1->sent == SCTP_DATAGRAM_RESEND) { sctp_ucount_incr(asoc->sent_queue_retran_cnt); } } } done_once = 1; goto again; } /**********************************/ /* Now what about shutdown issues */ /**********************************/ if (TAILQ_EMPTY(&asoc->send_queue) && TAILQ_EMPTY(&asoc->sent_queue)) { /* nothing left on sendqueue.. consider done */ /* clean up */ if ((asoc->stream_queue_cnt == 1) && ((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) || (asoc->state & SCTP_STATE_SHUTDOWN_RECEIVED)) && ((*asoc->ss_functions.sctp_ss_is_user_msgs_incomplete) (stcb, asoc))) { asoc->state |= SCTP_STATE_PARTIAL_MSG_LEFT; } if (((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) || (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED)) && (asoc->stream_queue_cnt == 1) && (asoc->state & SCTP_STATE_PARTIAL_MSG_LEFT)) { struct mbuf *op_err; *abort_now = 1; /* XXX */ op_err = sctp_generate_cause(SCTP_CAUSE_USER_INITIATED_ABT, ""); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_24; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); return; } if ((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) && (asoc->stream_queue_cnt == 0)) { struct sctp_nets *netp; if ((SCTP_GET_STATE(asoc) == SCTP_STATE_OPEN) || (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED)) { SCTP_STAT_DECR_GAUGE32(sctps_currestab); } SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT); SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING); sctp_stop_timers_for_shutdown(stcb); if (asoc->alternate) { netp = asoc->alternate; } else { netp = asoc->primary_destination; } sctp_send_shutdown(stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN, stcb->sctp_ep, stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNGUARD, stcb->sctp_ep, stcb, netp); } else if ((SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED) && (asoc->stream_queue_cnt == 0)) { struct sctp_nets *netp; SCTP_STAT_DECR_GAUGE32(sctps_currestab); SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_ACK_SENT); SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING); sctp_stop_timers_for_shutdown(stcb); if (asoc->alternate) { netp = asoc->alternate; } else { netp = asoc->primary_destination; } sctp_send_shutdown_ack(stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNACK, stcb->sctp_ep, stcb, netp); } } /*********************************************/ /* Here we perform PR-SCTP procedures */ /* (section 4.2) */ /*********************************************/ /* C1. update advancedPeerAckPoint */ if (SCTP_TSN_GT(cumack, asoc->advanced_peer_ack_point)) { asoc->advanced_peer_ack_point = cumack; } /* PR-Sctp issues need to be addressed too */ if ((asoc->prsctp_supported) && (asoc->pr_sctp_cnt > 0)) { struct sctp_tmit_chunk *lchk; uint32_t old_adv_peer_ack_point; old_adv_peer_ack_point = asoc->advanced_peer_ack_point; lchk = sctp_try_advance_peer_ack_point(stcb, asoc); /* C3. See if we need to send a Fwd-TSN */ if (SCTP_TSN_GT(asoc->advanced_peer_ack_point, cumack)) { /* * ISSUE with ECN, see FWD-TSN processing. */ if (SCTP_TSN_GT(asoc->advanced_peer_ack_point, old_adv_peer_ack_point)) { send_forward_tsn(stcb, asoc); } else if (lchk) { /* try to FR fwd-tsn's that get lost too */ if (lchk->rec.data.fwd_tsn_cnt >= 3) { send_forward_tsn(stcb, asoc); } } } if (lchk) { /* Assure a timer is up */ sctp_timer_start(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, lchk->whoTo); } } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_RWND_LOGGING_ENABLE) { sctp_misc_ints(SCTP_SACK_RWND_UPDATE, rwnd, stcb->asoc.peers_rwnd, stcb->asoc.total_flight, stcb->asoc.total_output_queue_size); } } void sctp_handle_sack(struct mbuf *m, int offset_seg, int offset_dup, struct sctp_tcb *stcb, uint16_t num_seg, uint16_t num_nr_seg, uint16_t num_dup, int *abort_now, uint8_t flags, uint32_t cum_ack, uint32_t rwnd, int ecne_seen) { struct sctp_association *asoc; struct sctp_tmit_chunk *tp1, *tp2; uint32_t last_tsn, biggest_tsn_acked, biggest_tsn_newly_acked, this_sack_lowest_newack; uint16_t wake_him = 0; uint32_t send_s = 0; long j; int accum_moved = 0; int will_exit_fast_recovery = 0; uint32_t a_rwnd, old_rwnd; int win_probe_recovery = 0; int win_probe_recovered = 0; struct sctp_nets *net = NULL; int done_once; int rto_ok = 1; uint8_t reneged_all = 0; uint8_t cmt_dac_flag; /* * we take any chance we can to service our queues since we cannot * get awoken when the socket is read from :< */ /* * Now perform the actual SACK handling: 1) Verify that it is not an * old sack, if so discard. 2) If there is nothing left in the send * queue (cum-ack is equal to last acked) then you have a duplicate * too, update any rwnd change and verify no timers are running. * then return. 3) Process any new consequtive data i.e. cum-ack * moved process these first and note that it moved. 4) Process any * sack blocks. 5) Drop any acked from the queue. 6) Check for any * revoked blocks and mark. 7) Update the cwnd. 8) Nothing left, * sync up flightsizes and things, stop all timers and also check * for shutdown_pending state. If so then go ahead and send off the * shutdown. If in shutdown recv, send off the shutdown-ack and * start that timer, Ret. 9) Strike any non-acked things and do FR * procedure if needed being sure to set the FR flag. 10) Do pr-sctp * procedures. 11) Apply any FR penalties. 12) Assure we will SACK * if in shutdown_recv state. */ SCTP_TCB_LOCK_ASSERT(stcb); /* CMT DAC algo */ this_sack_lowest_newack = 0; SCTP_STAT_INCR(sctps_slowpath_sack); last_tsn = cum_ack; cmt_dac_flag = flags & SCTP_SACK_CMT_DAC; #ifdef SCTP_ASOCLOG_OF_TSNS stcb->asoc.cumack_log[stcb->asoc.cumack_log_at] = cum_ack; stcb->asoc.cumack_log_at++; if (stcb->asoc.cumack_log_at > SCTP_TSN_LOG_SIZE) { stcb->asoc.cumack_log_at = 0; } #endif a_rwnd = rwnd; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_SACK_ARRIVALS_ENABLE) { sctp_misc_ints(SCTP_SACK_LOG_NORMAL, cum_ack, rwnd, stcb->asoc.last_acked_seq, stcb->asoc.peers_rwnd); } old_rwnd = stcb->asoc.peers_rwnd; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_THRESHOLD_LOGGING) { sctp_misc_ints(SCTP_THRESHOLD_CLEAR, stcb->asoc.overall_error_count, 0, SCTP_FROM_SCTP_INDATA, __LINE__); } stcb->asoc.overall_error_count = 0; asoc = &stcb->asoc; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(asoc->last_acked_seq, cum_ack, 0, num_seg, num_dup, SCTP_LOG_NEW_SACK); } if ((num_dup) && (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE)) { uint16_t i; uint32_t *dupdata, dblock; for (i = 0; i < num_dup; i++) { dupdata = (uint32_t *)sctp_m_getptr(m, offset_dup + i * sizeof(uint32_t), sizeof(uint32_t), (uint8_t *)&dblock); if (dupdata == NULL) { break; } sctp_log_fr(*dupdata, 0, 0, SCTP_FR_DUPED); } } /* reality check */ if (!TAILQ_EMPTY(&asoc->sent_queue)) { tp1 = TAILQ_LAST(&asoc->sent_queue, sctpchunk_listhead); send_s = tp1->rec.data.tsn + 1; } else { tp1 = NULL; send_s = asoc->sending_seq; } if (SCTP_TSN_GE(cum_ack, send_s)) { struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; /* * no way, we have not even sent this TSN out yet. Peer is * hopelessly messed up with us. */ SCTP_PRINTF("NEW cum_ack:%x send_s:%x is smaller or equal\n", cum_ack, send_s); if (tp1) { SCTP_PRINTF("Got send_s from tsn:%x + 1 of tp1: %p\n", tp1->rec.data.tsn, (void *)tp1); } hopeless_peer: *abort_now = 1; /* XXX */ snprintf(msg, sizeof(msg), "Cum ack %8.8x greater or equal than TSN %8.8x", cum_ack, send_s); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_25; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); return; } /**********************/ /* 1) check the range */ /**********************/ if (SCTP_TSN_GT(asoc->last_acked_seq, last_tsn)) { /* acking something behind */ return; } /* update the Rwnd of the peer */ if (TAILQ_EMPTY(&asoc->sent_queue) && TAILQ_EMPTY(&asoc->send_queue) && (asoc->stream_queue_cnt == 0)) { /* nothing left on send/sent and strmq */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_RWND_ENABLE) { sctp_log_rwnd_set(SCTP_SET_PEER_RWND_VIA_SACK, asoc->peers_rwnd, 0, 0, a_rwnd); } asoc->peers_rwnd = a_rwnd; if (asoc->sent_queue_retran_cnt) { asoc->sent_queue_retran_cnt = 0; } if (asoc->peers_rwnd < stcb->sctp_ep->sctp_ep.sctp_sws_sender) { /* SWS sender side engages */ asoc->peers_rwnd = 0; } /* stop any timers */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_26); net->partial_bytes_acked = 0; net->flight_size = 0; } asoc->total_flight = 0; asoc->total_flight_count = 0; return; } /* * We init netAckSz and netAckSz2 to 0. These are used to track 2 * things. The total byte count acked is tracked in netAckSz AND * netAck2 is used to track the total bytes acked that are un- * amibguious and were never retransmitted. We track these on a per * destination address basis. */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (SCTP_TSN_GT(cum_ack, net->cwr_window_tsn)) { /* Drag along the window_tsn for cwr's */ net->cwr_window_tsn = cum_ack; } net->prev_cwnd = net->cwnd; net->net_ack = 0; net->net_ack2 = 0; /* * CMT: Reset CUC and Fast recovery algo variables before * SACK processing */ net->new_pseudo_cumack = 0; net->will_exit_fast_recovery = 0; if (stcb->asoc.cc_functions.sctp_cwnd_prepare_net_for_sack) { (*stcb->asoc.cc_functions.sctp_cwnd_prepare_net_for_sack) (stcb, net); } } /* process the new consecutive TSN first */ TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (SCTP_TSN_GE(last_tsn, tp1->rec.data.tsn)) { if (tp1->sent != SCTP_DATAGRAM_UNSENT) { accum_moved = 1; if (tp1->sent < SCTP_DATAGRAM_ACKED) { /* * If it is less than ACKED, it is * now no-longer in flight. Higher * values may occur during marking */ if ((tp1->whoTo->dest_state & SCTP_ADDR_UNCONFIRMED) && (tp1->snd_count < 2)) { /* * If there was no retran * and the address is * un-confirmed and we sent * there and are now * sacked.. its confirmed, * mark it so. */ tp1->whoTo->dest_state &= ~SCTP_ADDR_UNCONFIRMED; } if (tp1->sent < SCTP_DATAGRAM_RESEND) { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_DOWN_CA, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } sctp_flight_size_decrease(tp1); sctp_total_flight_decrease(stcb, tp1); if (stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) { (*stcb->asoc.cc_functions.sctp_cwnd_update_tsn_acknowledged) (tp1->whoTo, tp1); } } tp1->whoTo->net_ack += tp1->send_size; /* CMT SFR and DAC algos */ this_sack_lowest_newack = tp1->rec.data.tsn; tp1->whoTo->saw_newack = 1; if (tp1->snd_count < 2) { /* * True non-retransmited * chunk */ tp1->whoTo->net_ack2 += tp1->send_size; /* update RTO too? */ if (tp1->do_rtt) { if (rto_ok) { tp1->whoTo->RTO = sctp_calculate_rto(stcb, asoc, tp1->whoTo, &tp1->sent_rcv_time, sctp_align_safe_nocopy, SCTP_RTT_FROM_DATA); rto_ok = 0; } if (tp1->whoTo->rto_needed == 0) { tp1->whoTo->rto_needed = 1; } tp1->do_rtt = 0; } } /* * CMT: CUCv2 algorithm. From the * cumack'd TSNs, for each TSN being * acked for the first time, set the * following variables for the * corresp destination. * new_pseudo_cumack will trigger a * cwnd update. * find_(rtx_)pseudo_cumack will * trigger search for the next * expected (rtx-)pseudo-cumack. */ tp1->whoTo->new_pseudo_cumack = 1; tp1->whoTo->find_pseudo_cumack = 1; tp1->whoTo->find_rtx_pseudo_cumack = 1; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(asoc->last_acked_seq, cum_ack, tp1->rec.data.tsn, 0, 0, SCTP_LOG_TSN_ACKED); } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) { sctp_log_cwnd(stcb, tp1->whoTo, tp1->rec.data.tsn, SCTP_CWND_LOG_FROM_SACK); } } if (tp1->sent == SCTP_DATAGRAM_RESEND) { sctp_ucount_decr(asoc->sent_queue_retran_cnt); #ifdef SCTP_AUDITING_ENABLED sctp_audit_log(0xB3, (asoc->sent_queue_retran_cnt & 0x000000ff)); #endif } if (tp1->rec.data.chunk_was_revoked) { /* deflate the cwnd */ tp1->whoTo->cwnd -= tp1->book_size; tp1->rec.data.chunk_was_revoked = 0; } if (tp1->sent != SCTP_DATAGRAM_NR_ACKED) { tp1->sent = SCTP_DATAGRAM_ACKED; } } } else { break; } } biggest_tsn_newly_acked = biggest_tsn_acked = last_tsn; /* always set this up to cum-ack */ asoc->this_sack_highest_gap = last_tsn; if ((num_seg > 0) || (num_nr_seg > 0)) { /* * CMT: SFR algo (and HTNA) - this_sack_highest_newack has * to be greater than the cumack. Also reset saw_newack to 0 * for all dests. */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { net->saw_newack = 0; net->this_sack_highest_newack = last_tsn; } /* * thisSackHighestGap will increase while handling NEW * segments this_sack_highest_newack will increase while * handling NEWLY ACKED chunks. this_sack_lowest_newack is * used for CMT DAC algo. saw_newack will also change. */ if (sctp_handle_segments(m, &offset_seg, stcb, asoc, last_tsn, &biggest_tsn_acked, &biggest_tsn_newly_acked, &this_sack_lowest_newack, num_seg, num_nr_seg, &rto_ok)) { wake_him++; } /* * validate the biggest_tsn_acked in the gap acks if strict * adherence is wanted. */ if (SCTP_TSN_GE(biggest_tsn_acked, send_s)) { /* * peer is either confused or we are under attack. * We must abort. */ SCTP_PRINTF("Hopeless peer! biggest_tsn_acked:%x largest seq:%x\n", biggest_tsn_acked, send_s); goto hopeless_peer; } } /*******************************************/ /* cancel ALL T3-send timer if accum moved */ /*******************************************/ if (asoc->sctp_cmt_on_off > 0) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (net->new_pseudo_cumack) sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_27); } } else { if (accum_moved) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_28); } } } /********************************************/ /* drop the acked chunks from the sentqueue */ /********************************************/ asoc->last_acked_seq = cum_ack; TAILQ_FOREACH_SAFE(tp1, &asoc->sent_queue, sctp_next, tp2) { if (SCTP_TSN_GT(tp1->rec.data.tsn, cum_ack)) { break; } if (tp1->sent != SCTP_DATAGRAM_NR_ACKED) { if (asoc->strmout[tp1->rec.data.sid].chunks_on_queues > 0) { asoc->strmout[tp1->rec.data.sid].chunks_on_queues--; #ifdef INVARIANTS } else { panic("No chunks on the queues for sid %u.", tp1->rec.data.sid); #endif } } if ((asoc->strmout[tp1->rec.data.sid].chunks_on_queues == 0) && (asoc->strmout[tp1->rec.data.sid].state == SCTP_STREAM_RESET_PENDING) && TAILQ_EMPTY(&asoc->strmout[tp1->rec.data.sid].outqueue)) { asoc->trigger_reset = 1; } TAILQ_REMOVE(&asoc->sent_queue, tp1, sctp_next); if (PR_SCTP_ENABLED(tp1->flags)) { if (asoc->pr_sctp_cnt != 0) asoc->pr_sctp_cnt--; } asoc->sent_queue_cnt--; if (tp1->data) { /* sa_ignore NO_NULL_CHK */ sctp_free_bufspace(stcb, asoc, tp1, 1); sctp_m_freem(tp1->data); tp1->data = NULL; if (asoc->prsctp_supported && PR_SCTP_BUF_ENABLED(tp1->flags)) { asoc->sent_queue_cnt_removeable--; } } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_LOGGING_ENABLE) { sctp_log_sack(asoc->last_acked_seq, cum_ack, tp1->rec.data.tsn, 0, 0, SCTP_LOG_FREE_SENT); } sctp_free_a_chunk(stcb, tp1, SCTP_SO_NOT_LOCKED); wake_him++; } if (TAILQ_EMPTY(&asoc->sent_queue) && (asoc->total_flight > 0)) { #ifdef INVARIANTS panic("Warning flight size is positive and should be 0"); #else SCTP_PRINTF("Warning flight size incorrect should be 0 is %d\n", asoc->total_flight); #endif asoc->total_flight = 0; } /* sa_ignore NO_NULL_CHK */ if ((wake_him) && (stcb->sctp_socket)) { #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) struct socket *so; #endif SOCKBUF_LOCK(&stcb->sctp_socket->so_snd); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_WAKE_LOGGING_ENABLE) { sctp_wakeup_log(stcb, wake_him, SCTP_WAKESND_FROM_SACK); } #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) so = SCTP_INP_SO(stcb->sctp_ep); atomic_add_int(&stcb->asoc.refcnt, 1); SCTP_TCB_UNLOCK(stcb); SCTP_SOCKET_LOCK(so, 1); SCTP_TCB_LOCK(stcb); atomic_subtract_int(&stcb->asoc.refcnt, 1); if (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET) { /* assoc was freed while we were unlocked */ SCTP_SOCKET_UNLOCK(so, 1); return; } #endif sctp_sowwakeup_locked(stcb->sctp_ep, stcb->sctp_socket); #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) SCTP_SOCKET_UNLOCK(so, 1); #endif } else { if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_WAKE_LOGGING_ENABLE) { sctp_wakeup_log(stcb, wake_him, SCTP_NOWAKE_FROM_SACK); } } if (asoc->fast_retran_loss_recovery && accum_moved) { if (SCTP_TSN_GE(asoc->last_acked_seq, asoc->fast_recovery_tsn)) { /* Setup so we will exit RFC2582 fast recovery */ will_exit_fast_recovery = 1; } } /* * Check for revoked fragments: * * if Previous sack - Had no frags then we can't have any revoked if * Previous sack - Had frag's then - If we now have frags aka * num_seg > 0 call sctp_check_for_revoked() to tell if peer revoked * some of them. else - The peer revoked all ACKED fragments, since * we had some before and now we have NONE. */ if (num_seg) { sctp_check_for_revoked(stcb, asoc, cum_ack, biggest_tsn_acked); asoc->saw_sack_with_frags = 1; } else if (asoc->saw_sack_with_frags) { int cnt_revoked = 0; /* Peer revoked all dg's marked or acked */ TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (tp1->sent == SCTP_DATAGRAM_ACKED) { tp1->sent = SCTP_DATAGRAM_SENT; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FLIGHT_LOGGING_ENABLE) { sctp_misc_ints(SCTP_FLIGHT_LOG_UP_REVOKE, tp1->whoTo->flight_size, tp1->book_size, (uint32_t)(uintptr_t)tp1->whoTo, tp1->rec.data.tsn); } sctp_flight_size_increase(tp1); sctp_total_flight_increase(stcb, tp1); tp1->rec.data.chunk_was_revoked = 1; /* * To ensure that this increase in * flightsize, which is artificial, does not * throttle the sender, we also increase the * cwnd artificially. */ tp1->whoTo->cwnd += tp1->book_size; cnt_revoked++; } } if (cnt_revoked) { reneged_all = 1; } asoc->saw_sack_with_frags = 0; } if (num_nr_seg > 0) asoc->saw_sack_with_nr_frags = 1; else asoc->saw_sack_with_nr_frags = 0; /* JRS - Use the congestion control given in the CC module */ if (ecne_seen == 0) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (net->net_ack2 > 0) { /* * Karn's rule applies to clearing error * count, this is optional. */ net->error_count = 0; if (!(net->dest_state & SCTP_ADDR_REACHABLE)) { /* addr came good */ net->dest_state |= SCTP_ADDR_REACHABLE; sctp_ulp_notify(SCTP_NOTIFY_INTERFACE_UP, stcb, 0, (void *)net, SCTP_SO_NOT_LOCKED); } if (net == stcb->asoc.primary_destination) { if (stcb->asoc.alternate) { /* * release the alternate, * primary is good */ sctp_free_remote_addr(stcb->asoc.alternate); stcb->asoc.alternate = NULL; } } if (net->dest_state & SCTP_ADDR_PF) { net->dest_state &= ~SCTP_ADDR_PF; sctp_timer_stop(SCTP_TIMER_TYPE_HEARTBEAT, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_29); sctp_timer_start(SCTP_TIMER_TYPE_HEARTBEAT, stcb->sctp_ep, stcb, net); asoc->cc_functions.sctp_cwnd_update_exit_pf(stcb, net); /* Done with this net */ net->net_ack = 0; } /* restore any doubled timers */ net->RTO = (net->lastsa >> SCTP_RTT_SHIFT) + net->lastsv; if (net->RTO < stcb->asoc.minrto) { net->RTO = stcb->asoc.minrto; } if (net->RTO > stcb->asoc.maxrto) { net->RTO = stcb->asoc.maxrto; } } } asoc->cc_functions.sctp_cwnd_update_after_sack(stcb, asoc, accum_moved, reneged_all, will_exit_fast_recovery); } if (TAILQ_EMPTY(&asoc->sent_queue)) { /* nothing left in-flight */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { /* stop all timers */ sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_30); net->flight_size = 0; net->partial_bytes_acked = 0; } asoc->total_flight = 0; asoc->total_flight_count = 0; } /**********************************/ /* Now what about shutdown issues */ /**********************************/ if (TAILQ_EMPTY(&asoc->send_queue) && TAILQ_EMPTY(&asoc->sent_queue)) { /* nothing left on sendqueue.. consider done */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_RWND_ENABLE) { sctp_log_rwnd_set(SCTP_SET_PEER_RWND_VIA_SACK, asoc->peers_rwnd, 0, 0, a_rwnd); } asoc->peers_rwnd = a_rwnd; if (asoc->peers_rwnd < stcb->sctp_ep->sctp_ep.sctp_sws_sender) { /* SWS sender side engages */ asoc->peers_rwnd = 0; } /* clean up */ if ((asoc->stream_queue_cnt == 1) && ((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) || (asoc->state & SCTP_STATE_SHUTDOWN_RECEIVED)) && ((*asoc->ss_functions.sctp_ss_is_user_msgs_incomplete) (stcb, asoc))) { asoc->state |= SCTP_STATE_PARTIAL_MSG_LEFT; } if (((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) || (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED)) && (asoc->stream_queue_cnt == 1) && (asoc->state & SCTP_STATE_PARTIAL_MSG_LEFT)) { struct mbuf *op_err; *abort_now = 1; /* XXX */ op_err = sctp_generate_cause(SCTP_CAUSE_USER_INITIATED_ABT, ""); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_24; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); return; } if ((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) && (asoc->stream_queue_cnt == 0)) { struct sctp_nets *netp; if ((SCTP_GET_STATE(asoc) == SCTP_STATE_OPEN) || (SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED)) { SCTP_STAT_DECR_GAUGE32(sctps_currestab); } SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT); SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING); sctp_stop_timers_for_shutdown(stcb); if (asoc->alternate) { netp = asoc->alternate; } else { netp = asoc->primary_destination; } sctp_send_shutdown(stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN, stcb->sctp_ep, stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNGUARD, stcb->sctp_ep, stcb, netp); return; } else if ((SCTP_GET_STATE(asoc) == SCTP_STATE_SHUTDOWN_RECEIVED) && (asoc->stream_queue_cnt == 0)) { struct sctp_nets *netp; SCTP_STAT_DECR_GAUGE32(sctps_currestab); SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_ACK_SENT); SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING); sctp_stop_timers_for_shutdown(stcb); if (asoc->alternate) { netp = asoc->alternate; } else { netp = asoc->primary_destination; } sctp_send_shutdown_ack(stcb, netp); sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNACK, stcb->sctp_ep, stcb, netp); return; } } /* * Now here we are going to recycle net_ack for a different use... * HEADS UP. */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { net->net_ack = 0; } /* * CMT DAC algorithm: If SACK DAC flag was 0, then no extra marking * to be done. Setting this_sack_lowest_newack to the cum_ack will * automatically ensure that. */ if ((asoc->sctp_cmt_on_off > 0) && SCTP_BASE_SYSCTL(sctp_cmt_use_dac) && (cmt_dac_flag == 0)) { this_sack_lowest_newack = cum_ack; } if ((num_seg > 0) || (num_nr_seg > 0)) { sctp_strike_gap_ack_chunks(stcb, asoc, biggest_tsn_acked, biggest_tsn_newly_acked, this_sack_lowest_newack, accum_moved); } /* JRS - Use the congestion control given in the CC module */ asoc->cc_functions.sctp_cwnd_update_after_fr(stcb, asoc); /* Now are we exiting loss recovery ? */ if (will_exit_fast_recovery) { /* Ok, we must exit fast recovery */ asoc->fast_retran_loss_recovery = 0; } if ((asoc->sat_t3_loss_recovery) && SCTP_TSN_GE(asoc->last_acked_seq, asoc->sat_t3_recovery_tsn)) { /* end satellite t3 loss recovery */ asoc->sat_t3_loss_recovery = 0; } /* * CMT Fast recovery */ TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (net->will_exit_fast_recovery) { /* Ok, we must exit fast recovery */ net->fast_retran_loss_recovery = 0; } } /* Adjust and set the new rwnd value */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_RWND_ENABLE) { sctp_log_rwnd_set(SCTP_SET_PEER_RWND_VIA_SACK, asoc->peers_rwnd, asoc->total_flight, (asoc->total_flight_count * SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)), a_rwnd); } asoc->peers_rwnd = sctp_sbspace_sub(a_rwnd, (uint32_t)(asoc->total_flight + (asoc->total_flight_count * SCTP_BASE_SYSCTL(sctp_peer_chunk_oh)))); if (asoc->peers_rwnd < stcb->sctp_ep->sctp_ep.sctp_sws_sender) { /* SWS sender side engages */ asoc->peers_rwnd = 0; } if (asoc->peers_rwnd > old_rwnd) { win_probe_recovery = 1; } /* * Now we must setup so we have a timer up for anyone with * outstanding data. */ done_once = 0; again: j = 0; TAILQ_FOREACH(net, &asoc->nets, sctp_next) { if (win_probe_recovery && (net->window_probe)) { win_probe_recovered = 1; /*- * Find first chunk that was used with * window probe and clear the event. Put * it back into the send queue as if has * not been sent. */ TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (tp1->window_probe) { sctp_window_probe_recovery(stcb, asoc, tp1); break; } } } if (net->flight_size) { j++; if (!SCTP_OS_TIMER_PENDING(&net->rxt_timer.timer)) { sctp_timer_start(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net); } if (net->window_probe) { net->window_probe = 0; } } else { if (net->window_probe) { /* * In window probes we must assure a timer * is still running there */ if (!SCTP_OS_TIMER_PENDING(&net->rxt_timer.timer)) { sctp_timer_start(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net); } } else if (SCTP_OS_TIMER_PENDING(&net->rxt_timer.timer)) { sctp_timer_stop(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, net, SCTP_FROM_SCTP_INDATA + SCTP_LOC_32); } } } if ((j == 0) && (!TAILQ_EMPTY(&asoc->sent_queue)) && (asoc->sent_queue_retran_cnt == 0) && (win_probe_recovered == 0) && (done_once == 0)) { /* * huh, this should not happen unless all packets are * PR-SCTP and marked to skip of course. */ if (sctp_fs_audit(asoc)) { TAILQ_FOREACH(net, &asoc->nets, sctp_next) { net->flight_size = 0; } asoc->total_flight = 0; asoc->total_flight_count = 0; asoc->sent_queue_retran_cnt = 0; TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) { if (tp1->sent < SCTP_DATAGRAM_RESEND) { sctp_flight_size_increase(tp1); sctp_total_flight_increase(stcb, tp1); } else if (tp1->sent == SCTP_DATAGRAM_RESEND) { sctp_ucount_incr(asoc->sent_queue_retran_cnt); } } } done_once = 1; goto again; } /*********************************************/ /* Here we perform PR-SCTP procedures */ /* (section 4.2) */ /*********************************************/ /* C1. update advancedPeerAckPoint */ if (SCTP_TSN_GT(cum_ack, asoc->advanced_peer_ack_point)) { asoc->advanced_peer_ack_point = cum_ack; } /* C2. try to further move advancedPeerAckPoint ahead */ if ((asoc->prsctp_supported) && (asoc->pr_sctp_cnt > 0)) { struct sctp_tmit_chunk *lchk; uint32_t old_adv_peer_ack_point; old_adv_peer_ack_point = asoc->advanced_peer_ack_point; lchk = sctp_try_advance_peer_ack_point(stcb, asoc); /* C3. See if we need to send a Fwd-TSN */ if (SCTP_TSN_GT(asoc->advanced_peer_ack_point, cum_ack)) { /* * ISSUE with ECN, see FWD-TSN processing. */ if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) { sctp_misc_ints(SCTP_FWD_TSN_CHECK, 0xee, cum_ack, asoc->advanced_peer_ack_point, old_adv_peer_ack_point); } if (SCTP_TSN_GT(asoc->advanced_peer_ack_point, old_adv_peer_ack_point)) { send_forward_tsn(stcb, asoc); } else if (lchk) { /* try to FR fwd-tsn's that get lost too */ if (lchk->rec.data.fwd_tsn_cnt >= 3) { send_forward_tsn(stcb, asoc); } } } if (lchk) { /* Assure a timer is up */ sctp_timer_start(SCTP_TIMER_TYPE_SEND, stcb->sctp_ep, stcb, lchk->whoTo); } } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SACK_RWND_LOGGING_ENABLE) { sctp_misc_ints(SCTP_SACK_RWND_UPDATE, a_rwnd, stcb->asoc.peers_rwnd, stcb->asoc.total_flight, stcb->asoc.total_output_queue_size); } } void sctp_update_acked(struct sctp_tcb *stcb, struct sctp_shutdown_chunk *cp, int *abort_flag) { /* Copy cum-ack */ uint32_t cum_ack, a_rwnd; cum_ack = ntohl(cp->cumulative_tsn_ack); /* Arrange so a_rwnd does NOT change */ a_rwnd = stcb->asoc.peers_rwnd + stcb->asoc.total_flight; /* Now call the express sack handling */ sctp_express_handle_sack(stcb, cum_ack, a_rwnd, abort_flag, 0); } static void sctp_kick_prsctp_reorder_queue(struct sctp_tcb *stcb, struct sctp_stream_in *strmin) { struct sctp_queued_to_read *control, *ncontrol; struct sctp_association *asoc; uint32_t mid; int need_reasm_check = 0; asoc = &stcb->asoc; mid = strmin->last_mid_delivered; /* * First deliver anything prior to and including the stream no that * came in. */ TAILQ_FOREACH_SAFE(control, &strmin->inqueue, next_instrm, ncontrol) { if (SCTP_MID_GE(asoc->idata_supported, mid, control->mid)) { /* this is deliverable now */ if (((control->sinfo_flags >> 8) & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { if (control->on_strm_q) { if (control->on_strm_q == SCTP_ON_ORDERED) { TAILQ_REMOVE(&strmin->inqueue, control, next_instrm); } else if (control->on_strm_q == SCTP_ON_UNORDERED) { TAILQ_REMOVE(&strmin->uno_inqueue, control, next_instrm); #ifdef INVARIANTS } else { panic("strmin: %p ctl: %p unknown %d", strmin, control, control->on_strm_q); #endif } control->on_strm_q = 0; } /* subtract pending on streams */ if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); /* deliver it to at least the delivery-q */ if (stcb->sctp_socket) { sctp_mark_non_revokable(asoc, control->sinfo_tsn); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_HELD, SCTP_SO_NOT_LOCKED); } } else { /* Its a fragmented message */ if (control->first_frag_seen) { /* * Make it so this is next to * deliver, we restore later */ strmin->last_mid_delivered = control->mid - 1; need_reasm_check = 1; break; } } } else { /* no more delivery now. */ break; } } if (need_reasm_check) { int ret; ret = sctp_deliver_reasm_check(stcb, &stcb->asoc, strmin, SCTP_READ_LOCK_HELD); if (SCTP_MID_GT(asoc->idata_supported, mid, strmin->last_mid_delivered)) { /* Restore the next to deliver unless we are ahead */ strmin->last_mid_delivered = mid; } if (ret == 0) { /* Left the front Partial one on */ return; } need_reasm_check = 0; } /* * now we must deliver things in queue the normal way if any are * now ready. */ mid = strmin->last_mid_delivered + 1; TAILQ_FOREACH_SAFE(control, &strmin->inqueue, next_instrm, ncontrol) { if (SCTP_MID_EQ(asoc->idata_supported, mid, control->mid)) { if (((control->sinfo_flags >> 8) & SCTP_DATA_NOT_FRAG) == SCTP_DATA_NOT_FRAG) { /* this is deliverable now */ if (control->on_strm_q) { if (control->on_strm_q == SCTP_ON_ORDERED) { TAILQ_REMOVE(&strmin->inqueue, control, next_instrm); } else if (control->on_strm_q == SCTP_ON_UNORDERED) { TAILQ_REMOVE(&strmin->uno_inqueue, control, next_instrm); #ifdef INVARIANTS } else { panic("strmin: %p ctl: %p unknown %d", strmin, control, control->on_strm_q); #endif } control->on_strm_q = 0; } /* subtract pending on streams */ if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); /* deliver it to at least the delivery-q */ strmin->last_mid_delivered = control->mid; if (stcb->sctp_socket) { sctp_mark_non_revokable(asoc, control->sinfo_tsn); sctp_add_to_readq(stcb->sctp_ep, stcb, control, &stcb->sctp_socket->so_rcv, 1, SCTP_READ_LOCK_HELD, SCTP_SO_NOT_LOCKED); } mid = strmin->last_mid_delivered + 1; } else { /* Its a fragmented message */ if (control->first_frag_seen) { /* * Make it so this is next to * deliver */ strmin->last_mid_delivered = control->mid - 1; need_reasm_check = 1; break; } } } else { break; } } if (need_reasm_check) { (void)sctp_deliver_reasm_check(stcb, &stcb->asoc, strmin, SCTP_READ_LOCK_HELD); } } static void sctp_flush_reassm_for_str_seq(struct sctp_tcb *stcb, struct sctp_association *asoc, uint16_t stream, uint32_t mid, int ordered, uint32_t cumtsn) { struct sctp_queued_to_read *control; struct sctp_stream_in *strm; struct sctp_tmit_chunk *chk, *nchk; int cnt_removed = 0; /* * For now large messages held on the stream reasm that are complete * will be tossed too. We could in theory do more work to spin * through and stop after dumping one msg aka seeing the start of a * new msg at the head, and call the delivery function... to see if * it can be delivered... But for now we just dump everything on the * queue. */ strm = &asoc->strmin[stream]; control = sctp_find_reasm_entry(strm, mid, ordered, asoc->idata_supported); if (control == NULL) { /* Not found */ return; } if (!asoc->idata_supported && !ordered && SCTP_TSN_GT(control->fsn_included, cumtsn)) { return; } TAILQ_FOREACH_SAFE(chk, &control->reasm, sctp_next, nchk) { /* Purge hanging chunks */ if (!asoc->idata_supported && (ordered == 0)) { if (SCTP_TSN_GT(chk->rec.data.tsn, cumtsn)) { break; } } cnt_removed++; TAILQ_REMOVE(&control->reasm, chk, sctp_next); if (asoc->size_on_reasm_queue >= chk->send_size) { asoc->size_on_reasm_queue -= chk->send_size; } else { #ifdef INVARIANTS panic("size_on_reasm_queue = %u smaller than chunk length %u", asoc->size_on_reasm_queue, chk->send_size); #else asoc->size_on_reasm_queue = 0; #endif } sctp_ucount_decr(asoc->cnt_on_reasm_queue); if (chk->data) { sctp_m_freem(chk->data); chk->data = NULL; } sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED); } if (!TAILQ_EMPTY(&control->reasm)) { /* This has to be old data, unordered */ if (control->data) { sctp_m_freem(control->data); control->data = NULL; } sctp_reset_a_control(control, stcb->sctp_ep, cumtsn); chk = TAILQ_FIRST(&control->reasm); if (chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG) { TAILQ_REMOVE(&control->reasm, chk, sctp_next); sctp_add_chk_to_control(control, strm, stcb, asoc, chk, SCTP_READ_LOCK_HELD); } sctp_deliver_reasm_check(stcb, asoc, strm, SCTP_READ_LOCK_HELD); return; } if (control->on_strm_q == SCTP_ON_ORDERED) { TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); control->on_strm_q = 0; } else if (control->on_strm_q == SCTP_ON_UNORDERED) { TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); control->on_strm_q = 0; #ifdef INVARIANTS } else if (control->on_strm_q) { panic("strm: %p ctl: %p unknown %d", strm, control, control->on_strm_q); #endif } control->on_strm_q = 0; if (control->on_read_q == 0) { sctp_free_remote_addr(control->whoFrom); if (control->data) { sctp_m_freem(control->data); control->data = NULL; } sctp_free_a_readq(stcb, control); } } void sctp_handle_forward_tsn(struct sctp_tcb *stcb, struct sctp_forward_tsn_chunk *fwd, int *abort_flag, struct mbuf *m, int offset) { /* The pr-sctp fwd tsn */ /* * here we will perform all the data receiver side steps for * processing FwdTSN, as required in by pr-sctp draft: * * Assume we get FwdTSN(x): * * 1) update local cumTSN to x 2) try to further advance cumTSN to x * + others we have 3) examine and update re-ordering queue on * pr-in-streams 4) clean up re-assembly queue 5) Send a sack to * report where we are. */ struct sctp_association *asoc; uint32_t new_cum_tsn, gap; unsigned int i, fwd_sz, m_size; uint32_t str_seq; struct sctp_stream_in *strm; struct sctp_queued_to_read *control, *sv; asoc = &stcb->asoc; if ((fwd_sz = ntohs(fwd->ch.chunk_length)) < sizeof(struct sctp_forward_tsn_chunk)) { SCTPDBG(SCTP_DEBUG_INDATA1, "Bad size too small/big fwd-tsn\n"); return; } m_size = (stcb->asoc.mapping_array_size << 3); /*************************************************************/ /* 1. Here we update local cumTSN and shift the bitmap array */ /*************************************************************/ new_cum_tsn = ntohl(fwd->new_cumulative_tsn); if (SCTP_TSN_GE(asoc->cumulative_tsn, new_cum_tsn)) { /* Already got there ... */ return; } /* * now we know the new TSN is more advanced, let's find the actual * gap */ SCTP_CALC_TSN_TO_GAP(gap, new_cum_tsn, asoc->mapping_array_base_tsn); asoc->cumulative_tsn = new_cum_tsn; if (gap >= m_size) { if ((long)gap > sctp_sbspace(&stcb->asoc, &stcb->sctp_socket->so_rcv)) { struct mbuf *op_err; char msg[SCTP_DIAG_INFO_LEN]; /* * out of range (of single byte chunks in the rwnd I * give out). This must be an attacker. */ *abort_flag = 1; snprintf(msg, sizeof(msg), "New cum ack %8.8x too high, highest TSN %8.8x", new_cum_tsn, asoc->highest_tsn_inside_map); op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg); stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_33; sctp_abort_an_association(stcb->sctp_ep, stcb, op_err, SCTP_SO_NOT_LOCKED); return; } SCTP_STAT_INCR(sctps_fwdtsn_map_over); memset(stcb->asoc.mapping_array, 0, stcb->asoc.mapping_array_size); asoc->mapping_array_base_tsn = new_cum_tsn + 1; asoc->highest_tsn_inside_map = new_cum_tsn; memset(stcb->asoc.nr_mapping_array, 0, stcb->asoc.mapping_array_size); asoc->highest_tsn_inside_nr_map = new_cum_tsn; if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MAP_LOGGING_ENABLE) { sctp_log_map(0, 3, asoc->highest_tsn_inside_map, SCTP_MAP_SLIDE_RESULT); } } else { SCTP_TCB_LOCK_ASSERT(stcb); for (i = 0; i <= gap; i++) { if (!SCTP_IS_TSN_PRESENT(asoc->mapping_array, i) && !SCTP_IS_TSN_PRESENT(asoc->nr_mapping_array, i)) { SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, i); if (SCTP_TSN_GT(asoc->mapping_array_base_tsn + i, asoc->highest_tsn_inside_nr_map)) { asoc->highest_tsn_inside_nr_map = asoc->mapping_array_base_tsn + i; } } } } /*************************************************************/ /* 2. Clear up re-assembly queue */ /*************************************************************/ /* This is now done as part of clearing up the stream/seq */ if (asoc->idata_supported == 0) { uint16_t sid; /* Flush all the un-ordered data based on cum-tsn */ SCTP_INP_READ_LOCK(stcb->sctp_ep); for (sid = 0; sid < asoc->streamincnt; sid++) { sctp_flush_reassm_for_str_seq(stcb, asoc, sid, 0, 0, new_cum_tsn); } SCTP_INP_READ_UNLOCK(stcb->sctp_ep); } /*******************************************************/ /* 3. Update the PR-stream re-ordering queues and fix */ /* delivery issues as needed. */ /*******************************************************/ fwd_sz -= sizeof(*fwd); if (m && fwd_sz) { /* New method. */ unsigned int num_str; uint32_t mid, cur_mid; uint16_t sid; uint16_t ordered, flags; struct sctp_strseq *stseq, strseqbuf; struct sctp_strseq_mid *stseq_m, strseqbuf_m; offset += sizeof(*fwd); SCTP_INP_READ_LOCK(stcb->sctp_ep); if (asoc->idata_supported) { num_str = fwd_sz / sizeof(struct sctp_strseq_mid); } else { num_str = fwd_sz / sizeof(struct sctp_strseq); } for (i = 0; i < num_str; i++) { if (asoc->idata_supported) { stseq_m = (struct sctp_strseq_mid *)sctp_m_getptr(m, offset, sizeof(struct sctp_strseq_mid), (uint8_t *)&strseqbuf_m); offset += sizeof(struct sctp_strseq_mid); if (stseq_m == NULL) { break; } sid = ntohs(stseq_m->sid); mid = ntohl(stseq_m->mid); flags = ntohs(stseq_m->flags); if (flags & PR_SCTP_UNORDERED_FLAG) { ordered = 0; } else { ordered = 1; } } else { stseq = (struct sctp_strseq *)sctp_m_getptr(m, offset, sizeof(struct sctp_strseq), (uint8_t *)&strseqbuf); offset += sizeof(struct sctp_strseq); if (stseq == NULL) { break; } sid = ntohs(stseq->sid); mid = (uint32_t)ntohs(stseq->ssn); ordered = 1; } /* Convert */ /* now process */ /* * Ok we now look for the stream/seq on the read * queue where its not all delivered. If we find it * we transmute the read entry into a PDI_ABORTED. */ if (sid >= asoc->streamincnt) { /* screwed up streams, stop! */ break; } if ((asoc->str_of_pdapi == sid) && (asoc->ssn_of_pdapi == mid)) { /* * If this is the one we were partially * delivering now then we no longer are. * Note this will change with the reassembly * re-write. */ asoc->fragmented_delivery_inprogress = 0; } strm = &asoc->strmin[sid]; for (cur_mid = strm->last_mid_delivered; SCTP_MID_GE(asoc->idata_supported, mid, cur_mid); cur_mid++) { sctp_flush_reassm_for_str_seq(stcb, asoc, sid, cur_mid, ordered, new_cum_tsn); } TAILQ_FOREACH(control, &stcb->sctp_ep->read_queue, next) { if ((control->sinfo_stream == sid) && (SCTP_MID_EQ(asoc->idata_supported, control->mid, mid))) { str_seq = (sid << 16) | (0x0000ffff & mid); control->pdapi_aborted = 1; sv = stcb->asoc.control_pdapi; control->end_added = 1; if (control->on_strm_q == SCTP_ON_ORDERED) { TAILQ_REMOVE(&strm->inqueue, control, next_instrm); if (asoc->size_on_all_streams >= control->length) { asoc->size_on_all_streams -= control->length; } else { #ifdef INVARIANTS panic("size_on_all_streams = %u smaller than control length %u", asoc->size_on_all_streams, control->length); #else asoc->size_on_all_streams = 0; #endif } sctp_ucount_decr(asoc->cnt_on_all_streams); } else if (control->on_strm_q == SCTP_ON_UNORDERED) { TAILQ_REMOVE(&strm->uno_inqueue, control, next_instrm); #ifdef INVARIANTS } else if (control->on_strm_q) { panic("strm: %p ctl: %p unknown %d", strm, control, control->on_strm_q); #endif } control->on_strm_q = 0; stcb->asoc.control_pdapi = control; sctp_ulp_notify(SCTP_NOTIFY_PARTIAL_DELVIERY_INDICATION, stcb, SCTP_PARTIAL_DELIVERY_ABORTED, (void *)&str_seq, SCTP_SO_NOT_LOCKED); stcb->asoc.control_pdapi = sv; break; } else if ((control->sinfo_stream == sid) && SCTP_MID_GT(asoc->idata_supported, control->mid, mid)) { /* We are past our victim SSN */ break; } } if (SCTP_MID_GT(asoc->idata_supported, mid, strm->last_mid_delivered)) { /* Update the sequence number */ strm->last_mid_delivered = mid; } /* now kick the stream the new way */ /* sa_ignore NO_NULL_CHK */ sctp_kick_prsctp_reorder_queue(stcb, strm); } SCTP_INP_READ_UNLOCK(stcb->sctp_ep); } /* * Now slide thing forward. */ sctp_slide_mapping_arrays(stcb); } Index: projects/runtime-coverage/sys/netinet/tcp_input.c =================================================================== --- projects/runtime-coverage/sys/netinet/tcp_input.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/tcp_input.c (revision 321585) @@ -1,3913 +1,3894 @@ /*- * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995 * The Regents of the University of California. All rights reserved. * Copyright (c) 2007-2008,2010 * Swinburne University of Technology, Melbourne, Australia. * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2010 The FreeBSD Foundation * Copyright (c) 2010-2011 Juniper Networks, Inc. * All rights reserved. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by Lawrence Stewart, * James Healy and David Hayes, made possible in part by a grant from the Cisco * University Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by David Hayes under sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by Robert N. M. Watson under * contract to Juniper Networks, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)tcp_input.c 8.12 (Berkeley) 5/24/95 */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_ipsec.h" #include "opt_tcpdebug.h" #include #include #ifdef TCP_HHOOK #include #endif #include #include #include /* for proc0 declaration */ #include #include #include #include #include #include #include #include #include /* before tcp_seq.h, for tcp_random18() */ #include #include #include #include #include #define TCPSTATES /* for logging */ #include #include #include #include #include #include /* required for icmp_var.h */ #include /* for ICMP_BANDLIM */ #include #include #include #include #include #include #include #include #ifdef TCP_RFC7413 #include #endif #include #include #include #include #include #include #include #include #ifdef TCPPCAP #include #endif #include #ifdef TCPDEBUG #include #endif /* TCPDEBUG */ #ifdef TCP_OFFLOAD #include #endif #include #include #include const int tcprexmtthresh = 3; int tcp_log_in_vain = 0; SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_in_vain, CTLFLAG_RW, &tcp_log_in_vain, 0, "Log all incoming TCP segments to closed ports"); VNET_DEFINE(int, blackhole) = 0; #define V_blackhole VNET(blackhole) SYSCTL_INT(_net_inet_tcp, OID_AUTO, blackhole, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(blackhole), 0, "Do not send RST on segments to closed ports"); VNET_DEFINE(int, tcp_delack_enabled) = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, delayed_ack, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_delack_enabled), 0, "Delay ACK to try and piggyback it onto a data packet"); VNET_DEFINE(int, drop_synfin) = 0; #define V_drop_synfin VNET(drop_synfin) SYSCTL_INT(_net_inet_tcp, OID_AUTO, drop_synfin, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(drop_synfin), 0, "Drop TCP packets with SYN+FIN set"); VNET_DEFINE(int, tcp_do_rfc6675_pipe) = 0; SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc6675_pipe, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_rfc6675_pipe), 0, "Use calculated pipe/in-flight bytes per RFC 6675"); VNET_DEFINE(int, tcp_do_rfc3042) = 1; #define V_tcp_do_rfc3042 VNET(tcp_do_rfc3042) SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3042, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_rfc3042), 0, "Enable RFC 3042 (Limited Transmit)"); VNET_DEFINE(int, tcp_do_rfc3390) = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_rfc3390), 0, "Enable RFC 3390 (Increasing TCP's Initial Congestion Window)"); VNET_DEFINE(int, tcp_initcwnd_segments) = 10; SYSCTL_INT(_net_inet_tcp, OID_AUTO, initcwnd_segments, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_initcwnd_segments), 0, "Slow-start flight size (initial congestion window) in number of segments"); VNET_DEFINE(int, tcp_do_rfc3465) = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_rfc3465), 0, "Enable RFC 3465 (Appropriate Byte Counting)"); VNET_DEFINE(int, tcp_abc_l_var) = 2; SYSCTL_INT(_net_inet_tcp, OID_AUTO, abc_l_var, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_abc_l_var), 2, "Cap the max cwnd increment during slow-start to this number of segments"); static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, ecn, CTLFLAG_RW, 0, "TCP ECN"); VNET_DEFINE(int, tcp_do_ecn) = 2; SYSCTL_INT(_net_inet_tcp_ecn, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_ecn), 0, "TCP ECN support"); VNET_DEFINE(int, tcp_ecn_maxretries) = 1; SYSCTL_INT(_net_inet_tcp_ecn, OID_AUTO, maxretries, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_ecn_maxretries), 0, "Max retries before giving up on ECN"); VNET_DEFINE(int, tcp_insecure_syn) = 0; #define V_tcp_insecure_syn VNET(tcp_insecure_syn) SYSCTL_INT(_net_inet_tcp, OID_AUTO, insecure_syn, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_insecure_syn), 0, "Follow RFC793 instead of RFC5961 criteria for accepting SYN packets"); VNET_DEFINE(int, tcp_insecure_rst) = 0; #define V_tcp_insecure_rst VNET(tcp_insecure_rst) SYSCTL_INT(_net_inet_tcp, OID_AUTO, insecure_rst, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_insecure_rst), 0, "Follow RFC793 instead of RFC5961 criteria for accepting RST packets"); VNET_DEFINE(int, tcp_recvspace) = 1024*64; #define V_tcp_recvspace VNET(tcp_recvspace) SYSCTL_INT(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_recvspace), 0, "Initial receive socket buffer size"); VNET_DEFINE(int, tcp_do_autorcvbuf) = 1; #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_auto, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_do_autorcvbuf), 0, "Enable automatic receive buffer sizing"); VNET_DEFINE(int, tcp_autorcvbuf_inc) = 16*1024; #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_inc, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_autorcvbuf_inc), 0, "Incrementor step size of automatic receive buffer"); VNET_DEFINE(int, tcp_autorcvbuf_max) = 2*1024*1024; #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbuf_max, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_autorcvbuf_max), 0, "Max size of automatic receive buffer"); VNET_DEFINE(struct inpcbhead, tcb); #define tcb6 tcb /* for KAME src sync over BSD*'s */ VNET_DEFINE(struct inpcbinfo, tcbinfo); /* * TCP statistics are stored in an array of counter(9)s, which size matches * size of struct tcpstat. TCP running connection count is a regular array. */ VNET_PCPUSTAT_DEFINE(struct tcpstat, tcpstat); SYSCTL_VNET_PCPUSTAT(_net_inet_tcp, TCPCTL_STATS, stats, struct tcpstat, tcpstat, "TCP statistics (struct tcpstat, netinet/tcp_var.h)"); VNET_DEFINE(counter_u64_t, tcps_states[TCP_NSTATES]); SYSCTL_COUNTER_U64_ARRAY(_net_inet_tcp, TCPCTL_STATES, states, CTLFLAG_RD | CTLFLAG_VNET, &VNET_NAME(tcps_states)[0], TCP_NSTATES, "TCP connection counts by TCP state"); static void tcp_vnet_init(const void *unused) { COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK); VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK); } VNET_SYSINIT(tcp_vnet_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, tcp_vnet_init, NULL); #ifdef VIMAGE static void tcp_vnet_uninit(const void *unused) { COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES); VNET_PCPUSTAT_FREE(tcpstat); } VNET_SYSUNINIT(tcp_vnet_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, tcp_vnet_uninit, NULL); #endif /* VIMAGE */ /* * Kernel module interface for updating tcpstat. The argument is an index * into tcpstat treated as an array. */ void kmod_tcpstat_inc(int statnum) { counter_u64_add(VNET(tcpstat)[statnum], 1); } #ifdef TCP_HHOOK /* * Wrapper for the TCP established input helper hook. */ void hhook_run_tcp_est_in(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to) { struct tcp_hhook_data hhook_data; if (V_tcp_hhh[HHOOK_TCP_EST_IN]->hhh_nhooks > 0) { hhook_data.tp = tp; hhook_data.th = th; hhook_data.to = to; hhook_run_hooks(V_tcp_hhh[HHOOK_TCP_EST_IN], &hhook_data, tp->osd); } } #endif /* * CC wrapper hook functions */ void cc_ack_received(struct tcpcb *tp, struct tcphdr *th, uint16_t nsegs, uint16_t type) { INP_WLOCK_ASSERT(tp->t_inpcb); tp->ccv->nsegs = nsegs; tp->ccv->bytes_this_ack = BYTES_THIS_ACK(tp, th); if (tp->snd_cwnd <= tp->snd_wnd) tp->ccv->flags |= CCF_CWND_LIMITED; else tp->ccv->flags &= ~CCF_CWND_LIMITED; if (type == CC_ACK) { if (tp->snd_cwnd > tp->snd_ssthresh) { tp->t_bytes_acked += min(tp->ccv->bytes_this_ack, nsegs * V_tcp_abc_l_var * tcp_maxseg(tp)); if (tp->t_bytes_acked >= tp->snd_cwnd) { tp->t_bytes_acked -= tp->snd_cwnd; tp->ccv->flags |= CCF_ABC_SENTAWND; } } else { tp->ccv->flags &= ~CCF_ABC_SENTAWND; tp->t_bytes_acked = 0; } } if (CC_ALGO(tp)->ack_received != NULL) { /* XXXLAS: Find a way to live without this */ tp->ccv->curack = th->th_ack; CC_ALGO(tp)->ack_received(tp->ccv, type); } } void cc_conn_init(struct tcpcb *tp) { struct hc_metrics_lite metrics; struct inpcb *inp = tp->t_inpcb; u_int maxseg; int rtt; INP_WLOCK_ASSERT(tp->t_inpcb); tcp_hc_get(&inp->inp_inc, &metrics); maxseg = tcp_maxseg(tp); if (tp->t_srtt == 0 && (rtt = metrics.rmx_rtt)) { tp->t_srtt = rtt; tp->t_rttbest = tp->t_srtt + TCP_RTT_SCALE; TCPSTAT_INC(tcps_usedrtt); if (metrics.rmx_rttvar) { tp->t_rttvar = metrics.rmx_rttvar; TCPSTAT_INC(tcps_usedrttvar); } else { /* default variation is +- 1 rtt */ tp->t_rttvar = tp->t_srtt * TCP_RTTVAR_SCALE / TCP_RTT_SCALE; } TCPT_RANGESET(tp->t_rxtcur, ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1, tp->t_rttmin, TCPTV_REXMTMAX); } if (metrics.rmx_ssthresh) { /* * There's some sort of gateway or interface * buffer limit on the path. Use this to set * the slow start threshold, but set the * threshold to no less than 2*mss. */ tp->snd_ssthresh = max(2 * maxseg, metrics.rmx_ssthresh); TCPSTAT_INC(tcps_usedssthresh); } /* * Set the initial slow-start flight size. * * RFC5681 Section 3.1 specifies the default conservative values. * RFC3390 specifies slightly more aggressive values. * RFC6928 increases it to ten segments. * Support for user specified value for initial flight size. * * If a SYN or SYN/ACK was lost and retransmitted, we have to * reduce the initial CWND to one segment as congestion is likely * requiring us to be cautious. */ if (tp->snd_cwnd == 1) tp->snd_cwnd = maxseg; /* SYN(-ACK) lost */ else if (V_tcp_initcwnd_segments) tp->snd_cwnd = min(V_tcp_initcwnd_segments * maxseg, max(2 * maxseg, V_tcp_initcwnd_segments * 1460)); else if (V_tcp_do_rfc3390) tp->snd_cwnd = min(4 * maxseg, max(2 * maxseg, 4380)); else { /* Per RFC5681 Section 3.1 */ if (maxseg > 2190) tp->snd_cwnd = 2 * maxseg; else if (maxseg > 1095) tp->snd_cwnd = 3 * maxseg; else tp->snd_cwnd = 4 * maxseg; } if (CC_ALGO(tp)->conn_init != NULL) CC_ALGO(tp)->conn_init(tp->ccv); } void inline cc_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type) { u_int maxseg; INP_WLOCK_ASSERT(tp->t_inpcb); switch(type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(tp->t_flags)) { tp->snd_recover = tp->snd_max; if (tp->t_flags & TF_ECN_PERMIT) tp->t_flags |= TF_ECN_SND_CWR; } break; case CC_ECN: if (!IN_CONGRECOVERY(tp->t_flags)) { TCPSTAT_INC(tcps_ecn_rcwnd); tp->snd_recover = tp->snd_max; if (tp->t_flags & TF_ECN_PERMIT) tp->t_flags |= TF_ECN_SND_CWR; } break; case CC_RTO: maxseg = tcp_maxseg(tp); tp->t_dupacks = 0; tp->t_bytes_acked = 0; EXIT_RECOVERY(tp->t_flags); - if (CC_ALGO(tp)->cong_signal == NULL) { - /* - * RFC5681 Section 3.1 - * ssthresh = max (FlightSize / 2, 2*SMSS) eq (4) - */ - tp->snd_ssthresh = - max((tp->snd_max - tp->snd_una) / 2 / maxseg, 2) - * maxseg; - tp->snd_cwnd = maxseg; - } + tp->snd_ssthresh = max(2, min(tp->snd_wnd, tp->snd_cwnd) / 2 / + maxseg) * maxseg; + tp->snd_cwnd = maxseg; break; case CC_RTO_ERR: TCPSTAT_INC(tcps_sndrexmitbad); /* RTO was unnecessary, so reset everything. */ tp->snd_cwnd = tp->snd_cwnd_prev; tp->snd_ssthresh = tp->snd_ssthresh_prev; tp->snd_recover = tp->snd_recover_prev; if (tp->t_flags & TF_WASFRECOVERY) ENTER_FASTRECOVERY(tp->t_flags); if (tp->t_flags & TF_WASCRECOVERY) ENTER_CONGRECOVERY(tp->t_flags); tp->snd_nxt = tp->snd_max; tp->t_flags &= ~TF_PREVVALID; tp->t_badrxtwin = 0; break; } if (CC_ALGO(tp)->cong_signal != NULL) { if (th != NULL) tp->ccv->curack = th->th_ack; CC_ALGO(tp)->cong_signal(tp->ccv, type); } } void inline cc_post_recovery(struct tcpcb *tp, struct tcphdr *th) { INP_WLOCK_ASSERT(tp->t_inpcb); /* XXXLAS: KASSERT that we're in recovery? */ if (CC_ALGO(tp)->post_recovery != NULL) { tp->ccv->curack = th->th_ack; CC_ALGO(tp)->post_recovery(tp->ccv); } /* XXXLAS: EXIT_RECOVERY ? */ tp->t_bytes_acked = 0; } /* * Indicate whether this ack should be delayed. We can delay the ack if * following conditions are met: * - There is no delayed ack timer in progress. * - Our last ack wasn't a 0-sized window. We never want to delay * the ack that opens up a 0-sized window. * - LRO wasn't used for this segment. We make sure by checking that the * segment size is not larger than the MSS. */ #define DELAY_ACK(tp, tlen) \ ((!tcp_timer_active(tp, TT_DELACK) && \ (tp->t_flags & TF_RXWIN0SENT) == 0) && \ (tlen <= tp->t_maxseg) && \ (V_tcp_delack_enabled || (tp->t_flags & TF_NEEDSYN))) static void inline cc_ecnpkt_handler(struct tcpcb *tp, struct tcphdr *th, uint8_t iptos) { INP_WLOCK_ASSERT(tp->t_inpcb); if (CC_ALGO(tp)->ecnpkt_handler != NULL) { switch (iptos & IPTOS_ECN_MASK) { case IPTOS_ECN_CE: tp->ccv->flags |= CCF_IPHDR_CE; break; case IPTOS_ECN_ECT0: tp->ccv->flags &= ~CCF_IPHDR_CE; break; case IPTOS_ECN_ECT1: tp->ccv->flags &= ~CCF_IPHDR_CE; break; } if (th->th_flags & TH_CWR) tp->ccv->flags |= CCF_TCPHDR_CWR; else tp->ccv->flags &= ~CCF_TCPHDR_CWR; if (tp->t_flags & TF_DELACK) tp->ccv->flags |= CCF_DELACK; else tp->ccv->flags &= ~CCF_DELACK; CC_ALGO(tp)->ecnpkt_handler(tp->ccv); if (tp->ccv->flags & CCF_ACKNOW) tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } } /* * TCP input handling is split into multiple parts: * tcp6_input is a thin wrapper around tcp_input for the extended * ip6_protox[] call format in ip6_input * tcp_input handles primary segment validation, inpcb lookup and * SYN processing on listen sockets * tcp_do_segment processes the ACK and text of the segment for * establishing, established and closing connections */ #ifdef INET6 int tcp6_input(struct mbuf **mp, int *offp, int proto) { struct mbuf *m = *mp; struct in6_ifaddr *ia6; struct ip6_hdr *ip6; IP6_EXTHDR_CHECK(m, *offp, sizeof(struct tcphdr), IPPROTO_DONE); /* * draft-itojun-ipv6-tcp-to-anycast * better place to put this in? */ ip6 = mtod(m, struct ip6_hdr *); ia6 = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */); if (ia6 && (ia6->ia6_flags & IN6_IFF_ANYCAST)) { struct ip6_hdr *ip6; ifa_free(&ia6->ia_ifa); ip6 = mtod(m, struct ip6_hdr *); icmp6_error(m, ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_ADDR, (caddr_t)&ip6->ip6_dst - (caddr_t)ip6); return (IPPROTO_DONE); } if (ia6) ifa_free(&ia6->ia_ifa); return (tcp_input(mp, offp, proto)); } #endif /* INET6 */ int tcp_input(struct mbuf **mp, int *offp, int proto) { struct mbuf *m = *mp; struct tcphdr *th = NULL; struct ip *ip = NULL; struct inpcb *inp = NULL; struct tcpcb *tp = NULL; struct socket *so = NULL; u_char *optp = NULL; int off0; int optlen = 0; #ifdef INET int len; #endif int tlen = 0, off; int drop_hdrlen; int thflags; int rstreason = 0; /* For badport_bandlim accounting purposes */ uint8_t iptos; struct m_tag *fwd_tag = NULL; #ifdef INET6 struct ip6_hdr *ip6 = NULL; int isipv6; #else const void *ip6 = NULL; #endif /* INET6 */ struct tcpopt to; /* options in this segment */ char *s = NULL; /* address and port logging */ int ti_locked; #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif #ifdef INET6 isipv6 = (mtod(m, struct ip *)->ip_v == 6) ? 1 : 0; #endif off0 = *offp; m = *mp; *mp = NULL; to.to_flags = 0; TCPSTAT_INC(tcps_rcvtotal); #ifdef INET6 if (isipv6) { /* IP6_EXTHDR_CHECK() is already done at tcp6_input(). */ if (m->m_len < (sizeof(*ip6) + sizeof(*th))) { m = m_pullup(m, sizeof(*ip6) + sizeof(*th)); if (m == NULL) { TCPSTAT_INC(tcps_rcvshort); return (IPPROTO_DONE); } } ip6 = mtod(m, struct ip6_hdr *); th = (struct tcphdr *)((caddr_t)ip6 + off0); tlen = sizeof(*ip6) + ntohs(ip6->ip6_plen) - off0; if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID_IPV6) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) th->th_sum = m->m_pkthdr.csum_data; else th->th_sum = in6_cksum_pseudo(ip6, tlen, IPPROTO_TCP, m->m_pkthdr.csum_data); th->th_sum ^= 0xffff; } else th->th_sum = in6_cksum(m, IPPROTO_TCP, off0, tlen); if (th->th_sum) { TCPSTAT_INC(tcps_rcvbadsum); goto drop; } /* * Be proactive about unspecified IPv6 address in source. * As we use all-zero to indicate unbounded/unconnected pcb, * unspecified IPv6 address can be used to confuse us. * * Note that packets with unspecified IPv6 destination is * already dropped in ip6_input. */ if (IN6_IS_ADDR_UNSPECIFIED(&ip6->ip6_src)) { /* XXX stat */ goto drop; } iptos = (ntohl(ip6->ip6_flow) >> 20) & 0xff; } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { /* * Get IP and TCP header together in first mbuf. * Note: IP leaves IP header in first mbuf. */ if (off0 > sizeof (struct ip)) { ip_stripoptions(m); off0 = sizeof(struct ip); } if (m->m_len < sizeof (struct tcpiphdr)) { if ((m = m_pullup(m, sizeof (struct tcpiphdr))) == NULL) { TCPSTAT_INC(tcps_rcvshort); return (IPPROTO_DONE); } } ip = mtod(m, struct ip *); th = (struct tcphdr *)((caddr_t)ip + off0); tlen = ntohs(ip->ip_len) - off0; iptos = ip->ip_tos; if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) th->th_sum = m->m_pkthdr.csum_data; else th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htonl(m->m_pkthdr.csum_data + tlen + IPPROTO_TCP)); th->th_sum ^= 0xffff; } else { struct ipovly *ipov = (struct ipovly *)ip; /* * Checksum extended TCP header and data. */ len = off0 + tlen; bzero(ipov->ih_x1, sizeof(ipov->ih_x1)); ipov->ih_len = htons(tlen); th->th_sum = in_cksum(m, len); /* Reset length for SDT probes. */ ip->ip_len = htons(len); /* Reset TOS bits */ ip->ip_tos = iptos; /* Re-initialization for later version check */ ip->ip_v = IPVERSION; } if (th->th_sum) { TCPSTAT_INC(tcps_rcvbadsum); goto drop; } } #endif /* INET */ /* * Check that TCP offset makes sense, * pull out TCP options and adjust length. XXX */ off = th->th_off << 2; if (off < sizeof (struct tcphdr) || off > tlen) { TCPSTAT_INC(tcps_rcvbadoff); goto drop; } tlen -= off; /* tlen is used instead of ti->ti_len */ if (off > sizeof (struct tcphdr)) { #ifdef INET6 if (isipv6) { IP6_EXTHDR_CHECK(m, off0, off, IPPROTO_DONE); ip6 = mtod(m, struct ip6_hdr *); th = (struct tcphdr *)((caddr_t)ip6 + off0); } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { if (m->m_len < sizeof(struct ip) + off) { if ((m = m_pullup(m, sizeof (struct ip) + off)) == NULL) { TCPSTAT_INC(tcps_rcvshort); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); th = (struct tcphdr *)((caddr_t)ip + off0); } } #endif optlen = off - sizeof (struct tcphdr); optp = (u_char *)(th + 1); } thflags = th->th_flags; /* * Convert TCP protocol specific fields to host format. */ tcp_fields_to_host(th); /* * Delay dropping TCP, IP headers, IPv6 ext headers, and TCP options. */ drop_hdrlen = off0 + off; /* * Locate pcb for segment; if we're likely to add or remove a * connection then first acquire pcbinfo lock. There are three cases * where we might discover later we need a write lock despite the * flags: ACKs moving a connection out of the syncache, ACKs for a * connection in TIMEWAIT and SYNs not targeting a listening socket. */ if ((thflags & (TH_FIN | TH_RST)) != 0) { INP_INFO_RLOCK(&V_tcbinfo); ti_locked = TI_RLOCKED; } else ti_locked = TI_UNLOCKED; /* * Grab info from PACKET_TAG_IPFORWARD tag prepended to the chain. */ if ( #ifdef INET6 (isipv6 && (m->m_flags & M_IP6_NEXTHOP)) #ifdef INET || (!isipv6 && (m->m_flags & M_IP_NEXTHOP)) #endif #endif #if defined(INET) && !defined(INET6) (m->m_flags & M_IP_NEXTHOP) #endif ) fwd_tag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL); findpcb: #ifdef INVARIANTS if (ti_locked == TI_RLOCKED) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif #ifdef INET6 if (isipv6 && fwd_tag != NULL) { struct sockaddr_in6 *next_hop6; next_hop6 = (struct sockaddr_in6 *)(fwd_tag + 1); /* * Transparently forwarded. Pretend to be the destination. * Already got one like this? */ inp = in6_pcblookup_mbuf(&V_tcbinfo, &ip6->ip6_src, th->th_sport, &ip6->ip6_dst, th->th_dport, INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif, m); if (!inp) { /* * It's new. Try to find the ambushing socket. * Because we've rewritten the destination address, * any hardware-generated hash is ignored. */ inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_src, th->th_sport, &next_hop6->sin6_addr, next_hop6->sin6_port ? ntohs(next_hop6->sin6_port) : th->th_dport, INPLOOKUP_WILDCARD | INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif); } } else if (isipv6) { inp = in6_pcblookup_mbuf(&V_tcbinfo, &ip6->ip6_src, th->th_sport, &ip6->ip6_dst, th->th_dport, INPLOOKUP_WILDCARD | INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif, m); } #endif /* INET6 */ #if defined(INET6) && defined(INET) else #endif #ifdef INET if (fwd_tag != NULL) { struct sockaddr_in *next_hop; next_hop = (struct sockaddr_in *)(fwd_tag+1); /* * Transparently forwarded. Pretend to be the destination. * already got one like this? */ inp = in_pcblookup_mbuf(&V_tcbinfo, ip->ip_src, th->th_sport, ip->ip_dst, th->th_dport, INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif, m); if (!inp) { /* * It's new. Try to find the ambushing socket. * Because we've rewritten the destination address, * any hardware-generated hash is ignored. */ inp = in_pcblookup(&V_tcbinfo, ip->ip_src, th->th_sport, next_hop->sin_addr, next_hop->sin_port ? ntohs(next_hop->sin_port) : th->th_dport, INPLOOKUP_WILDCARD | INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif); } } else inp = in_pcblookup_mbuf(&V_tcbinfo, ip->ip_src, th->th_sport, ip->ip_dst, th->th_dport, INPLOOKUP_WILDCARD | INPLOOKUP_WLOCKPCB, m->m_pkthdr.rcvif, m); #endif /* INET */ /* * If the INPCB does not exist then all data in the incoming * segment is discarded and an appropriate RST is sent back. * XXX MRT Send RST using which routing table? */ if (inp == NULL) { /* * Log communication attempts to ports that are not * in use. */ if ((tcp_log_in_vain == 1 && (thflags & TH_SYN)) || tcp_log_in_vain == 2) { if ((s = tcp_log_vain(NULL, th, (void *)ip, ip6))) log(LOG_INFO, "%s; %s: Connection attempt " "to closed port\n", s, __func__); } /* * When blackholing do not respond with a RST but * completely ignore the segment and drop it. */ if ((V_blackhole == 1 && (thflags & TH_SYN)) || V_blackhole == 2) goto dropunlock; rstreason = BANDLIM_RST_CLOSEDPORT; goto dropwithreset; } INP_WLOCK_ASSERT(inp); /* * While waiting for inp lock during the lookup, another thread * can have dropped the inpcb, in which case we need to loop back * and try to find a new inpcb to deliver to. */ if (inp->inp_flags & INP_DROPPED) { INP_WUNLOCK(inp); inp = NULL; goto findpcb; } if ((inp->inp_flowtype == M_HASHTYPE_NONE) && (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) && ((inp->inp_socket == NULL) || (inp->inp_socket->so_options & SO_ACCEPTCONN) == 0)) { inp->inp_flowid = m->m_pkthdr.flowid; inp->inp_flowtype = M_HASHTYPE_GET(m); } #if defined(IPSEC) || defined(IPSEC_SUPPORT) #ifdef INET6 if (isipv6 && IPSEC_ENABLED(ipv6) && IPSEC_CHECK_POLICY(ipv6, m, inp) != 0) { goto dropunlock; } #ifdef INET else #endif #endif /* INET6 */ #ifdef INET if (IPSEC_ENABLED(ipv4) && IPSEC_CHECK_POLICY(ipv4, m, inp) != 0) { goto dropunlock; } #endif /* INET */ #endif /* IPSEC */ /* * Check the minimum TTL for socket. */ if (inp->inp_ip_minttl != 0) { #ifdef INET6 if (isipv6) { if (inp->inp_ip_minttl > ip6->ip6_hlim) goto dropunlock; } else #endif if (inp->inp_ip_minttl > ip->ip_ttl) goto dropunlock; } /* * A previous connection in TIMEWAIT state is supposed to catch stray * or duplicate segments arriving late. If this segment was a * legitimate new connection attempt, the old INPCB gets removed and * we can try again to find a listening socket. * * At this point, due to earlier optimism, we may hold only an inpcb * lock, and not the inpcbinfo write lock. If so, we need to try to * acquire it, or if that fails, acquire a reference on the inpcb, * drop all locks, acquire a global write lock, and then re-acquire * the inpcb lock. We may at that point discover that another thread * has tried to free the inpcb, in which case we need to loop back * and try to find a new inpcb to deliver to. * * XXXRW: It may be time to rethink timewait locking. */ relocked: if (inp->inp_flags & INP_TIMEWAIT) { if (ti_locked == TI_UNLOCKED) { if (INP_INFO_TRY_RLOCK(&V_tcbinfo) == 0) { in_pcbref(inp); INP_WUNLOCK(inp); INP_INFO_RLOCK(&V_tcbinfo); ti_locked = TI_RLOCKED; INP_WLOCK(inp); if (in_pcbrele_wlocked(inp)) { inp = NULL; goto findpcb; } else if (inp->inp_flags & INP_DROPPED) { INP_WUNLOCK(inp); inp = NULL; goto findpcb; } } else ti_locked = TI_RLOCKED; } INP_INFO_RLOCK_ASSERT(&V_tcbinfo); if (thflags & TH_SYN) tcp_dooptions(&to, optp, optlen, TO_SYN); /* * NB: tcp_twcheck unlocks the INP and frees the mbuf. */ if (tcp_twcheck(inp, &to, th, m, tlen)) goto findpcb; INP_INFO_RUNLOCK(&V_tcbinfo); return (IPPROTO_DONE); } /* * The TCPCB may no longer exist if the connection is winding * down or it is in the CLOSED state. Either way we drop the * segment and send an appropriate response. */ tp = intotcpcb(inp); if (tp == NULL || tp->t_state == TCPS_CLOSED) { rstreason = BANDLIM_RST_CLOSEDPORT; goto dropwithreset; } #ifdef TCP_OFFLOAD if (tp->t_flags & TF_TOE) { tcp_offload_input(tp, m); m = NULL; /* consumed by the TOE driver */ goto dropunlock; } #endif /* * We've identified a valid inpcb, but it could be that we need an * inpcbinfo write lock but don't hold it. In this case, attempt to * acquire using the same strategy as the TIMEWAIT case above. If we * relock, we have to jump back to 'relocked' as the connection might * now be in TIMEWAIT. */ #ifdef INVARIANTS if ((thflags & (TH_FIN | TH_RST)) != 0) INP_INFO_RLOCK_ASSERT(&V_tcbinfo); #endif if (!((tp->t_state == TCPS_ESTABLISHED && (thflags & TH_SYN) == 0) || (tp->t_state == TCPS_LISTEN && (thflags & TH_SYN) && !IS_FASTOPEN(tp->t_flags)))) { if (ti_locked == TI_UNLOCKED) { if (INP_INFO_TRY_RLOCK(&V_tcbinfo) == 0) { in_pcbref(inp); INP_WUNLOCK(inp); INP_INFO_RLOCK(&V_tcbinfo); ti_locked = TI_RLOCKED; INP_WLOCK(inp); if (in_pcbrele_wlocked(inp)) { inp = NULL; goto findpcb; } else if (inp->inp_flags & INP_DROPPED) { INP_WUNLOCK(inp); inp = NULL; goto findpcb; } goto relocked; } else ti_locked = TI_RLOCKED; } INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } #ifdef MAC INP_WLOCK_ASSERT(inp); if (mac_inpcb_check_deliver(inp, m)) goto dropunlock; #endif so = inp->inp_socket; KASSERT(so != NULL, ("%s: so == NULL", __func__)); #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) { ostate = tp->t_state; #ifdef INET6 if (isipv6) { bcopy((char *)ip6, (char *)tcp_saveipgen, sizeof(*ip6)); } else #endif bcopy((char *)ip, (char *)tcp_saveipgen, sizeof(*ip)); tcp_savetcp = *th; } #endif /* TCPDEBUG */ /* * When the socket is accepting connections (the INPCB is in LISTEN * state) we look into the SYN cache if this is a new connection * attempt or the completion of a previous one. */ KASSERT(tp->t_state == TCPS_LISTEN || !(so->so_options & SO_ACCEPTCONN), ("%s: so accepting but tp %p not listening", __func__, tp)); if (tp->t_state == TCPS_LISTEN && (so->so_options & SO_ACCEPTCONN)) { struct in_conninfo inc; bzero(&inc, sizeof(inc)); #ifdef INET6 if (isipv6) { inc.inc_flags |= INC_ISIPV6; inc.inc6_faddr = ip6->ip6_src; inc.inc6_laddr = ip6->ip6_dst; } else #endif { inc.inc_faddr = ip->ip_src; inc.inc_laddr = ip->ip_dst; } inc.inc_fport = th->th_sport; inc.inc_lport = th->th_dport; inc.inc_fibnum = so->so_fibnum; /* * Check for an existing connection attempt in syncache if * the flag is only ACK. A successful lookup creates a new * socket appended to the listen queue in SYN_RECEIVED state. */ if ((thflags & (TH_RST|TH_ACK|TH_SYN)) == TH_ACK) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* * Parse the TCP options here because * syncookies need access to the reflected * timestamp. */ tcp_dooptions(&to, optp, optlen, 0); /* * NB: syncache_expand() doesn't unlock * inp and tcpinfo locks. */ rstreason = syncache_expand(&inc, &to, th, &so, m); if (rstreason < 0) { /* * A failing TCP MD5 signature comparison * must result in the segment being dropped * and must not produce any response back * to the sender. */ goto dropunlock; } else if (rstreason == 0) { /* * No syncache entry or ACK was not * for our SYN/ACK. Send a RST. * NB: syncache did its own logging * of the failure cause. */ rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } #ifdef TCP_RFC7413 tfo_socket_result: #endif if (so == NULL) { /* * We completed the 3-way handshake * but could not allocate a socket * either due to memory shortage, * listen queue length limits or * global socket limits. Send RST * or wait and have the remote end * retransmit the ACK for another * try. */ if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Socket allocation failed due to " "limits or memory shortage, %s\n", s, __func__, V_tcp_sc_rst_sock_fail ? "sending RST" : "try again"); if (V_tcp_sc_rst_sock_fail) { rstreason = BANDLIM_UNLIMITED; goto dropwithreset; } else goto dropunlock; } /* * Socket is created in state SYN_RECEIVED. * Unlock the listen socket, lock the newly * created socket and update the tp variable. */ INP_WUNLOCK(inp); /* listen socket */ inp = sotoinpcb(so); /* * New connection inpcb is already locked by * syncache_expand(). */ INP_WLOCK_ASSERT(inp); tp = intotcpcb(inp); KASSERT(tp->t_state == TCPS_SYN_RECEIVED, ("%s: ", __func__)); /* * Process the segment and the data it * contains. tcp_do_segment() consumes * the mbuf chain and unlocks the inpcb. */ tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos, ti_locked); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); return (IPPROTO_DONE); } /* * Segment flag validation for new connection attempts: * * Our (SYN|ACK) response was rejected. * Check with syncache and remove entry to prevent * retransmits. * * NB: syncache_chkrst does its own logging of failure * causes. */ if (thflags & TH_RST) { syncache_chkrst(&inc, th); goto dropunlock; } /* * We can't do anything without SYN. */ if ((thflags & TH_SYN) == 0) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "SYN is missing, segment ignored\n", s, __func__); TCPSTAT_INC(tcps_badsyn); goto dropunlock; } /* * (SYN|ACK) is bogus on a listen socket. */ if (thflags & TH_ACK) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "SYN|ACK invalid, segment rejected\n", s, __func__); syncache_badack(&inc); /* XXX: Not needed! */ TCPSTAT_INC(tcps_badsyn); rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } /* * If the drop_synfin option is enabled, drop all * segments with both the SYN and FIN bits set. * This prevents e.g. nmap from identifying the * TCP/IP stack. * XXX: Poor reasoning. nmap has other methods * and is constantly refining its stack detection * strategies. * XXX: This is a violation of the TCP specification * and was used by RFC1644. */ if ((thflags & TH_FIN) && V_drop_synfin) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "SYN|FIN segment ignored (based on " "sysctl setting)\n", s, __func__); TCPSTAT_INC(tcps_badsyn); goto dropunlock; } /* * Segment's flags are (SYN) or (SYN|FIN). * * TH_PUSH, TH_URG, TH_ECE, TH_CWR are ignored * as they do not affect the state of the TCP FSM. * The data pointed to by TH_URG and th_urp is ignored. */ KASSERT((thflags & (TH_RST|TH_ACK)) == 0, ("%s: Listen socket: TH_RST or TH_ACK set", __func__)); KASSERT(thflags & (TH_SYN), ("%s: Listen socket: TH_SYN not set", __func__)); #ifdef INET6 /* * If deprecated address is forbidden, * we do not accept SYN to deprecated interface * address to prevent any new inbound connection from * getting established. * When we do not accept SYN, we send a TCP RST, * with deprecated source address (instead of dropping * it). We compromise it as it is much better for peer * to send a RST, and RST will be the final packet * for the exchange. * * If we do not forbid deprecated addresses, we accept * the SYN packet. RFC2462 does not suggest dropping * SYN in this case. * If we decipher RFC2462 5.5.4, it says like this: * 1. use of deprecated addr with existing * communication is okay - "SHOULD continue to be * used" * 2. use of it with new communication: * (2a) "SHOULD NOT be used if alternate address * with sufficient scope is available" * (2b) nothing mentioned otherwise. * Here we fall into (2b) case as we have no choice in * our source address selection - we must obey the peer. * * The wording in RFC2462 is confusing, and there are * multiple description text for deprecated address * handling - worse, they are not exactly the same. * I believe 5.5.4 is the best one, so we follow 5.5.4. */ if (isipv6 && !V_ip6_use_deprecated) { struct in6_ifaddr *ia6; ia6 = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */); if (ia6 != NULL && (ia6->ia6_flags & IN6_IFF_DEPRECATED)) { ifa_free(&ia6->ia_ifa); if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt to deprecated " "IPv6 address rejected\n", s, __func__); rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } if (ia6) ifa_free(&ia6->ia_ifa); } #endif /* INET6 */ /* * Basic sanity checks on incoming SYN requests: * Don't respond if the destination is a link layer * broadcast according to RFC1122 4.2.3.10, p. 104. * If it is from this socket it must be forged. * Don't respond if the source or destination is a * global or subnet broad- or multicast address. * Note that it is quite possible to receive unicast * link-layer packets with a broadcast IP address. Use * in_broadcast() to find them. */ if (m->m_flags & (M_BCAST|M_MCAST)) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt from broad- or multicast " "link layer address ignored\n", s, __func__); goto dropunlock; } #ifdef INET6 if (isipv6) { if (th->th_dport == th->th_sport && IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &ip6->ip6_src)) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt to/from self " "ignored\n", s, __func__); goto dropunlock; } if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) || IN6_IS_ADDR_MULTICAST(&ip6->ip6_src)) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt from/to multicast " "address ignored\n", s, __func__); goto dropunlock; } } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { if (th->th_dport == th->th_sport && ip->ip_dst.s_addr == ip->ip_src.s_addr) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt from/to self " "ignored\n", s, __func__); goto dropunlock; } if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) || IN_MULTICAST(ntohl(ip->ip_src.s_addr)) || ip->ip_src.s_addr == htonl(INADDR_BROADCAST) || in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) { if ((s = tcp_log_addrs(&inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Listen socket: " "Connection attempt from/to broad- " "or multicast address ignored\n", s, __func__); goto dropunlock; } } #endif /* * SYN appears to be valid. Create compressed TCP state * for syncache. */ #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); tcp_dooptions(&to, optp, optlen, TO_SYN); #ifdef TCP_RFC7413 if (syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL)) goto tfo_socket_result; #else syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL); #endif /* * Entry added to syncache and mbuf consumed. * Only the listen socket is unlocked by syncache_add(). */ if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; } INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); return (IPPROTO_DONE); } else if (tp->t_state == TCPS_LISTEN) { /* * When a listen socket is torn down the SO_ACCEPTCONN * flag is removed first while connections are drained * from the accept queue in a unlock/lock cycle of the * ACCEPT_LOCK, opening a race condition allowing a SYN * attempt go through unhandled. */ goto dropunlock; } #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) if (tp->t_flags & TF_SIGNATURE) { tcp_dooptions(&to, optp, optlen, thflags); if ((to.to_flags & TOF_SIGNATURE) == 0) { TCPSTAT_INC(tcps_sig_err_nosigopt); goto dropunlock; } if (!TCPMD5_ENABLED() || TCPMD5_INPUT(m, th, to.to_signature) != 0) goto dropunlock; } #endif TCP_PROBE5(receive, NULL, tp, m, tp, th); /* * Segment belongs to a connection in SYN_SENT, ESTABLISHED or later * state. tcp_do_segment() always consumes the mbuf chain, unlocks * the inpcb, and unlocks pcbinfo. */ tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos, ti_locked); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); return (IPPROTO_DONE); dropwithreset: TCP_PROBE5(receive, NULL, tp, m, tp, th); if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS else { KASSERT(ti_locked == TI_UNLOCKED, ("%s: dropwithreset " "ti_locked: %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif if (inp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(inp); } else tcp_dropwithreset(m, th, NULL, tlen, rstreason); m = NULL; /* mbuf chain got consumed. */ goto drop; dropunlock: if (m != NULL) TCP_PROBE5(receive, NULL, tp, m, tp, th); if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS else { KASSERT(ti_locked == TI_UNLOCKED, ("%s: dropunlock " "ti_locked: %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif if (inp != NULL) INP_WUNLOCK(inp); drop: INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); if (s != NULL) free(s, M_TCPLOG); if (m != NULL) m_freem(m); return (IPPROTO_DONE); } /* * Automatic sizing of receive socket buffer. Often the send * buffer size is not optimally adjusted to the actual network * conditions at hand (delay bandwidth product). Setting the * buffer size too small limits throughput on links with high * bandwidth and high delay (eg. trans-continental/oceanic links). * * On the receive side the socket buffer memory is only rarely * used to any significant extent. This allows us to be much * more aggressive in scaling the receive socket buffer. For * the case that the buffer space is actually used to a large * extent and we run out of kernel memory we can simply drop * the new segments; TCP on the sender will just retransmit it * later. Setting the buffer size too big may only consume too * much kernel memory if the application doesn't read() from * the socket or packet loss or reordering makes use of the * reassembly queue. * * The criteria to step up the receive buffer one notch are: * 1. Application has not set receive buffer size with * SO_RCVBUF. Setting SO_RCVBUF clears SB_AUTOSIZE. * 2. the number of bytes received during the time it takes * one timestamp to be reflected back to us (the RTT); * 3. received bytes per RTT is within seven eighth of the * current socket buffer size; * 4. receive buffer size has not hit maximal automatic size; * * This algorithm does one step per RTT at most and only if * we receive a bulk stream w/o packet losses or reorderings. * Shrinking the buffer during idle times is not necessary as * it doesn't consume any memory when idle. * * TODO: Only step up if the application is actually serving * the buffer to better manage the socket buffer resources. */ int tcp_autorcvbuf(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int tlen) { int newsize = 0; if (V_tcp_do_autorcvbuf && (so->so_rcv.sb_flags & SB_AUTOSIZE) && tp->t_srtt != 0 && tp->rfbuf_ts != 0 && TCP_TS_TO_TICKS(tcp_ts_getticks() - tp->rfbuf_ts) > (tp->t_srtt >> TCP_RTT_SHIFT)) { if (tp->rfbuf_cnt > (so->so_rcv.sb_hiwat / 8 * 7) && so->so_rcv.sb_hiwat < V_tcp_autorcvbuf_max) { newsize = min(so->so_rcv.sb_hiwat + V_tcp_autorcvbuf_inc, V_tcp_autorcvbuf_max); } TCP_PROBE6(receive__autoresize, NULL, tp, m, tp, th, newsize); /* Start over with next RTT. */ tp->rfbuf_ts = 0; tp->rfbuf_cnt = 0; } else { tp->rfbuf_cnt += tlen; /* add up */ } return (newsize); } void tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, int ti_locked) { int thflags, acked, ourfinisacked, needoutput = 0, sack_changed; int rstreason, todrop, win; uint32_t tiwin; uint16_t nsegs; char *s; struct in_conninfo *inc; struct mbuf *mfree; struct tcpopt to; #ifdef TCP_RFC7413 int tfo_syn; #endif #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif thflags = th->th_flags; inc = &tp->t_inpcb->inp_inc; tp->sackhint.last_sack_ack = 0; sack_changed = 0; nsegs = max(1, m->m_pkthdr.lro_nsegs); /* * If this is either a state-changing packet or current state isn't * established, we require a write lock on tcbinfo. Otherwise, we * allow the tcbinfo to be in either alocked or unlocked, as the * caller may have unnecessarily acquired a write lock due to a race. */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { #ifdef INVARIANTS if (ti_locked == TI_RLOCKED) INP_INFO_RLOCK_ASSERT(&V_tcbinfo); else { KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " "ti_locked: %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", __func__)); KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", __func__)); #ifdef TCPPCAP /* Save segment, if requested. */ tcp_pcap_add(th, m, &(tp->t_inpkts)); #endif if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: " "SYN|FIN segment ignored (based on " "sysctl setting)\n", s, __func__); free(s, M_TCPLOG); } goto drop; } /* * If a segment with the ACK-bit set arrives in the SYN-SENT state * check SEQ.ACK first. */ if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { rstreason = BANDLIM_UNLIMITED; goto dropwithreset; } /* * Segment received on connection. * Reset idle time and keep-alive timer. * XXX: This should be done after segment * validation to ignore broken/spoofed segs. */ tp->t_rcvtime = ticks; /* * Scale up the window into a 32-bit value. * For the SYN_SENT state the scale is zero. */ tiwin = th->th_win << tp->snd_scale; /* * TCP ECN processing. */ if (tp->t_flags & TF_ECN_PERMIT) { if (thflags & TH_CWR) tp->t_flags &= ~TF_ECN_SND_ECE; switch (iptos & IPTOS_ECN_MASK) { case IPTOS_ECN_CE: tp->t_flags |= TF_ECN_SND_ECE; TCPSTAT_INC(tcps_ecn_ce); break; case IPTOS_ECN_ECT0: TCPSTAT_INC(tcps_ecn_ect0); break; case IPTOS_ECN_ECT1: TCPSTAT_INC(tcps_ecn_ect1); break; } /* Process a packet differently from RFC3168. */ cc_ecnpkt_handler(tp, th, iptos); /* Congestion experienced. */ if (thflags & TH_ECE) { cc_cong_signal(tp, th, CC_ECN); } } /* * Parse options on any incoming segment. */ tcp_dooptions(&to, (u_char *)(th + 1), (th->th_off << 2) - sizeof(struct tcphdr), (thflags & TH_SYN) ? TO_SYN : 0); #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) if ((tp->t_flags & TF_SIGNATURE) != 0 && (to.to_flags & TOF_SIGNATURE) == 0) { TCPSTAT_INC(tcps_sig_err_sigopt); /* XXX: should drop? */ } #endif /* * If echoed timestamp is later than the current time, * fall back to non RFC1323 RTT calculation. Normalize * timestamp if syncookies were used when this connection * was established. */ if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) { to.to_tsecr -= tp->ts_offset; if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks())) to.to_tsecr = 0; } /* * If timestamps were negotiated during SYN/ACK they should * appear on every segment during this session and vice versa. */ if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp missing, " "no action\n", s, __func__); free(s, M_TCPLOG); } } if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp not expected, " "no action\n", s, __func__); free(s, M_TCPLOG); } } /* * Process options only when we get SYN/ACK back. The SYN case * for incoming connections is handled in tcp_syncache. * According to RFC1323 the window field in a SYN (i.e., a * or ) segment itself is never scaled. * XXX this is traditional behavior, may need to be cleaned up. */ if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { if ((to.to_flags & TOF_SCALE) && (tp->t_flags & TF_REQ_SCALE)) { tp->t_flags |= TF_RCVD_SCALE; tp->snd_scale = to.to_wscale; } /* * Initial send window. It will be updated with * the next incoming segment to the scaled value. */ tp->snd_wnd = th->th_win; if (to.to_flags & TOF_TS) { tp->t_flags |= TF_RCVD_TSTMP; tp->ts_recent = to.to_tsval; tp->ts_recent_age = tcp_ts_getticks(); } if (to.to_flags & TOF_MSS) tcp_mss(tp, to.to_mss); if ((tp->t_flags & TF_SACK_PERMIT) && (to.to_flags & TOF_SACKPERM) == 0) tp->t_flags &= ~TF_SACK_PERMIT; } /* * Header prediction: check for the two common cases * of a uni-directional data xfer. If the packet has * no control flags, is in-sequence, the window didn't * change and we're not retransmitting, it's a * candidate. If the length is zero and the ack moved * forward, we're the sender side of the xfer. Just * free the data acked & wake any higher level process * that was blocked waiting for space. If the length * is non-zero and the ack didn't move, we're the * receiver side. If we're getting packets in-order * (the reassembly queue is empty), add the data to * the socket buffer and note that we need a delayed ack. * Make sure that the hidden state-flags are also off. * Since we check for TCPS_ESTABLISHED first, it can only * be TH_NEEDSYN. */ if (tp->t_state == TCPS_ESTABLISHED && th->th_seq == tp->rcv_nxt && (thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK && tp->snd_nxt == tp->snd_max && tiwin && tiwin == tp->snd_wnd && ((tp->t_flags & (TF_NEEDSYN|TF_NEEDFIN)) == 0) && LIST_EMPTY(&tp->t_segq) && ((to.to_flags & TOF_TS) == 0 || TSTMP_GEQ(to.to_tsval, tp->ts_recent)) ) { /* * If last ACK falls within this segment's sequence numbers, * record the timestamp. * NOTE that the test is modified according to the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). */ if ((to.to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to.to_tsval; } if (tlen == 0) { if (SEQ_GT(th->th_ack, tp->snd_una) && SEQ_LEQ(th->th_ack, tp->snd_max) && !IN_RECOVERY(tp->t_flags) && (to.to_flags & TOF_SACK) == 0 && TAILQ_EMPTY(&tp->snd_holes)) { /* * This is a pure ack for outstanding data. */ if (ti_locked == TI_RLOCKED) INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; TCPSTAT_INC(tcps_predack); /* * "bad retransmit" recovery. */ if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) { cc_cong_signal(tp, th, CC_RTO_ERR); } /* * Recalculate the transmit timer / rtt. * * Some boxes send broken timestamp replies * during the SYN+ACK phase, ignore * timestamps of 0 or we could calculate a * huge RTT and blow up the retransmit timer. */ if ((to.to_flags & TOF_TS) != 0 && to.to_tsecr) { uint32_t t; t = tcp_ts_getticks() - to.to_tsecr; if (!tp->t_rttlow || tp->t_rttlow > t) tp->t_rttlow = t; tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) tp->t_rttlow = ticks - tp->t_rtttime; tcp_xmit_timer(tp, ticks - tp->t_rtttime); } acked = BYTES_THIS_ACK(tp, th); #ifdef TCP_HHOOK /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ hhook_run_tcp_est_in(tp, th, &to); #endif TCPSTAT_ADD(tcps_rcvackpack, nsegs); TCPSTAT_ADD(tcps_rcvackbyte, acked); sbdrop(&so->so_snd, acked); if (SEQ_GT(tp->snd_una, tp->snd_recover) && SEQ_LEQ(th->th_ack, tp->snd_recover)) tp->snd_recover = th->th_ack - 1; /* * Let the congestion control algorithm update * congestion control related information. This * typically means increasing the congestion * window. */ cc_ack_received(tp, th, nsegs, CC_ACK); tp->snd_una = th->th_ack; /* * Pull snd_wl2 up to prevent seq wrap relative * to th_ack. */ tp->snd_wl2 = th->th_ack; tp->t_dupacks = 0; m_freem(m); /* * If all outstanding data are acked, stop * retransmit timer, otherwise restart timer * using current (possibly backed-off) value. * If process is waiting for space, * wakeup/selwakeup/signal. If data * are ready to send, let tcp_output * decide between more output or persist. */ #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); if (tp->snd_una == tp->snd_max) tcp_timer_activate(tp, TT_REXMT, 0); else if (!tcp_timer_active(tp, TT_PERSIST)) tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); sowwakeup(so); if (sbavail(&so->so_snd)) (void) tp->t_fb->tfb_tcp_output(tp); goto check_delack; } } else if (th->th_ack == tp->snd_una && tlen <= sbspace(&so->so_rcv)) { int newsize = 0; /* automatic sockbuf scaling */ /* * This is a pure, in-sequence data packet with * nothing on the reassembly queue and we have enough * buffer space to take it. */ if (ti_locked == TI_RLOCKED) INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; /* Clean receiver SACK report if present */ if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks) tcp_clean_sackreport(tp); TCPSTAT_INC(tcps_preddat); tp->rcv_nxt += tlen; /* * Pull snd_wl1 up to prevent seq wrap relative to * th_seq. */ tp->snd_wl1 = th->th_seq; /* * Pull rcv_up up to prevent seq wrap relative to * rcv_nxt. */ tp->rcv_up = tp->rcv_nxt; TCPSTAT_ADD(tcps_rcvpack, nsegs); TCPSTAT_ADD(tcps_rcvbyte, tlen); #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); newsize = tcp_autorcvbuf(m, th, so, tp, tlen); /* Add data to socket buffer. */ SOCKBUF_LOCK(&so->so_rcv); if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { m_freem(m); } else { /* * Set new socket buffer size. * Give up when limit is reached. */ if (newsize) if (!sbreserve_locked(&so->so_rcv, newsize, so, NULL)) so->so_rcv.sb_flags &= ~SB_AUTOSIZE; m_adj(m, drop_hdrlen); /* delayed header drop */ sbappendstream_locked(&so->so_rcv, m, 0); } /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); if (DELAY_ACK(tp, tlen)) { tp->t_flags |= TF_DELACK; } else { tp->t_flags |= TF_ACKNOW; tp->t_fb->tfb_tcp_output(tp); } goto check_delack; } } /* * Calculate amount of space in receive window, * and then do TCP input processing. * Receive window is amount of space in rcv queue, * but not less than advertised window. */ win = sbspace(&so->so_rcv); if (win < 0) win = 0; tp->rcv_wnd = imax(win, (int)(tp->rcv_adv - tp->rcv_nxt)); switch (tp->t_state) { /* * If the state is SYN_RECEIVED: * if seg contains an ACK, but not for our SYN/ACK, send a RST. */ case TCPS_SYN_RECEIVED: if ((thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->snd_una) || SEQ_GT(th->th_ack, tp->snd_max))) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } #ifdef TCP_RFC7413 if (IS_FASTOPEN(tp->t_flags)) { /* * When a TFO connection is in SYN_RECEIVED, the * only valid packets are the initial SYN, a * retransmit/copy of the initial SYN (possibly with * a subset of the original data), a valid ACK, a * FIN, or a RST. */ if ((thflags & (TH_SYN|TH_ACK)) == (TH_SYN|TH_ACK)) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } else if (thflags & TH_SYN) { /* non-initial SYN is ignored */ if ((tcp_timer_active(tp, TT_DELACK) || tcp_timer_active(tp, TT_REXMT))) goto drop; } else if (!(thflags & (TH_ACK|TH_FIN|TH_RST))) { goto drop; } } #endif break; /* * If the state is SYN_SENT: * if seg contains a RST with valid ACK (SEQ.ACK has already * been verified), then drop the connection. * if seg contains a RST without an ACK, drop the seg. * if seg does not contain SYN, then drop the seg. * Otherwise this is an acceptable SYN segment * initialize tp->rcv_nxt and tp->irs * if seg contains ack then advance tp->snd_una * if seg contains an ECE and ECN support is enabled, the stream * is ECN capable. * if SYN has been acked change to ESTABLISHED else SYN_RCVD state * arrange for segment to be acked (eventually) * continue processing rest of data/controls, beginning with URG */ case TCPS_SYN_SENT: if ((thflags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) { TCP_PROBE5(connect__refused, NULL, tp, m, tp, th); tp = tcp_drop(tp, ECONNREFUSED); } if (thflags & TH_RST) goto drop; if (!(thflags & TH_SYN)) goto drop; tp->irs = th->th_seq; tcp_rcvseqinit(tp); if (thflags & TH_ACK) { TCPSTAT_INC(tcps_connects); soisconnected(so); #ifdef MAC mac_socketpeer_set_from_mbuf(m, so); #endif /* Do window scaling on this connection? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; } tp->rcv_adv += min(tp->rcv_wnd, TCP_MAXWIN << tp->rcv_scale); tp->snd_una++; /* SYN is acked */ /* * If there's data, delay ACK; if there's also a FIN * ACKNOW will be turned on later. */ if (DELAY_ACK(tp, tlen) && tlen != 0) tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); else tp->t_flags |= TF_ACKNOW; if ((thflags & TH_ECE) && V_tcp_do_ecn) { tp->t_flags |= TF_ECN_PERMIT; TCPSTAT_INC(tcps_ecn_shs); } /* * Received in SYN_SENT[*] state. * Transitions: * SYN_SENT --> ESTABLISHED * SYN_SENT* --> FIN_WAIT_1 */ tp->t_starttime = ticks; if (tp->t_flags & TF_NEEDFIN) { tcp_state_change(tp, TCPS_FIN_WAIT_1); tp->t_flags &= ~TF_NEEDFIN; thflags &= ~TH_SYN; } else { tcp_state_change(tp, TCPS_ESTABLISHED); TCP_PROBE5(connect__established, NULL, tp, m, tp, th); cc_conn_init(tp); tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); } } else { /* * Received initial SYN in SYN-SENT[*] state => * simultaneous open. * If it succeeds, connection is * half-synchronized. * Otherwise, do 3-way handshake: * SYN-SENT -> SYN-RECEIVED * SYN-SENT* -> SYN-RECEIVED* */ tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN); tcp_timer_activate(tp, TT_REXMT, 0); tcp_state_change(tp, TCPS_SYN_RECEIVED); } KASSERT(ti_locked == TI_RLOCKED, ("%s: trimthenstep6: " "ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); /* * Advance th->th_seq to correspond to first data byte. * If data, trim to stay within window, * dropping FIN if necessary. */ th->th_seq++; if (tlen > tp->rcv_wnd) { todrop = tlen - tp->rcv_wnd; m_adj(m, -todrop); tlen = tp->rcv_wnd; thflags &= ~TH_FIN; TCPSTAT_INC(tcps_rcvpackafterwin); TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); } tp->snd_wl1 = th->th_seq - 1; tp->rcv_up = th->th_seq; /* * Client side of transaction: already sent SYN and data. * If the remote host used T/TCP to validate the SYN, * our data will be ACK'd; if so, enter normal data segment * processing in the middle of step 5, ack processing. * Otherwise, goto step 6. */ if (thflags & TH_ACK) goto process_ACK; goto step6; /* * If the state is LAST_ACK or CLOSING or TIME_WAIT: * do normal processing. * * NB: Leftover from RFC1644 T/TCP. Cases to be reused later. */ case TCPS_LAST_ACK: case TCPS_CLOSING: break; /* continue normal processing */ } /* * States other than LISTEN or SYN_SENT. * First check the RST flag and sequence number since reset segments * are exempt from the timestamp and connection count tests. This * fixes a bug introduced by the Stevens, vol. 2, p. 960 bugfix * below which allowed reset segments in half the sequence space * to fall though and be processed (which gives forged reset * segments with a random sequence number a 50 percent chance of * killing a connection). * Then check timestamp, if present. * Then check the connection count, if present. * Then check that at least some bytes of segment are within * receive window. If segment begins before rcv_nxt, * drop leading data (and SYN); if nothing left, just ack. */ if (thflags & TH_RST) { /* * RFC5961 Section 3.2 * * - RST drops connection only if SEG.SEQ == RCV.NXT. * - If RST is in window, we send challenge ACK. * * Note: to take into account delayed ACKs, we should * test against last_ack_sent instead of rcv_nxt. * Note 2: we handle special case of closed window, not * covered by the RFC. */ if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) && SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) || (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); KASSERT(ti_locked == TI_RLOCKED, ("%s: TH_RST ti_locked %d, th %p tp %p", __func__, ti_locked, th, tp)); KASSERT(tp->t_state != TCPS_SYN_SENT, ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p", __func__, th, tp)); if (V_tcp_insecure_rst || tp->last_ack_sent == th->th_seq) { TCPSTAT_INC(tcps_drops); /* Drop the connection. */ switch (tp->t_state) { case TCPS_SYN_RECEIVED: so->so_error = ECONNREFUSED; goto close; case TCPS_ESTABLISHED: case TCPS_FIN_WAIT_1: case TCPS_FIN_WAIT_2: case TCPS_CLOSE_WAIT: case TCPS_CLOSING: case TCPS_LAST_ACK: so->so_error = ECONNRESET; close: /* FALLTHROUGH */ default: tp = tcp_close(tp); } } else { TCPSTAT_INC(tcps_badrst); /* Send challenge ACK. */ tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, tp->snd_nxt, TH_ACK); tp->last_ack_sent = tp->rcv_nxt; m = NULL; } } goto drop; } /* * RFC5961 Section 4.2 * Send challenge ACK for any SYN in synchronized state. */ if ((thflags & TH_SYN) && tp->t_state != TCPS_SYN_SENT && tp->t_state != TCPS_SYN_RECEIVED) { KASSERT(ti_locked == TI_RLOCKED, ("tcp_do_segment: TH_SYN ti_locked %d", ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); TCPSTAT_INC(tcps_badsyn); if (V_tcp_insecure_syn && SEQ_GEQ(th->th_seq, tp->last_ack_sent) && SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) { tp = tcp_drop(tp, ECONNRESET); rstreason = BANDLIM_UNLIMITED; } else { /* Send challenge ACK. */ tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, tp->snd_nxt, TH_ACK); tp->last_ack_sent = tp->rcv_nxt; m = NULL; } goto drop; } /* * RFC 1323 PAWS: If we have a timestamp reply on this segment * and it's less than ts_recent, drop it. */ if ((to.to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to.to_tsval, tp->ts_recent)) { /* Check to see if ts_recent is over 24 days old. */ if (tcp_ts_getticks() - tp->ts_recent_age > TCP_PAWS_IDLE) { /* * Invalidate ts_recent. If this segment updates * ts_recent, the age will be reset later and ts_recent * will get a valid value. If it does not, setting * ts_recent to zero will at least satisfy the * requirement that zero be placed in the timestamp * echo reply when ts_recent isn't valid. The * age isn't reset until we get a valid ts_recent * because we don't want out-of-order segments to be * dropped when ts_recent is old. */ tp->ts_recent = 0; } else { TCPSTAT_INC(tcps_rcvduppack); TCPSTAT_ADD(tcps_rcvdupbyte, tlen); TCPSTAT_INC(tcps_pawsdrop); if (tlen) goto dropafterack; goto drop; } } /* * In the SYN-RECEIVED state, validate that the packet belongs to * this connection before trimming the data to fit the receive * window. Check the sequence number versus IRS since we know * the sequence numbers haven't wrapped. This is a partial fix * for the "LAND" DoS attack. */ if (tp->t_state == TCPS_SYN_RECEIVED && SEQ_LT(th->th_seq, tp->irs)) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } todrop = tp->rcv_nxt - th->th_seq; if (todrop > 0) { if (thflags & TH_SYN) { thflags &= ~TH_SYN; th->th_seq++; if (th->th_urp > 1) th->th_urp--; else thflags &= ~TH_URG; todrop--; } /* * Following if statement from Stevens, vol. 2, p. 960. */ if (todrop > tlen || (todrop == tlen && (thflags & TH_FIN) == 0)) { /* * Any valid FIN must be to the left of the window. * At this point the FIN must be a duplicate or out * of sequence; drop it. */ thflags &= ~TH_FIN; /* * Send an ACK to resynchronize and drop any data. * But keep on processing for RST or ACK. */ tp->t_flags |= TF_ACKNOW; todrop = tlen; TCPSTAT_INC(tcps_rcvduppack); TCPSTAT_ADD(tcps_rcvdupbyte, todrop); } else { TCPSTAT_INC(tcps_rcvpartduppack); TCPSTAT_ADD(tcps_rcvpartdupbyte, todrop); } drop_hdrlen += todrop; /* drop from the top afterwards */ th->th_seq += todrop; tlen -= todrop; if (th->th_urp > todrop) th->th_urp -= todrop; else { thflags &= ~TH_URG; th->th_urp = 0; } } /* * If new data are received on a connection after the * user processes are gone, then RST the other end. */ if ((so->so_state & SS_NOFDREF) && tp->t_state > TCPS_CLOSE_WAIT && tlen) { KASSERT(ti_locked == TI_RLOCKED, ("%s: SS_NOFDEREF && " "CLOSE_WAIT && tlen ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: %s: Received %d bytes of data " "after socket was closed, " "sending RST and removing tcpcb\n", s, __func__, tcpstates[tp->t_state], tlen); free(s, M_TCPLOG); } tp = tcp_close(tp); TCPSTAT_INC(tcps_rcvafterclose); rstreason = BANDLIM_UNLIMITED; goto dropwithreset; } /* * If segment ends after window, drop trailing data * (and PUSH and FIN); if nothing left, just ACK. */ todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd); if (todrop > 0) { TCPSTAT_INC(tcps_rcvpackafterwin); if (todrop >= tlen) { TCPSTAT_ADD(tcps_rcvbyteafterwin, tlen); /* * If window is closed can only take segments at * window edge, and have to drop data and PUSH from * incoming segments. Continue processing, but * remember to ack. Otherwise, drop segment * and ack. */ if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) { tp->t_flags |= TF_ACKNOW; TCPSTAT_INC(tcps_rcvwinprobe); } else goto dropafterack; } else TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); m_adj(m, -todrop); tlen -= todrop; thflags &= ~(TH_PUSH|TH_FIN); } /* * If last ACK falls within this segment's sequence numbers, * record its timestamp. * NOTE: * 1) That the test incorporates suggestions from the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). * 2) That updating only on newer timestamps interferes with * our earlier PAWS tests, so this check should be solely * predicated on the sequence space of this segment. * 3) That we modify the segment boundary check to be * Last.ACK.Sent <= SEG.SEQ + SEG.Len * instead of RFC1323's * Last.ACK.Sent < SEG.SEQ + SEG.Len, * This modified check allows us to overcome RFC1323's * limitations as described in Stevens TCP/IP Illustrated * Vol. 2 p.869. In such cases, we can still calculate the * RTT correctly when RCV.NXT == Last.ACK.Sent. */ if ((to.to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent) && SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + ((thflags & (TH_SYN|TH_FIN)) != 0))) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to.to_tsval; } /* * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN * flag is on (half-synchronized state), then queue data for * later processing; else drop segment and return. */ if ((thflags & TH_ACK) == 0) { if (tp->t_state == TCPS_SYN_RECEIVED || (tp->t_flags & TF_NEEDSYN)) { #ifdef TCP_RFC7413 if (tp->t_state == TCPS_SYN_RECEIVED && IS_FASTOPEN(tp->t_flags)) { tp->snd_wnd = tiwin; cc_conn_init(tp); } #endif goto step6; } else if (tp->t_flags & TF_ACKNOW) goto dropafterack; else goto drop; } /* * Ack processing. */ switch (tp->t_state) { /* * In SYN_RECEIVED state, the ack ACKs our SYN, so enter * ESTABLISHED state and continue processing. * The ACK was checked above. */ case TCPS_SYN_RECEIVED: TCPSTAT_INC(tcps_connects); soisconnected(so); /* Do window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; tp->snd_wnd = tiwin; } /* * Make transitions: * SYN-RECEIVED -> ESTABLISHED * SYN-RECEIVED* -> FIN-WAIT-1 */ tp->t_starttime = ticks; if (tp->t_flags & TF_NEEDFIN) { tcp_state_change(tp, TCPS_FIN_WAIT_1); tp->t_flags &= ~TF_NEEDFIN; } else { tcp_state_change(tp, TCPS_ESTABLISHED); TCP_PROBE5(accept__established, NULL, tp, m, tp, th); #ifdef TCP_RFC7413 if (tp->t_tfo_pending) { tcp_fastopen_decrement_counter(tp->t_tfo_pending); tp->t_tfo_pending = NULL; /* * Account for the ACK of our SYN prior to * regular ACK processing below. */ tp->snd_una++; } /* * TFO connections call cc_conn_init() during SYN * processing. Calling it again here for such * connections is not harmless as it would undo the * snd_cwnd reduction that occurs when a TFO SYN|ACK * is retransmitted. */ if (!IS_FASTOPEN(tp->t_flags)) #endif cc_conn_init(tp); tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); } /* * If segment contains data or ACK, will call tcp_reass() * later; if not, do so now to pass queued data to user. */ if (tlen == 0 && (thflags & TH_FIN) == 0) (void) tcp_reass(tp, (struct tcphdr *)0, 0, (struct mbuf *)0); tp->snd_wl1 = th->th_seq - 1; /* FALLTHROUGH */ /* * In ESTABLISHED state: drop duplicate ACKs; ACK out of range * ACKs. If the ack is in the range * tp->snd_una < th->th_ack <= tp->snd_max * then advance tp->snd_una to th->th_ack and drop * data from the retransmission queue. If this ACK reflects * more up to date window information we update our window information. */ case TCPS_ESTABLISHED: case TCPS_FIN_WAIT_1: case TCPS_FIN_WAIT_2: case TCPS_CLOSE_WAIT: case TCPS_CLOSING: case TCPS_LAST_ACK: if (SEQ_GT(th->th_ack, tp->snd_max)) { TCPSTAT_INC(tcps_rcvacktoomuch); goto dropafterack; } if ((tp->t_flags & TF_SACK_PERMIT) && ((to.to_flags & TOF_SACK) || !TAILQ_EMPTY(&tp->snd_holes))) sack_changed = tcp_sack_doack(tp, &to, th->th_ack); else /* * Reset the value so that previous (valid) value * from the last ack with SACK doesn't get used. */ tp->sackhint.sacked_bytes = 0; #ifdef TCP_HHOOK /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ hhook_run_tcp_est_in(tp, th, &to); #endif if (SEQ_LEQ(th->th_ack, tp->snd_una)) { u_int maxseg; maxseg = tcp_maxseg(tp); if (tlen == 0 && (tiwin == tp->snd_wnd || (tp->t_flags & TF_SACK_PERMIT))) { /* * If this is the first time we've seen a * FIN from the remote, this is not a * duplicate and it needs to be processed * normally. This happens during a * simultaneous close. */ if ((thflags & TH_FIN) && (TCPS_HAVERCVDFIN(tp->t_state) == 0)) { tp->t_dupacks = 0; break; } TCPSTAT_INC(tcps_rcvdupack); /* * If we have outstanding data (other than * a window probe), this is a completely * duplicate ack (ie, window info didn't * change and FIN isn't set), * the ack is the biggest we've * seen and we've seen exactly our rexmt * threshold of them, assume a packet * has been dropped and retransmit it. * Kludge snd_nxt & the congestion * window so we send only this one * packet. * * We know we're losing at the current * window size so do congestion avoidance * (set ssthresh to half the current window * and pull our congestion window back to * the new ssthresh). * * Dup acks mean that packets have left the * network (they're now cached at the receiver) * so bump cwnd by the amount in the receiver * to keep a constant cwnd packets in the * network. * * When using TCP ECN, notify the peer that * we reduced the cwnd. */ /* * Following 2 kinds of acks should not affect * dupack counting: * 1) Old acks * 2) Acks with SACK but without any new SACK * information in them. These could result from * any anomaly in the network like a switch * duplicating packets or a possible DoS attack. */ if (th->th_ack != tp->snd_una || ((tp->t_flags & TF_SACK_PERMIT) && !sack_changed)) break; else if (!tcp_timer_active(tp, TT_REXMT)) tp->t_dupacks = 0; else if (++tp->t_dupacks > tcprexmtthresh || IN_FASTRECOVERY(tp->t_flags)) { cc_ack_received(tp, th, nsegs, CC_DUPACK); if ((tp->t_flags & TF_SACK_PERMIT) && IN_FASTRECOVERY(tp->t_flags)) { int awnd; /* * Compute the amount of data in flight first. * We can inject new data into the pipe iff * we have less than 1/2 the original window's * worth of data in flight. */ if (V_tcp_do_rfc6675_pipe) awnd = tcp_compute_pipe(tp); else awnd = (tp->snd_nxt - tp->snd_fack) + tp->sackhint.sack_bytes_rexmit; if (awnd < tp->snd_ssthresh) { tp->snd_cwnd += maxseg; - /* - * RFC5681 Section 3.2 talks about cwnd - * inflation on additional dupacks and - * deflation on recovering from loss. - * - * We keep cwnd into check so that - * we don't have to 'deflate' it when we - * get out of recovery. - */ if (tp->snd_cwnd > tp->snd_ssthresh) tp->snd_cwnd = tp->snd_ssthresh; } } else tp->snd_cwnd += maxseg; (void) tp->t_fb->tfb_tcp_output(tp); goto drop; } else if (tp->t_dupacks == tcprexmtthresh) { tcp_seq onxt = tp->snd_nxt; /* * If we're doing sack, check to * see if we're already in sack * recovery. If we're not doing sack, * check to see if we're in newreno * recovery. */ if (tp->t_flags & TF_SACK_PERMIT) { if (IN_FASTRECOVERY(tp->t_flags)) { tp->t_dupacks = 0; break; } } else { if (SEQ_LEQ(th->th_ack, tp->snd_recover)) { tp->t_dupacks = 0; break; } } /* Congestion signal before ack. */ cc_cong_signal(tp, th, CC_NDUPACK); cc_ack_received(tp, th, nsegs, CC_DUPACK); tcp_timer_activate(tp, TT_REXMT, 0); tp->t_rtttime = 0; if (tp->t_flags & TF_SACK_PERMIT) { TCPSTAT_INC( tcps_sack_recovery_episode); tp->sack_newdata = tp->snd_nxt; - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = maxseg; + tp->snd_cwnd = maxseg; (void) tp->t_fb->tfb_tcp_output(tp); goto drop; } tp->snd_nxt = th->th_ack; - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = maxseg; + tp->snd_cwnd = maxseg; (void) tp->t_fb->tfb_tcp_output(tp); KASSERT(tp->snd_limited <= 2, ("%s: tp->snd_limited too big", __func__)); - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = tp->snd_ssthresh + - maxseg * - (tp->t_dupacks - tp->snd_limited); + tp->snd_cwnd = tp->snd_ssthresh + + maxseg * + (tp->t_dupacks - tp->snd_limited); if (SEQ_GT(onxt, tp->snd_nxt)) tp->snd_nxt = onxt; goto drop; } else if (V_tcp_do_rfc3042) { /* * Process first and second duplicate * ACKs. Each indicates a segment * leaving the network, creating room * for more. Make sure we can send a * packet on reception of each duplicate * ACK by increasing snd_cwnd by one * segment. Restore the original * snd_cwnd after packet transmission. */ cc_ack_received(tp, th, nsegs, CC_DUPACK); uint32_t oldcwnd = tp->snd_cwnd; tcp_seq oldsndmax = tp->snd_max; u_int sent; int avail; KASSERT(tp->t_dupacks == 1 || tp->t_dupacks == 2, ("%s: dupacks not 1 or 2", __func__)); if (tp->t_dupacks == 1) tp->snd_limited = 0; tp->snd_cwnd = (tp->snd_nxt - tp->snd_una) + (tp->t_dupacks - tp->snd_limited) * maxseg; /* * Only call tcp_output when there * is new data available to be sent. * Otherwise we would send pure ACKs. */ SOCKBUF_LOCK(&so->so_snd); avail = sbavail(&so->so_snd) - (tp->snd_nxt - tp->snd_una); SOCKBUF_UNLOCK(&so->so_snd); if (avail > 0) (void) tp->t_fb->tfb_tcp_output(tp); sent = tp->snd_max - oldsndmax; if (sent > maxseg) { KASSERT((tp->t_dupacks == 2 && tp->snd_limited == 0) || (sent == maxseg + 1 && tp->t_flags & TF_SENTFIN), ("%s: sent too much", __func__)); tp->snd_limited = 2; } else if (sent > 0) ++tp->snd_limited; tp->snd_cwnd = oldcwnd; goto drop; } } break; } else { /* * This ack is advancing the left edge, reset the * counter. */ tp->t_dupacks = 0; /* * If this ack also has new SACK info, increment the * counter as per rfc6675. */ if ((tp->t_flags & TF_SACK_PERMIT) && sack_changed) tp->t_dupacks++; } KASSERT(SEQ_GT(th->th_ack, tp->snd_una), ("%s: th_ack <= snd_una", __func__)); /* * If the congestion window was inflated to account * for the other side's cached packets, retract it. */ if (IN_FASTRECOVERY(tp->t_flags)) { if (SEQ_LT(th->th_ack, tp->snd_recover)) { if (tp->t_flags & TF_SACK_PERMIT) tcp_sack_partialack(tp, th); else tcp_newreno_partial_ack(tp, th); } else cc_post_recovery(tp, th); } /* * If we reach this point, ACK is not a duplicate, * i.e., it ACKs something we sent. */ if (tp->t_flags & TF_NEEDSYN) { /* * T/TCP: Connection was half-synchronized, and our * SYN has been ACK'd (so connection is now fully * synchronized). Go to non-starred state, * increment snd_una for ACK of SYN, and check if * we can do window scaling. */ tp->t_flags &= ~TF_NEEDSYN; tp->snd_una++; /* Do window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; /* Send window already scaled. */ } } process_ACK: INP_WLOCK_ASSERT(tp->t_inpcb); acked = BYTES_THIS_ACK(tp, th); KASSERT(acked >= 0, ("%s: acked unexepectedly negative " "(tp->snd_una=%u, th->th_ack=%u, tp=%p, m=%p)", __func__, tp->snd_una, th->th_ack, tp, m)); TCPSTAT_ADD(tcps_rcvackpack, nsegs); TCPSTAT_ADD(tcps_rcvackbyte, acked); /* * If we just performed our first retransmit, and the ACK * arrives within our recovery window, then it was a mistake * to do the retransmit in the first place. Recover our * original cwnd and ssthresh, and proceed to transmit where * we left off. */ if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) cc_cong_signal(tp, th, CC_RTO_ERR); /* * If we have a timestamp reply, update smoothed * round trip time. If no timestamp is present but * transmit timer is running and timed sequence * number was acked, update smoothed round trip time. * Since we now have an rtt measurement, cancel the * timer backoff (cf., Phil Karn's retransmit alg.). * Recompute the initial retransmit timer. * * Some boxes send broken timestamp replies * during the SYN+ACK phase, ignore * timestamps of 0 or we could calculate a * huge RTT and blow up the retransmit timer. */ if ((to.to_flags & TOF_TS) != 0 && to.to_tsecr) { uint32_t t; t = tcp_ts_getticks() - to.to_tsecr; if (!tp->t_rttlow || tp->t_rttlow > t) tp->t_rttlow = t; tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) tp->t_rttlow = ticks - tp->t_rtttime; tcp_xmit_timer(tp, ticks - tp->t_rtttime); } /* * If all outstanding data is acked, stop retransmit * timer and remember to restart (more output or persist). * If there is more data to be acked, restart retransmit * timer, using current (possibly backed-off) value. */ if (th->th_ack == tp->snd_max) { tcp_timer_activate(tp, TT_REXMT, 0); needoutput = 1; } else if (!tcp_timer_active(tp, TT_PERSIST)) tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); /* * If no data (only SYN) was ACK'd, * skip rest of ACK processing. */ if (acked == 0) goto step6; /* * Let the congestion control algorithm update congestion * control related information. This typically means increasing * the congestion window. */ cc_ack_received(tp, th, nsegs, CC_ACK); SOCKBUF_LOCK(&so->so_snd); if (acked > sbavail(&so->so_snd)) { if (tp->snd_wnd >= sbavail(&so->so_snd)) tp->snd_wnd -= sbavail(&so->so_snd); else tp->snd_wnd = 0; mfree = sbcut_locked(&so->so_snd, (int)sbavail(&so->so_snd)); ourfinisacked = 1; } else { mfree = sbcut_locked(&so->so_snd, acked); if (tp->snd_wnd >= (uint32_t) acked) tp->snd_wnd -= acked; else tp->snd_wnd = 0; ourfinisacked = 0; } /* NB: sowwakeup_locked() does an implicit unlock. */ sowwakeup_locked(so); m_freem(mfree); /* Detect una wraparound. */ if (!IN_RECOVERY(tp->t_flags) && SEQ_GT(tp->snd_una, tp->snd_recover) && SEQ_LEQ(th->th_ack, tp->snd_recover)) tp->snd_recover = th->th_ack - 1; /* XXXLAS: Can this be moved up into cc_post_recovery? */ if (IN_RECOVERY(tp->t_flags) && SEQ_GEQ(th->th_ack, tp->snd_recover)) { EXIT_RECOVERY(tp->t_flags); } tp->snd_una = th->th_ack; if (tp->t_flags & TF_SACK_PERMIT) { if (SEQ_GT(tp->snd_una, tp->snd_recover)) tp->snd_recover = tp->snd_una; } if (SEQ_LT(tp->snd_nxt, tp->snd_una)) tp->snd_nxt = tp->snd_una; switch (tp->t_state) { /* * In FIN_WAIT_1 STATE in addition to the processing * for the ESTABLISHED state if our FIN is now acknowledged * then enter FIN_WAIT_2. */ case TCPS_FIN_WAIT_1: if (ourfinisacked) { /* * If we can't receive any more * data, then closing user can proceed. * Starting the timer is contrary to the * specification, but if we don't get a FIN * we'll hang forever. * * XXXjl: * we should release the tp also, and use a * compressed state. */ if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { soisdisconnected(so); tcp_timer_activate(tp, TT_2MSL, (tcp_fast_finwait2_recycle ? tcp_finwait2_timeout : TP_MAXIDLE(tp))); } tcp_state_change(tp, TCPS_FIN_WAIT_2); } break; /* * In CLOSING STATE in addition to the processing for * the ESTABLISHED state if the ACK acknowledges our FIN * then enter the TIME-WAIT state, otherwise ignore * the segment. */ case TCPS_CLOSING: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tcp_twstart(tp); INP_INFO_RUNLOCK(&V_tcbinfo); m_freem(m); return; } break; /* * In LAST_ACK, we may still be waiting for data to drain * and/or to be acked, as well as for the ack of our FIN. * If our FIN is now acknowledged, delete the TCB, * enter the closed state and return. */ case TCPS_LAST_ACK: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tp = tcp_close(tp); goto drop; } break; } } step6: INP_WLOCK_ASSERT(tp->t_inpcb); /* * Update window information. * Don't look at window if no ACK: TAC's send garbage on first SYN. */ if ((thflags & TH_ACK) && (SEQ_LT(tp->snd_wl1, th->th_seq) || (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { /* keep track of pure window updates */ if (tlen == 0 && tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) TCPSTAT_INC(tcps_rcvwinupd); tp->snd_wnd = tiwin; tp->snd_wl1 = th->th_seq; tp->snd_wl2 = th->th_ack; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; needoutput = 1; } /* * Process segments with URG. */ if ((thflags & TH_URG) && th->th_urp && TCPS_HAVERCVDFIN(tp->t_state) == 0) { /* * This is a kludge, but if we receive and accept * random urgent pointers, we'll crash in * soreceive. It's hard to imagine someone * actually wanting to send this much urgent data. */ SOCKBUF_LOCK(&so->so_rcv); if (th->th_urp + sbavail(&so->so_rcv) > sb_max) { th->th_urp = 0; /* XXX */ thflags &= ~TH_URG; /* XXX */ SOCKBUF_UNLOCK(&so->so_rcv); /* XXX */ goto dodata; /* XXX */ } /* * If this segment advances the known urgent pointer, * then mark the data stream. This should not happen * in CLOSE_WAIT, CLOSING, LAST_ACK or TIME_WAIT STATES since * a FIN has been received from the remote side. * In these states we ignore the URG. * * According to RFC961 (Assigned Protocols), * the urgent pointer points to the last octet * of urgent data. We continue, however, * to consider it to indicate the first octet * of data past the urgent section as the original * spec states (in one of two places). */ if (SEQ_GT(th->th_seq+th->th_urp, tp->rcv_up)) { tp->rcv_up = th->th_seq + th->th_urp; so->so_oobmark = sbavail(&so->so_rcv) + (tp->rcv_up - tp->rcv_nxt) - 1; if (so->so_oobmark == 0) so->so_rcv.sb_state |= SBS_RCVATMARK; sohasoutofband(so); tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA); } SOCKBUF_UNLOCK(&so->so_rcv); /* * Remove out of band data so doesn't get presented to user. * This can happen independent of advancing the URG pointer, * but if two URG's are pending at once, some out-of-band * data may creep in... ick. */ if (th->th_urp <= (uint32_t)tlen && !(so->so_options & SO_OOBINLINE)) { /* hdr drop is delayed */ tcp_pulloutofband(so, th, m, drop_hdrlen); } } else { /* * If no out of band data is expected, * pull receive urgent pointer along * with the receive window. */ if (SEQ_GT(tp->rcv_nxt, tp->rcv_up)) tp->rcv_up = tp->rcv_nxt; } dodata: /* XXX */ INP_WLOCK_ASSERT(tp->t_inpcb); /* * Process the segment text, merging it into the TCP sequencing queue, * and arranging for acknowledgment of receipt if necessary. * This process logically involves adjusting tp->rcv_wnd as data * is presented to the user (this happens in tcp_usrreq.c, * case PRU_RCVD). If a FIN has already been received on this * connection then we just ignore the text. */ #ifdef TCP_RFC7413 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && IS_FASTOPEN(tp->t_flags)); #else #define tfo_syn (false) #endif if ((tlen || (thflags & TH_FIN) || tfo_syn) && TCPS_HAVERCVDFIN(tp->t_state) == 0) { tcp_seq save_start = th->th_seq; m_adj(m, drop_hdrlen); /* delayed header drop */ /* * Insert segment which includes th into TCP reassembly queue * with control block tp. Set thflags to whether reassembly now * includes a segment with FIN. This handles the common case * inline (segment is the next to be received on an established * connection, and the queue is empty), avoiding linkage into * and removal from the queue and repetition of various * conversions. * Set DELACK for segments received in order, but ack * immediately when segments are out of order (so * fast retransmit can work). */ if (th->th_seq == tp->rcv_nxt && LIST_EMPTY(&tp->t_segq) && (TCPS_HAVEESTABLISHED(tp->t_state) || tfo_syn)) { if (DELAY_ACK(tp, tlen) || tfo_syn) tp->t_flags |= TF_DELACK; else tp->t_flags |= TF_ACKNOW; tp->rcv_nxt += tlen; thflags = th->th_flags & TH_FIN; TCPSTAT_INC(tcps_rcvpack); TCPSTAT_ADD(tcps_rcvbyte, tlen); SOCKBUF_LOCK(&so->so_rcv); if (so->so_rcv.sb_state & SBS_CANTRCVMORE) m_freem(m); else sbappendstream_locked(&so->so_rcv, m, 0); /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); } else { /* * XXX: Due to the header drop above "th" is * theoretically invalid by now. Fortunately * m_adj() doesn't actually frees any mbufs * when trimming from the head. */ thflags = tcp_reass(tp, th, &tlen, m); tp->t_flags |= TF_ACKNOW; } if (tlen > 0 && (tp->t_flags & TF_SACK_PERMIT)) tcp_update_sack_list(tp, save_start, save_start + tlen); #if 0 /* * Note the amount of data that peer has sent into * our window, in order to estimate the sender's * buffer size. * XXX: Unused. */ if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) len = so->so_rcv.sb_hiwat - (tp->rcv_adv - tp->rcv_nxt); else len = so->so_rcv.sb_hiwat; #endif } else { m_freem(m); thflags &= ~TH_FIN; } /* * If FIN is received ACK the FIN and let the user know * that the connection is closing. */ if (thflags & TH_FIN) { if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { socantrcvmore(so); /* * If connection is half-synchronized * (ie NEEDSYN flag on) then delay ACK, * so it may be piggybacked when SYN is sent. * Otherwise, since we received a FIN then no * more input can be expected, send ACK now. */ if (tp->t_flags & TF_NEEDSYN) tp->t_flags |= TF_DELACK; else tp->t_flags |= TF_ACKNOW; tp->rcv_nxt++; } switch (tp->t_state) { /* * In SYN_RECEIVED and ESTABLISHED STATES * enter the CLOSE_WAIT state. */ case TCPS_SYN_RECEIVED: tp->t_starttime = ticks; /* FALLTHROUGH */ case TCPS_ESTABLISHED: tcp_state_change(tp, TCPS_CLOSE_WAIT); break; /* * If still in FIN_WAIT_1 STATE FIN has not been acked so * enter the CLOSING state. */ case TCPS_FIN_WAIT_1: tcp_state_change(tp, TCPS_CLOSING); break; /* * In FIN_WAIT_2 state enter the TIME_WAIT state, * starting the time-wait timer, turning off the other * standard timers. */ case TCPS_FIN_WAIT_2: INP_INFO_RLOCK_ASSERT(&V_tcbinfo); KASSERT(ti_locked == TI_RLOCKED, ("%s: dodata " "TCP_FIN_WAIT_2 ti_locked: %d", __func__, ti_locked)); tcp_twstart(tp); INP_INFO_RUNLOCK(&V_tcbinfo); return; } } if (ti_locked == TI_RLOCKED) INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); /* * Return any desired output. */ if (needoutput || (tp->t_flags & TF_ACKNOW)) (void) tp->t_fb->tfb_tcp_output(tp); check_delack: KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { tp->t_flags &= ~TF_DELACK; tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } INP_WUNLOCK(tp->t_inpcb); return; dropafterack: /* * Generate an ACK dropping incoming segment if it occupies * sequence space, where the ACK reflects our state. * * We can now skip the test for the RST flag since all * paths to this code happen after packets containing * RST have been dropped. * * In the SYN-RECEIVED state, don't send an ACK unless the * segment we received passes the SYN-RECEIVED ACK test. * If it fails send a RST. This breaks the loop in the * "LAND" DoS attack, and also prevents an ACK storm * between two listening ports that have been sent forged * SYN segments, each with the source address of the other. */ if (tp->t_state == TCPS_SYN_RECEIVED && (thflags & TH_ACK) && (SEQ_GT(tp->snd_una, th->th_ack) || SEQ_GT(th->th_ack, tp->snd_max)) ) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); if (ti_locked == TI_RLOCKED) INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; tp->t_flags |= TF_ACKNOW; (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; dropwithreset: if (ti_locked == TI_RLOCKED) INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; if (tp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(tp->t_inpcb); } else tcp_dropwithreset(m, th, NULL, tlen, rstreason); return; drop: if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS else INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); #endif /* * Drop space held by incoming segment and return. */ #ifdef TCPDEBUG if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); if (tp != NULL) INP_WUNLOCK(tp->t_inpcb); m_freem(m); #ifndef TCP_RFC7413 #undef tfo_syn #endif } /* * Issue RST and make ACK acceptable to originator of segment. * The mbuf must still include the original packet header. * tp may be NULL. */ void tcp_dropwithreset(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int tlen, int rstreason) { #ifdef INET struct ip *ip; #endif #ifdef INET6 struct ip6_hdr *ip6; #endif if (tp != NULL) { INP_WLOCK_ASSERT(tp->t_inpcb); } /* Don't bother if destination was broadcast/multicast. */ if ((th->th_flags & TH_RST) || m->m_flags & (M_BCAST|M_MCAST)) goto drop; #ifdef INET6 if (mtod(m, struct ip *)->ip_v == 6) { ip6 = mtod(m, struct ip6_hdr *); if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) || IN6_IS_ADDR_MULTICAST(&ip6->ip6_src)) goto drop; /* IPv6 anycast check is done at tcp6_input() */ } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { ip = mtod(m, struct ip *); if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) || IN_MULTICAST(ntohl(ip->ip_src.s_addr)) || ip->ip_src.s_addr == htonl(INADDR_BROADCAST) || in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) goto drop; } #endif /* Perform bandwidth limiting. */ if (badport_bandlim(rstreason) < 0) goto drop; /* tcp_respond consumes the mbuf chain. */ if (th->th_flags & TH_ACK) { tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0, th->th_ack, TH_RST); } else { if (th->th_flags & TH_SYN) tlen++; if (th->th_flags & TH_FIN) tlen++; tcp_respond(tp, mtod(m, void *), th, m, th->th_seq+tlen, (tcp_seq)0, TH_RST|TH_ACK); } return; drop: m_freem(m); } /* * Parse TCP options and place in tcpopt. */ void tcp_dooptions(struct tcpopt *to, u_char *cp, int cnt, int flags) { int opt, optlen; to->to_flags = 0; for (; cnt > 0; cnt -= optlen, cp += optlen) { opt = cp[0]; if (opt == TCPOPT_EOL) break; if (opt == TCPOPT_NOP) optlen = 1; else { if (cnt < 2) break; optlen = cp[1]; if (optlen < 2 || optlen > cnt) break; } switch (opt) { case TCPOPT_MAXSEG: if (optlen != TCPOLEN_MAXSEG) continue; if (!(flags & TO_SYN)) continue; to->to_flags |= TOF_MSS; bcopy((char *)cp + 2, (char *)&to->to_mss, sizeof(to->to_mss)); to->to_mss = ntohs(to->to_mss); break; case TCPOPT_WINDOW: if (optlen != TCPOLEN_WINDOW) continue; if (!(flags & TO_SYN)) continue; to->to_flags |= TOF_SCALE; to->to_wscale = min(cp[2], TCP_MAX_WINSHIFT); break; case TCPOPT_TIMESTAMP: if (optlen != TCPOLEN_TIMESTAMP) continue; to->to_flags |= TOF_TS; bcopy((char *)cp + 2, (char *)&to->to_tsval, sizeof(to->to_tsval)); to->to_tsval = ntohl(to->to_tsval); bcopy((char *)cp + 6, (char *)&to->to_tsecr, sizeof(to->to_tsecr)); to->to_tsecr = ntohl(to->to_tsecr); break; case TCPOPT_SIGNATURE: /* * In order to reply to a host which has set the * TCP_SIGNATURE option in its initial SYN, we have * to record the fact that the option was observed * here for the syncache code to perform the correct * response. */ if (optlen != TCPOLEN_SIGNATURE) continue; to->to_flags |= TOF_SIGNATURE; to->to_signature = cp + 2; break; case TCPOPT_SACK_PERMITTED: if (optlen != TCPOLEN_SACK_PERMITTED) continue; if (!(flags & TO_SYN)) continue; if (!V_tcp_do_sack) continue; to->to_flags |= TOF_SACKPERM; break; case TCPOPT_SACK: if (optlen <= 2 || (optlen - 2) % TCPOLEN_SACK != 0) continue; if (flags & TO_SYN) continue; to->to_flags |= TOF_SACK; to->to_nsacks = (optlen - 2) / TCPOLEN_SACK; to->to_sacks = cp + 2; TCPSTAT_INC(tcps_sack_rcv_blocks); break; #ifdef TCP_RFC7413 case TCPOPT_FAST_OPEN: if ((optlen != TCPOLEN_FAST_OPEN_EMPTY) && (optlen < TCPOLEN_FAST_OPEN_MIN) && (optlen > TCPOLEN_FAST_OPEN_MAX)) continue; if (!(flags & TO_SYN)) continue; if (!V_tcp_fastopen_enabled) continue; to->to_flags |= TOF_FASTOPEN; to->to_tfo_len = optlen - 2; to->to_tfo_cookie = to->to_tfo_len ? cp + 2 : NULL; break; #endif default: continue; } } } /* * Pull out of band byte out of a segment so * it doesn't appear in the user's data queue. * It is still reflected in the segment length for * sequencing purposes. */ void tcp_pulloutofband(struct socket *so, struct tcphdr *th, struct mbuf *m, int off) { int cnt = off + th->th_urp - 1; while (cnt >= 0) { if (m->m_len > cnt) { char *cp = mtod(m, caddr_t) + cnt; struct tcpcb *tp = sototcpcb(so); INP_WLOCK_ASSERT(tp->t_inpcb); tp->t_iobc = *cp; tp->t_oobflags |= TCPOOB_HAVEDATA; bcopy(cp+1, cp, (unsigned)(m->m_len - cnt - 1)); m->m_len--; if (m->m_flags & M_PKTHDR) m->m_pkthdr.len--; return; } cnt -= m->m_len; m = m->m_next; if (m == NULL) break; } panic("tcp_pulloutofband"); } /* * Collect new round-trip time estimate * and update averages and current timeout. */ void tcp_xmit_timer(struct tcpcb *tp, int rtt) { int delta; INP_WLOCK_ASSERT(tp->t_inpcb); TCPSTAT_INC(tcps_rttupdated); tp->t_rttupdated++; if ((tp->t_srtt != 0) && (tp->t_rxtshift <= TCP_RTT_INVALIDATE)) { /* * srtt is stored as fixed point with 5 bits after the * binary point (i.e., scaled by 8). The following magic * is equivalent to the smoothing algorithm in rfc793 with * an alpha of .875 (srtt = rtt/8 + srtt*7/8 in fixed * point). Adjust rtt to origin 0. */ delta = ((rtt - 1) << TCP_DELTA_SHIFT) - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT)); if ((tp->t_srtt += delta) <= 0) tp->t_srtt = 1; /* * We accumulate a smoothed rtt variance (actually, a * smoothed mean difference), then set the retransmit * timer to smoothed rtt + 4 times the smoothed variance. * rttvar is stored as fixed point with 4 bits after the * binary point (scaled by 16). The following is * equivalent to rfc793 smoothing with an alpha of .75 * (rttvar = rttvar*3/4 + |delta| / 4). This replaces * rfc793's wired-in beta. */ if (delta < 0) delta = -delta; delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT); if ((tp->t_rttvar += delta) <= 0) tp->t_rttvar = 1; if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar) tp->t_rttbest = tp->t_srtt + tp->t_rttvar; } else { /* * No rtt measurement yet - use the unsmoothed rtt. * Set the variance to half the rtt (so our first * retransmit happens at 3*rtt). */ tp->t_srtt = rtt << TCP_RTT_SHIFT; tp->t_rttvar = rtt << (TCP_RTTVAR_SHIFT - 1); tp->t_rttbest = tp->t_srtt + tp->t_rttvar; } tp->t_rtttime = 0; tp->t_rxtshift = 0; /* * the retransmit should happen at rtt + 4 * rttvar. * Because of the way we do the smoothing, srtt and rttvar * will each average +1/2 tick of bias. When we compute * the retransmit timer, we want 1/2 tick of rounding and * 1 extra tick because of +-1/2 tick uncertainty in the * firing of the timer. The bias will give us exactly the * 1.5 tick we need. But, because the bias is * statistical, we have to test that we don't drop below * the minimum feasible timer (which is 2 ticks). */ TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp), max(tp->t_rttmin, rtt + 2), TCPTV_REXMTMAX); /* * We received an ack for a packet that wasn't retransmitted; * it is probably safe to discard any error indications we've * received recently. This isn't quite right, but close enough * for now (a route might have failed after we sent a segment, * and the return path might not be symmetrical). */ tp->t_softerror = 0; } /* * Determine a reasonable value for maxseg size. * If the route is known, check route for mtu. * If none, use an mss that can be handled on the outgoing interface * without forcing IP to fragment. If no route is found, route has no mtu, * or the destination isn't local, use a default, hopefully conservative * size (usually 512 or the default IP max size, but no more than the mtu * of the interface), as we can't discover anything about intervening * gateways or networks. We also initialize the congestion/slow start * window to be a single segment if the destination isn't local. * While looking at the routing entry, we also initialize other path-dependent * parameters from pre-set or cached values in the routing entry. * * NOTE that resulting t_maxseg doesn't include space for TCP options or * IP options, e.g. IPSEC data, since length of this data may vary, and * thus it is calculated for every segment separately in tcp_output(). * * NOTE that this routine is only called when we process an incoming * segment, or an ICMP need fragmentation datagram. Outgoing SYN/ACK MSS * settings are handled in tcp_mssopt(). */ void tcp_mss_update(struct tcpcb *tp, int offer, int mtuoffer, struct hc_metrics_lite *metricptr, struct tcp_ifcap *cap) { int mss = 0; uint32_t maxmtu = 0; struct inpcb *inp = tp->t_inpcb; struct hc_metrics_lite metrics; #ifdef INET6 int isipv6 = ((inp->inp_vflag & INP_IPV6) != 0) ? 1 : 0; size_t min_protoh = isipv6 ? sizeof (struct ip6_hdr) + sizeof (struct tcphdr) : sizeof (struct tcpiphdr); #else const size_t min_protoh = sizeof(struct tcpiphdr); #endif INP_WLOCK_ASSERT(tp->t_inpcb); if (mtuoffer != -1) { KASSERT(offer == -1, ("%s: conflict", __func__)); offer = mtuoffer - min_protoh; } /* Initialize. */ #ifdef INET6 if (isipv6) { maxmtu = tcp_maxmtu6(&inp->inp_inc, cap); tp->t_maxseg = V_tcp_v6mssdflt; } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { maxmtu = tcp_maxmtu(&inp->inp_inc, cap); tp->t_maxseg = V_tcp_mssdflt; } #endif /* * No route to sender, stay with default mss and return. */ if (maxmtu == 0) { /* * In case we return early we need to initialize metrics * to a defined state as tcp_hc_get() would do for us * if there was no cache hit. */ if (metricptr != NULL) bzero(metricptr, sizeof(struct hc_metrics_lite)); return; } /* What have we got? */ switch (offer) { case 0: /* * Offer == 0 means that there was no MSS on the SYN * segment, in this case we use tcp_mssdflt as * already assigned to t_maxseg above. */ offer = tp->t_maxseg; break; case -1: /* * Offer == -1 means that we didn't receive SYN yet. */ /* FALLTHROUGH */ default: /* * Prevent DoS attack with too small MSS. Round up * to at least minmss. */ offer = max(offer, V_tcp_minmss); } /* * rmx information is now retrieved from tcp_hostcache. */ tcp_hc_get(&inp->inp_inc, &metrics); if (metricptr != NULL) bcopy(&metrics, metricptr, sizeof(struct hc_metrics_lite)); /* * If there's a discovered mtu in tcp hostcache, use it. * Else, use the link mtu. */ if (metrics.rmx_mtu) mss = min(metrics.rmx_mtu, maxmtu) - min_protoh; else { #ifdef INET6 if (isipv6) { mss = maxmtu - min_protoh; if (!V_path_mtu_discovery && !in6_localaddr(&inp->in6p_faddr)) mss = min(mss, V_tcp_v6mssdflt); } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { mss = maxmtu - min_protoh; if (!V_path_mtu_discovery && !in_localaddr(inp->inp_faddr)) mss = min(mss, V_tcp_mssdflt); } #endif /* * XXX - The above conditional (mss = maxmtu - min_protoh) * probably violates the TCP spec. * The problem is that, since we don't know the * other end's MSS, we are supposed to use a conservative * default. But, if we do that, then MTU discovery will * never actually take place, because the conservative * default is much less than the MTUs typically seen * on the Internet today. For the moment, we'll sweep * this under the carpet. * * The conservative default might not actually be a problem * if the only case this occurs is when sending an initial * SYN with options and data to a host we've never talked * to before. Then, they will reply with an MSS value which * will get recorded and the new parameters should get * recomputed. For Further Study. */ } mss = min(mss, offer); /* * Sanity check: make sure that maxseg will be large * enough to allow some data on segments even if the * all the option space is used (40bytes). Otherwise * funny things may happen in tcp_output. * * XXXGL: shouldn't we reserve space for IP/IPv6 options? */ mss = max(mss, 64); tp->t_maxseg = mss; } void tcp_mss(struct tcpcb *tp, int offer) { int mss; uint32_t bufsize; struct inpcb *inp; struct socket *so; struct hc_metrics_lite metrics; struct tcp_ifcap cap; KASSERT(tp != NULL, ("%s: tp == NULL", __func__)); bzero(&cap, sizeof(cap)); tcp_mss_update(tp, offer, -1, &metrics, &cap); mss = tp->t_maxseg; inp = tp->t_inpcb; /* * If there's a pipesize, change the socket buffer to that size, * don't change if sb_hiwat is different than default (then it * has been changed on purpose with setsockopt). * Make the socket buffers an integral number of mss units; * if the mss is larger than the socket buffer, decrease the mss. */ so = inp->inp_socket; SOCKBUF_LOCK(&so->so_snd); if ((so->so_snd.sb_hiwat == V_tcp_sendspace) && metrics.rmx_sendpipe) bufsize = metrics.rmx_sendpipe; else bufsize = so->so_snd.sb_hiwat; if (bufsize < mss) mss = bufsize; else { bufsize = roundup(bufsize, mss); if (bufsize > sb_max) bufsize = sb_max; if (bufsize > so->so_snd.sb_hiwat) (void)sbreserve_locked(&so->so_snd, bufsize, so, NULL); } SOCKBUF_UNLOCK(&so->so_snd); /* * Sanity check: make sure that maxseg will be large * enough to allow some data on segments even if the * all the option space is used (40bytes). Otherwise * funny things may happen in tcp_output. * * XXXGL: shouldn't we reserve space for IP/IPv6 options? */ tp->t_maxseg = max(mss, 64); SOCKBUF_LOCK(&so->so_rcv); if ((so->so_rcv.sb_hiwat == V_tcp_recvspace) && metrics.rmx_recvpipe) bufsize = metrics.rmx_recvpipe; else bufsize = so->so_rcv.sb_hiwat; if (bufsize > mss) { bufsize = roundup(bufsize, mss); if (bufsize > sb_max) bufsize = sb_max; if (bufsize > so->so_rcv.sb_hiwat) (void)sbreserve_locked(&so->so_rcv, bufsize, so, NULL); } SOCKBUF_UNLOCK(&so->so_rcv); /* Check the interface for TSO capabilities. */ if (cap.ifcap & CSUM_TSO) { tp->t_flags |= TF_TSO; tp->t_tsomax = cap.tsomax; tp->t_tsomaxsegcount = cap.tsomaxsegcount; tp->t_tsomaxsegsize = cap.tsomaxsegsize; } } /* * Determine the MSS option to send on an outgoing SYN. */ int tcp_mssopt(struct in_conninfo *inc) { int mss = 0; uint32_t thcmtu = 0; uint32_t maxmtu = 0; size_t min_protoh; KASSERT(inc != NULL, ("tcp_mssopt with NULL in_conninfo pointer")); #ifdef INET6 if (inc->inc_flags & INC_ISIPV6) { mss = V_tcp_v6mssdflt; maxmtu = tcp_maxmtu6(inc, NULL); min_protoh = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); } #endif #if defined(INET) && defined(INET6) else #endif #ifdef INET { mss = V_tcp_mssdflt; maxmtu = tcp_maxmtu(inc, NULL); min_protoh = sizeof(struct tcpiphdr); } #endif #if defined(INET6) || defined(INET) thcmtu = tcp_hc_getmtu(inc); /* IPv4 and IPv6 */ #endif if (maxmtu && thcmtu) mss = min(maxmtu, thcmtu) - min_protoh; else if (maxmtu || thcmtu) mss = max(maxmtu, thcmtu) - min_protoh; return (mss); } /* * On a partial ack arrives, force the retransmission of the * next unacknowledged segment. Do not clear tp->t_dupacks. * By setting snd_nxt to ti_ack, this forces retransmission timer to * be started again. */ void tcp_newreno_partial_ack(struct tcpcb *tp, struct tcphdr *th) { tcp_seq onxt = tp->snd_nxt; uint32_t ocwnd = tp->snd_cwnd; u_int maxseg = tcp_maxseg(tp); INP_WLOCK_ASSERT(tp->t_inpcb); tcp_timer_activate(tp, TT_REXMT, 0); tp->t_rtttime = 0; tp->snd_nxt = th->th_ack; /* * Set snd_cwnd to one segment beyond acknowledged offset. * (tp->snd_una has not yet been updated when this function is called.) */ tp->snd_cwnd = maxseg + BYTES_THIS_ACK(tp, th); tp->t_flags |= TF_ACKNOW; (void) tp->t_fb->tfb_tcp_output(tp); tp->snd_cwnd = ocwnd; if (SEQ_GT(onxt, tp->snd_nxt)) tp->snd_nxt = onxt; /* * Partial window deflation. Relies on fact that tp->snd_una * not updated yet. */ if (tp->snd_cwnd > BYTES_THIS_ACK(tp, th)) tp->snd_cwnd -= BYTES_THIS_ACK(tp, th); else tp->snd_cwnd = 0; tp->snd_cwnd += maxseg; } int tcp_compute_pipe(struct tcpcb *tp) { return (tp->snd_max - tp->snd_una + tp->sackhint.sack_bytes_rexmit - tp->sackhint.sacked_bytes); } Index: projects/runtime-coverage/sys/netinet/tcp_stacks/fastpath.c =================================================================== --- projects/runtime-coverage/sys/netinet/tcp_stacks/fastpath.c (revision 321584) +++ projects/runtime-coverage/sys/netinet/tcp_stacks/fastpath.c (revision 321585) @@ -1,2461 +1,2449 @@ /*- * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995 * The Regents of the University of California. All rights reserved. * Copyright (c) 2007-2008,2010 * Swinburne University of Technology, Melbourne, Australia. * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2010 The FreeBSD Foundation * Copyright (c) 2010-2011 Juniper Networks, Inc. * Copyright (c) 2015 Netflix Inc. * All rights reserved. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by Lawrence Stewart, * James Healy and David Hayes, made possible in part by a grant from the Cisco * University Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by David Hayes under sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by Robert N. M. Watson under * contract to Juniper Networks, Inc. * * Portions of this software were developed by Randall R. Stewart while * working for Netflix Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)tcp_input.c 8.12 (Berkeley) 5/24/95 */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_tcpdebug.h" #include #include #include #include #include #ifdef TCP_HHOOK #include #endif #include #include #include /* for proc0 declaration */ #include #include #include #include #include #include #include #include #include /* before tcp_seq.h, for tcp_random18() */ #include #include #include #define TCPSTATES /* for logging */ #include #include #include #include #include #include /* required for icmp_var.h */ #include /* for ICMP_BANDLIM */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef TCPDEBUG #include #endif /* TCPDEBUG */ #ifdef TCP_OFFLOAD #include #endif #include #include VNET_DECLARE(int, tcp_autorcvbuf_inc); #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) VNET_DECLARE(int, tcp_autorcvbuf_max); #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) VNET_DECLARE(int, tcp_do_rfc3042); #define V_tcp_do_rfc3042 VNET(tcp_do_rfc3042) VNET_DECLARE(int, tcp_do_autorcvbuf); #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) VNET_DECLARE(int, tcp_insecure_rst); #define V_tcp_insecure_rst VNET(tcp_insecure_rst) VNET_DECLARE(int, tcp_insecure_syn); #define V_tcp_insecure_syn VNET(tcp_insecure_syn) VNET_DECLARE(int, drop_synfin); #define V_drop_synfin VNET(drop_synfin) static void tcp_do_segment_fastslow(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int, int, uint8_t, int); static void tcp_do_segment_fastack(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int, int, uint8_t, int); /* * Indicate whether this ack should be delayed. We can delay the ack if * following conditions are met: * - There is no delayed ack timer in progress. * - Our last ack wasn't a 0-sized window. We never want to delay * the ack that opens up a 0-sized window. * - LRO wasn't used for this segment. We make sure by checking that the * segment size is not larger than the MSS. */ #define DELAY_ACK(tp, tlen) \ ((!tcp_timer_active(tp, TT_DELACK) && \ (tp->t_flags & TF_RXWIN0SENT) == 0) && \ (tlen <= tp->t_maxseg) && \ (V_tcp_delack_enabled || (tp->t_flags & TF_NEEDSYN))) /* * So how is this faster than the normal fast ack? * It basically allows us to also stay in the fastpath * when a window-update ack also arrives. In testing * we saw only 25-30% of connections doing fastpath * due to the fact that along with moving forward * in sequence the window was also updated. */ static void tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, int ti_locked, uint32_t tiwin) { int acked; uint16_t nsegs; int winup_only=0; nsegs = max(1, m->m_pkthdr.lro_nsegs); #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif /* * The following if statement will be true if * we are doing the win_up_in_fp * - We have more new data (SEQ_LT(tp->snd_wl1, th->th_seq)) * - No more new data, but we have an ack for new data * (tp->snd_wl1 == th->th_seq && SEQ_LT(tp->snd_wl2, th->th_ack)) * - No more new data, the same ack point but the window grew * (tp->snd_wl1 == th->th_seq && tp->snd_wl2 == th->th_ack && twin > tp->snd_wnd) */ if ((SEQ_LT(tp->snd_wl1, th->th_seq) || (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { /* keep track of pure window updates */ if (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) { winup_only = 1; TCPSTAT_INC(tcps_rcvwinupd); } tp->snd_wnd = tiwin; tp->snd_wl1 = th->th_seq; tp->snd_wl2 = th->th_ack; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; } /* * If last ACK falls within this segment's sequence numbers, * record the timestamp. * NOTE that the test is modified according to the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). */ if ((to->to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to->to_tsval; } /* * This is a pure ack for outstanding data. */ if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; TCPSTAT_INC(tcps_predack); /* * "bad retransmit" recovery. */ if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) { cc_cong_signal(tp, th, CC_RTO_ERR); } /* * Recalculate the transmit timer / rtt. * * Some boxes send broken timestamp replies * during the SYN+ACK phase, ignore * timestamps of 0 or we could calculate a * huge RTT and blow up the retransmit timer. */ if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) { uint32_t t; t = tcp_ts_getticks() - to->to_tsecr; if (!tp->t_rttlow || tp->t_rttlow > t) tp->t_rttlow = t; tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) tp->t_rttlow = ticks - tp->t_rtttime; tcp_xmit_timer(tp, ticks - tp->t_rtttime); } if (winup_only == 0) { acked = BYTES_THIS_ACK(tp, th); #ifdef TCP_HHOOK /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ hhook_run_tcp_est_in(tp, th, to); #endif TCPSTAT_ADD(tcps_rcvackbyte, acked); sbdrop(&so->so_snd, acked); if (SEQ_GT(tp->snd_una, tp->snd_recover) && SEQ_LEQ(th->th_ack, tp->snd_recover)) tp->snd_recover = th->th_ack - 1; /* * Let the congestion control algorithm update * congestion control related information. This * typically means increasing the congestion * window. */ cc_ack_received(tp, th, nsegs, CC_ACK); tp->snd_una = th->th_ack; /* * Pull snd_wl2 up to prevent seq wrap relative * to th_ack. */ tp->snd_wl2 = th->th_ack; tp->t_dupacks = 0; /* * If all outstanding data are acked, stop * retransmit timer, otherwise restart timer * using current (possibly backed-off) value. * If process is waiting for space, * wakeup/selwakeup/signal. If data * are ready to send, let tcp_output * decide between more output or persist. */ #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); m_freem(m); if (tp->snd_una == tp->snd_max) tcp_timer_activate(tp, TT_REXMT, 0); else if (!tcp_timer_active(tp, TT_PERSIST)) tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); } else { /* * Window update only, just free the mbufs and * send out whatever we can. */ m_freem(m); } sowwakeup(so); if (sbavail(&so->so_snd)) (void) tcp_output(tp); KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { tp->t_flags &= ~TF_DELACK; tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } INP_WUNLOCK(tp->t_inpcb); } /* * Here nothing is really faster, its just that we * have broken out the fast-data path also just like * the fast-ack. */ static void tcp_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, int ti_locked, uint32_t tiwin) { int newsize = 0; /* automatic sockbuf scaling */ #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif /* * If last ACK falls within this segment's sequence numbers, * record the timestamp. * NOTE that the test is modified according to the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). */ if ((to->to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to->to_tsval; } /* * This is a pure, in-sequence data packet with * nothing on the reassembly queue and we have enough * buffer space to take it. */ if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; /* Clean receiver SACK report if present */ if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks) tcp_clean_sackreport(tp); TCPSTAT_INC(tcps_preddat); tp->rcv_nxt += tlen; /* * Pull snd_wl1 up to prevent seq wrap relative to * th_seq. */ tp->snd_wl1 = th->th_seq; /* * Pull rcv_up up to prevent seq wrap relative to * rcv_nxt. */ tp->rcv_up = tp->rcv_nxt; TCPSTAT_ADD(tcps_rcvbyte, tlen); #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); newsize = tcp_autorcvbuf(m, th, so, tp, tlen); /* Add data to socket buffer. */ SOCKBUF_LOCK(&so->so_rcv); if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { m_freem(m); } else { /* * Set new socket buffer size. * Give up when limit is reached. */ if (newsize) if (!sbreserve_locked(&so->so_rcv, newsize, so, NULL)) so->so_rcv.sb_flags &= ~SB_AUTOSIZE; m_adj(m, drop_hdrlen); /* delayed header drop */ sbappendstream_locked(&so->so_rcv, m, 0); } /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); if (DELAY_ACK(tp, tlen)) { tp->t_flags |= TF_DELACK; } else { tp->t_flags |= TF_ACKNOW; tcp_output(tp); } KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { tp->t_flags &= ~TF_DELACK; tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } INP_WUNLOCK(tp->t_inpcb); } /* * The slow-path is the clone of the long long part * of tcp_do_segment past all the fast-path stuff. We * use it here by two different callers, the fast/slow and * the fastack only. */ static void tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, int ti_locked, uint32_t tiwin, int thflags) { int acked, ourfinisacked, needoutput = 0; int rstreason, todrop, win; uint16_t nsegs; char *s; struct in_conninfo *inc; struct mbuf *mfree = NULL; nsegs = max(1, m->m_pkthdr.lro_nsegs); #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif /* * Calculate amount of space in receive window, * and then do TCP input processing. * Receive window is amount of space in rcv queue, * but not less than advertised window. */ inc = &tp->t_inpcb->inp_inc; win = sbspace(&so->so_rcv); if (win < 0) win = 0; tp->rcv_wnd = imax(win, (int)(tp->rcv_adv - tp->rcv_nxt)); switch (tp->t_state) { /* * If the state is SYN_RECEIVED: * if seg contains an ACK, but not for our SYN/ACK, send a RST. */ case TCPS_SYN_RECEIVED: if ((thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->snd_una) || SEQ_GT(th->th_ack, tp->snd_max))) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } break; /* * If the state is SYN_SENT: * if seg contains a RST with valid ACK (SEQ.ACK has already * been verified), then drop the connection. * if seg contains a RST without an ACK, drop the seg. * if seg does not contain SYN, then drop the seg. * Otherwise this is an acceptable SYN segment * initialize tp->rcv_nxt and tp->irs * if seg contains ack then advance tp->snd_una * if seg contains an ECE and ECN support is enabled, the stream * is ECN capable. * if SYN has been acked change to ESTABLISHED else SYN_RCVD state * arrange for segment to be acked (eventually) * continue processing rest of data/controls, beginning with URG */ case TCPS_SYN_SENT: if ((thflags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) { TCP_PROBE5(connect__refused, NULL, tp, m, tp, th); tp = tcp_drop(tp, ECONNREFUSED); } if (thflags & TH_RST) goto drop; if (!(thflags & TH_SYN)) goto drop; tp->irs = th->th_seq; tcp_rcvseqinit(tp); if (thflags & TH_ACK) { TCPSTAT_INC(tcps_connects); soisconnected(so); #ifdef MAC mac_socketpeer_set_from_mbuf(m, so); #endif /* Do window scaling on this connection? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; } tp->rcv_adv += min(tp->rcv_wnd, TCP_MAXWIN << tp->rcv_scale); tp->snd_una++; /* SYN is acked */ /* * If there's data, delay ACK; if there's also a FIN * ACKNOW will be turned on later. */ if (DELAY_ACK(tp, tlen) && tlen != 0) tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); else tp->t_flags |= TF_ACKNOW; if ((thflags & TH_ECE) && V_tcp_do_ecn) { tp->t_flags |= TF_ECN_PERMIT; TCPSTAT_INC(tcps_ecn_shs); } /* * Received in SYN_SENT[*] state. * Transitions: * SYN_SENT --> ESTABLISHED * SYN_SENT* --> FIN_WAIT_1 */ tp->t_starttime = ticks; if (tp->t_flags & TF_NEEDFIN) { tcp_state_change(tp, TCPS_FIN_WAIT_1); tp->t_flags &= ~TF_NEEDFIN; thflags &= ~TH_SYN; } else { tcp_state_change(tp, TCPS_ESTABLISHED); TCP_PROBE5(connect__established, NULL, tp, m, tp, th); cc_conn_init(tp); tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); } } else { /* * Received initial SYN in SYN-SENT[*] state => * simultaneous open. * If it succeeds, connection is * half-synchronized. * Otherwise, do 3-way handshake: * SYN-SENT -> SYN-RECEIVED * SYN-SENT* -> SYN-RECEIVED* */ tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN); tcp_timer_activate(tp, TT_REXMT, 0); tcp_state_change(tp, TCPS_SYN_RECEIVED); } KASSERT(ti_locked == TI_RLOCKED, ("%s: trimthenstep6: " "ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); /* * Advance th->th_seq to correspond to first data byte. * If data, trim to stay within window, * dropping FIN if necessary. */ th->th_seq++; if (tlen > tp->rcv_wnd) { todrop = tlen - tp->rcv_wnd; m_adj(m, -todrop); tlen = tp->rcv_wnd; thflags &= ~TH_FIN; TCPSTAT_INC(tcps_rcvpackafterwin); TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); } tp->snd_wl1 = th->th_seq - 1; tp->rcv_up = th->th_seq; /* * Client side of transaction: already sent SYN and data. * If the remote host used T/TCP to validate the SYN, * our data will be ACK'd; if so, enter normal data segment * processing in the middle of step 5, ack processing. * Otherwise, goto step 6. */ if (thflags & TH_ACK) goto process_ACK; goto step6; /* * If the state is LAST_ACK or CLOSING or TIME_WAIT: * do normal processing. * * NB: Leftover from RFC1644 T/TCP. Cases to be reused later. */ case TCPS_LAST_ACK: case TCPS_CLOSING: break; /* continue normal processing */ } /* * States other than LISTEN or SYN_SENT. * First check the RST flag and sequence number since reset segments * are exempt from the timestamp and connection count tests. This * fixes a bug introduced by the Stevens, vol. 2, p. 960 bugfix * below which allowed reset segments in half the sequence space * to fall though and be processed (which gives forged reset * segments with a random sequence number a 50 percent chance of * killing a connection). * Then check timestamp, if present. * Then check the connection count, if present. * Then check that at least some bytes of segment are within * receive window. If segment begins before rcv_nxt, * drop leading data (and SYN); if nothing left, just ack. */ if (thflags & TH_RST) { /* * RFC5961 Section 3.2 * * - RST drops connection only if SEG.SEQ == RCV.NXT. * - If RST is in window, we send challenge ACK. * * Note: to take into account delayed ACKs, we should * test against last_ack_sent instead of rcv_nxt. * Note 2: we handle special case of closed window, not * covered by the RFC. */ if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) && SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) || (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); KASSERT(ti_locked == TI_RLOCKED, ("%s: TH_RST ti_locked %d, th %p tp %p", __func__, ti_locked, th, tp)); KASSERT(tp->t_state != TCPS_SYN_SENT, ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p", __func__, th, tp)); if (V_tcp_insecure_rst || tp->last_ack_sent == th->th_seq) { TCPSTAT_INC(tcps_drops); /* Drop the connection. */ switch (tp->t_state) { case TCPS_SYN_RECEIVED: so->so_error = ECONNREFUSED; goto close; case TCPS_ESTABLISHED: case TCPS_FIN_WAIT_1: case TCPS_FIN_WAIT_2: case TCPS_CLOSE_WAIT: case TCPS_CLOSING: case TCPS_LAST_ACK: so->so_error = ECONNRESET; close: /* FALLTHROUGH */ default: tp = tcp_close(tp); } } else { TCPSTAT_INC(tcps_badrst); /* Send challenge ACK. */ tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, tp->snd_nxt, TH_ACK); tp->last_ack_sent = tp->rcv_nxt; m = NULL; } } goto drop; } /* * RFC5961 Section 4.2 * Send challenge ACK for any SYN in synchronized state. */ if ((thflags & TH_SYN) && tp->t_state != TCPS_SYN_SENT) { KASSERT(ti_locked == TI_RLOCKED, ("tcp_do_segment: TH_SYN ti_locked %d", ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); TCPSTAT_INC(tcps_badsyn); if (V_tcp_insecure_syn && SEQ_GEQ(th->th_seq, tp->last_ack_sent) && SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) { tp = tcp_drop(tp, ECONNRESET); rstreason = BANDLIM_UNLIMITED; } else { /* Send challenge ACK. */ tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, tp->snd_nxt, TH_ACK); tp->last_ack_sent = tp->rcv_nxt; m = NULL; } goto drop; } /* * RFC 1323 PAWS: If we have a timestamp reply on this segment * and it's less than ts_recent, drop it. */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { /* Check to see if ts_recent is over 24 days old. */ if (tcp_ts_getticks() - tp->ts_recent_age > TCP_PAWS_IDLE) { /* * Invalidate ts_recent. If this segment updates * ts_recent, the age will be reset later and ts_recent * will get a valid value. If it does not, setting * ts_recent to zero will at least satisfy the * requirement that zero be placed in the timestamp * echo reply when ts_recent isn't valid. The * age isn't reset until we get a valid ts_recent * because we don't want out-of-order segments to be * dropped when ts_recent is old. */ tp->ts_recent = 0; } else { TCPSTAT_INC(tcps_rcvduppack); TCPSTAT_ADD(tcps_rcvdupbyte, tlen); TCPSTAT_INC(tcps_pawsdrop); if (tlen) goto dropafterack; goto drop; } } /* * In the SYN-RECEIVED state, validate that the packet belongs to * this connection before trimming the data to fit the receive * window. Check the sequence number versus IRS since we know * the sequence numbers haven't wrapped. This is a partial fix * for the "LAND" DoS attack. */ if (tp->t_state == TCPS_SYN_RECEIVED && SEQ_LT(th->th_seq, tp->irs)) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } todrop = tp->rcv_nxt - th->th_seq; if (todrop > 0) { if (thflags & TH_SYN) { thflags &= ~TH_SYN; th->th_seq++; if (th->th_urp > 1) th->th_urp--; else thflags &= ~TH_URG; todrop--; } /* * Following if statement from Stevens, vol. 2, p. 960. */ if (todrop > tlen || (todrop == tlen && (thflags & TH_FIN) == 0)) { /* * Any valid FIN must be to the left of the window. * At this point the FIN must be a duplicate or out * of sequence; drop it. */ thflags &= ~TH_FIN; /* * Send an ACK to resynchronize and drop any data. * But keep on processing for RST or ACK. */ tp->t_flags |= TF_ACKNOW; todrop = tlen; TCPSTAT_INC(tcps_rcvduppack); TCPSTAT_ADD(tcps_rcvdupbyte, todrop); } else { TCPSTAT_INC(tcps_rcvpartduppack); TCPSTAT_ADD(tcps_rcvpartdupbyte, todrop); } drop_hdrlen += todrop; /* drop from the top afterwards */ th->th_seq += todrop; tlen -= todrop; if (th->th_urp > todrop) th->th_urp -= todrop; else { thflags &= ~TH_URG; th->th_urp = 0; } } /* * If new data are received on a connection after the * user processes are gone, then RST the other end. */ if ((so->so_state & SS_NOFDREF) && tp->t_state > TCPS_CLOSE_WAIT && tlen) { KASSERT(ti_locked == TI_RLOCKED, ("%s: SS_NOFDEREF && " "CLOSE_WAIT && tlen ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: %s: Received %d bytes of data " "after socket was closed, " "sending RST and removing tcpcb\n", s, __func__, tcpstates[tp->t_state], tlen); free(s, M_TCPLOG); } tp = tcp_close(tp); TCPSTAT_INC(tcps_rcvafterclose); rstreason = BANDLIM_UNLIMITED; goto dropwithreset; } /* * If segment ends after window, drop trailing data * (and PUSH and FIN); if nothing left, just ACK. */ todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd); if (todrop > 0) { TCPSTAT_INC(tcps_rcvpackafterwin); if (todrop >= tlen) { TCPSTAT_ADD(tcps_rcvbyteafterwin, tlen); /* * If window is closed can only take segments at * window edge, and have to drop data and PUSH from * incoming segments. Continue processing, but * remember to ack. Otherwise, drop segment * and ack. */ if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) { tp->t_flags |= TF_ACKNOW; TCPSTAT_INC(tcps_rcvwinprobe); } else goto dropafterack; } else TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); m_adj(m, -todrop); tlen -= todrop; thflags &= ~(TH_PUSH|TH_FIN); } /* * If last ACK falls within this segment's sequence numbers, * record its timestamp. * NOTE: * 1) That the test incorporates suggestions from the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). * 2) That updating only on newer timestamps interferes with * our earlier PAWS tests, so this check should be solely * predicated on the sequence space of this segment. * 3) That we modify the segment boundary check to be * Last.ACK.Sent <= SEG.SEQ + SEG.Len * instead of RFC1323's * Last.ACK.Sent < SEG.SEQ + SEG.Len, * This modified check allows us to overcome RFC1323's * limitations as described in Stevens TCP/IP Illustrated * Vol. 2 p.869. In such cases, we can still calculate the * RTT correctly when RCV.NXT == Last.ACK.Sent. */ if ((to->to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent) && SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + ((thflags & (TH_SYN|TH_FIN)) != 0))) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to->to_tsval; } /* * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN * flag is on (half-synchronized state), then queue data for * later processing; else drop segment and return. */ if ((thflags & TH_ACK) == 0) { if (tp->t_state == TCPS_SYN_RECEIVED || (tp->t_flags & TF_NEEDSYN)) goto step6; else if (tp->t_flags & TF_ACKNOW) goto dropafterack; else goto drop; } /* * Ack processing. */ switch (tp->t_state) { /* * In SYN_RECEIVED state, the ack ACKs our SYN, so enter * ESTABLISHED state and continue processing. * The ACK was checked above. */ case TCPS_SYN_RECEIVED: TCPSTAT_INC(tcps_connects); soisconnected(so); /* Do window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; tp->snd_wnd = tiwin; } /* * Make transitions: * SYN-RECEIVED -> ESTABLISHED * SYN-RECEIVED* -> FIN-WAIT-1 */ tp->t_starttime = ticks; if (tp->t_flags & TF_NEEDFIN) { tcp_state_change(tp, TCPS_FIN_WAIT_1); tp->t_flags &= ~TF_NEEDFIN; } else { tcp_state_change(tp, TCPS_ESTABLISHED); TCP_PROBE5(accept__established, NULL, tp, m, tp, th); cc_conn_init(tp); tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); } /* * If segment contains data or ACK, will call tcp_reass() * later; if not, do so now to pass queued data to user. */ if (tlen == 0 && (thflags & TH_FIN) == 0) (void) tcp_reass(tp, (struct tcphdr *)0, 0, (struct mbuf *)0); tp->snd_wl1 = th->th_seq - 1; /* FALLTHROUGH */ /* * In ESTABLISHED state: drop duplicate ACKs; ACK out of range * ACKs. If the ack is in the range * tp->snd_una < th->th_ack <= tp->snd_max * then advance tp->snd_una to th->th_ack and drop * data from the retransmission queue. If this ACK reflects * more up to date window information we update our window information. */ case TCPS_ESTABLISHED: case TCPS_FIN_WAIT_1: case TCPS_FIN_WAIT_2: case TCPS_CLOSE_WAIT: case TCPS_CLOSING: case TCPS_LAST_ACK: if (SEQ_GT(th->th_ack, tp->snd_max)) { TCPSTAT_INC(tcps_rcvacktoomuch); goto dropafterack; } if ((tp->t_flags & TF_SACK_PERMIT) && ((to->to_flags & TOF_SACK) || !TAILQ_EMPTY(&tp->snd_holes))) tcp_sack_doack(tp, to, th->th_ack); else /* * Reset the value so that previous (valid) value * from the last ack with SACK doesn't get used. */ tp->sackhint.sacked_bytes = 0; #ifdef TCP_HHOOK /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ hhook_run_tcp_est_in(tp, th, to); #endif if (SEQ_LEQ(th->th_ack, tp->snd_una)) { if (tlen == 0 && tiwin == tp->snd_wnd) { /* * If this is the first time we've seen a * FIN from the remote, this is not a * duplicate and it needs to be processed * normally. This happens during a * simultaneous close. */ if ((thflags & TH_FIN) && (TCPS_HAVERCVDFIN(tp->t_state) == 0)) { tp->t_dupacks = 0; break; } TCPSTAT_INC(tcps_rcvdupack); /* * If we have outstanding data (other than * a window probe), this is a completely * duplicate ack (ie, window info didn't * change and FIN isn't set), * the ack is the biggest we've * seen and we've seen exactly our rexmt * threshold of them, assume a packet * has been dropped and retransmit it. * Kludge snd_nxt & the congestion * window so we send only this one * packet. * * We know we're losing at the current * window size so do congestion avoidance * (set ssthresh to half the current window * and pull our congestion window back to * the new ssthresh). * * Dup acks mean that packets have left the * network (they're now cached at the receiver) * so bump cwnd by the amount in the receiver * to keep a constant cwnd packets in the * network. * * When using TCP ECN, notify the peer that * we reduced the cwnd. */ if (!tcp_timer_active(tp, TT_REXMT) || th->th_ack != tp->snd_una) tp->t_dupacks = 0; else if (++tp->t_dupacks > tcprexmtthresh || IN_FASTRECOVERY(tp->t_flags)) { cc_ack_received(tp, th, nsegs, CC_DUPACK); if ((tp->t_flags & TF_SACK_PERMIT) && IN_FASTRECOVERY(tp->t_flags)) { int awnd; /* * Compute the amount of data in flight first. * We can inject new data into the pipe iff * we have less than 1/2 the original window's * worth of data in flight. */ if (V_tcp_do_rfc6675_pipe) awnd = tcp_compute_pipe(tp); else awnd = (tp->snd_nxt - tp->snd_fack) + tp->sackhint.sack_bytes_rexmit; if (awnd < tp->snd_ssthresh) { tp->snd_cwnd += tp->t_maxseg; - /* - * RFC5681 Section 3.2 talks about cwnd - * inflation on additional dupacks and - * deflation on recovering from loss. - * - * We keep cwnd into check so that - * we don't have to 'deflate' it when we - * get out of recovery. - */ if (tp->snd_cwnd > tp->snd_ssthresh) tp->snd_cwnd = tp->snd_ssthresh; } } else tp->snd_cwnd += tp->t_maxseg; (void) tp->t_fb->tfb_tcp_output(tp); goto drop; } else if (tp->t_dupacks == tcprexmtthresh) { tcp_seq onxt = tp->snd_nxt; /* * If we're doing sack, check to * see if we're already in sack * recovery. If we're not doing sack, * check to see if we're in newreno * recovery. */ if (tp->t_flags & TF_SACK_PERMIT) { if (IN_FASTRECOVERY(tp->t_flags)) { tp->t_dupacks = 0; break; } } else { if (SEQ_LEQ(th->th_ack, tp->snd_recover)) { tp->t_dupacks = 0; break; } } /* Congestion signal before ack. */ cc_cong_signal(tp, th, CC_NDUPACK); cc_ack_received(tp, th, nsegs, CC_DUPACK); tcp_timer_activate(tp, TT_REXMT, 0); tp->t_rtttime = 0; if (tp->t_flags & TF_SACK_PERMIT) { TCPSTAT_INC( tcps_sack_recovery_episode); tp->sack_newdata = tp->snd_nxt; - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = tp->t_maxseg; + tp->snd_cwnd = tp->t_maxseg; (void) tp->t_fb->tfb_tcp_output(tp); goto drop; } tp->snd_nxt = th->th_ack; - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = tp->t_maxseg; + tp->snd_cwnd = tp->t_maxseg; (void) tp->t_fb->tfb_tcp_output(tp); KASSERT(tp->snd_limited <= 2, ("%s: tp->snd_limited too big", __func__)); - if (CC_ALGO(tp)->cong_signal == NULL) - tp->snd_cwnd = tp->snd_ssthresh + - tp->t_maxseg * - (tp->t_dupacks - tp->snd_limited); + tp->snd_cwnd = tp->snd_ssthresh + + tp->t_maxseg * + (tp->t_dupacks - tp->snd_limited); if (SEQ_GT(onxt, tp->snd_nxt)) tp->snd_nxt = onxt; goto drop; } else if (V_tcp_do_rfc3042) { /* * Process first and second duplicate * ACKs. Each indicates a segment * leaving the network, creating room * for more. Make sure we can send a * packet on reception of each duplicate * ACK by increasing snd_cwnd by one * segment. Restore the original * snd_cwnd after packet transmission. */ cc_ack_received(tp, th, nsegs, CC_DUPACK); uint32_t oldcwnd = tp->snd_cwnd; tcp_seq oldsndmax = tp->snd_max; u_int sent; int avail; KASSERT(tp->t_dupacks == 1 || tp->t_dupacks == 2, ("%s: dupacks not 1 or 2", __func__)); if (tp->t_dupacks == 1) tp->snd_limited = 0; tp->snd_cwnd = (tp->snd_nxt - tp->snd_una) + (tp->t_dupacks - tp->snd_limited) * tp->t_maxseg; /* * Only call tcp_output when there * is new data available to be sent. * Otherwise we would send pure ACKs. */ SOCKBUF_LOCK(&so->so_snd); avail = sbavail(&so->so_snd) - (tp->snd_nxt - tp->snd_una); SOCKBUF_UNLOCK(&so->so_snd); if (avail > 0) (void) tp->t_fb->tfb_tcp_output(tp); sent = tp->snd_max - oldsndmax; if (sent > tp->t_maxseg) { KASSERT((tp->t_dupacks == 2 && tp->snd_limited == 0) || (sent == tp->t_maxseg + 1 && tp->t_flags & TF_SENTFIN), ("%s: sent too much", __func__)); tp->snd_limited = 2; } else if (sent > 0) ++tp->snd_limited; tp->snd_cwnd = oldcwnd; goto drop; } } else tp->t_dupacks = 0; break; } KASSERT(SEQ_GT(th->th_ack, tp->snd_una), ("%s: th_ack <= snd_una", __func__)); /* * If the congestion window was inflated to account * for the other side's cached packets, retract it. */ if (IN_FASTRECOVERY(tp->t_flags)) { if (SEQ_LT(th->th_ack, tp->snd_recover)) { if (tp->t_flags & TF_SACK_PERMIT) tcp_sack_partialack(tp, th); else tcp_newreno_partial_ack(tp, th); } else cc_post_recovery(tp, th); } tp->t_dupacks = 0; /* * If we reach this point, ACK is not a duplicate, * i.e., it ACKs something we sent. */ if (tp->t_flags & TF_NEEDSYN) { /* * T/TCP: Connection was half-synchronized, and our * SYN has been ACK'd (so connection is now fully * synchronized). Go to non-starred state, * increment snd_una for ACK of SYN, and check if * we can do window scaling. */ tp->t_flags &= ~TF_NEEDSYN; tp->snd_una++; /* Do window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == (TF_RCVD_SCALE|TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; /* Send window already scaled. */ } } process_ACK: INP_WLOCK_ASSERT(tp->t_inpcb); acked = BYTES_THIS_ACK(tp, th); TCPSTAT_INC(tcps_rcvackpack); TCPSTAT_ADD(tcps_rcvackbyte, acked); /* * If we just performed our first retransmit, and the ACK * arrives within our recovery window, then it was a mistake * to do the retransmit in the first place. Recover our * original cwnd and ssthresh, and proceed to transmit where * we left off. */ if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) cc_cong_signal(tp, th, CC_RTO_ERR); /* * If we have a timestamp reply, update smoothed * round trip time. If no timestamp is present but * transmit timer is running and timed sequence * number was acked, update smoothed round trip time. * Since we now have an rtt measurement, cancel the * timer backoff (cf., Phil Karn's retransmit alg.). * Recompute the initial retransmit timer. * * Some boxes send broken timestamp replies * during the SYN+ACK phase, ignore * timestamps of 0 or we could calculate a * huge RTT and blow up the retransmit timer. */ if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) { uint32_t t; t = tcp_ts_getticks() - to->to_tsecr; if (!tp->t_rttlow || tp->t_rttlow > t) tp->t_rttlow = t; tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) tp->t_rttlow = ticks - tp->t_rtttime; tcp_xmit_timer(tp, ticks - tp->t_rtttime); } /* * If all outstanding data is acked, stop retransmit * timer and remember to restart (more output or persist). * If there is more data to be acked, restart retransmit * timer, using current (possibly backed-off) value. */ if (th->th_ack == tp->snd_max) { tcp_timer_activate(tp, TT_REXMT, 0); needoutput = 1; } else if (!tcp_timer_active(tp, TT_PERSIST)) tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); /* * If no data (only SYN) was ACK'd, * skip rest of ACK processing. */ if (acked == 0) goto step6; /* * Let the congestion control algorithm update congestion * control related information. This typically means increasing * the congestion window. */ cc_ack_received(tp, th, nsegs, CC_ACK); SOCKBUF_LOCK(&so->so_snd); if (acked > sbavail(&so->so_snd)) { tp->snd_wnd -= sbavail(&so->so_snd); mfree = sbcut_locked(&so->so_snd, (int)sbavail(&so->so_snd)); ourfinisacked = 1; } else { mfree = sbcut_locked(&so->so_snd, acked); tp->snd_wnd -= acked; ourfinisacked = 0; } /* NB: sowwakeup_locked() does an implicit unlock. */ sowwakeup_locked(so); m_freem(mfree); /* Detect una wraparound. */ if (!IN_RECOVERY(tp->t_flags) && SEQ_GT(tp->snd_una, tp->snd_recover) && SEQ_LEQ(th->th_ack, tp->snd_recover)) tp->snd_recover = th->th_ack - 1; /* XXXLAS: Can this be moved up into cc_post_recovery? */ if (IN_RECOVERY(tp->t_flags) && SEQ_GEQ(th->th_ack, tp->snd_recover)) { EXIT_RECOVERY(tp->t_flags); } tp->snd_una = th->th_ack; if (tp->t_flags & TF_SACK_PERMIT) { if (SEQ_GT(tp->snd_una, tp->snd_recover)) tp->snd_recover = tp->snd_una; } if (SEQ_LT(tp->snd_nxt, tp->snd_una)) tp->snd_nxt = tp->snd_una; switch (tp->t_state) { /* * In FIN_WAIT_1 STATE in addition to the processing * for the ESTABLISHED state if our FIN is now acknowledged * then enter FIN_WAIT_2. */ case TCPS_FIN_WAIT_1: if (ourfinisacked) { /* * If we can't receive any more * data, then closing user can proceed. * Starting the timer is contrary to the * specification, but if we don't get a FIN * we'll hang forever. * * XXXjl: * we should release the tp also, and use a * compressed state. */ if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { soisdisconnected(so); tcp_timer_activate(tp, TT_2MSL, (tcp_fast_finwait2_recycle ? tcp_finwait2_timeout : TP_MAXIDLE(tp))); } tcp_state_change(tp, TCPS_FIN_WAIT_2); } break; /* * In CLOSING STATE in addition to the processing for * the ESTABLISHED state if the ACK acknowledges our FIN * then enter the TIME-WAIT state, otherwise ignore * the segment. */ case TCPS_CLOSING: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tcp_twstart(tp); INP_INFO_RUNLOCK(&V_tcbinfo); m_freem(m); return; } break; /* * In LAST_ACK, we may still be waiting for data to drain * and/or to be acked, as well as for the ack of our FIN. * If our FIN is now acknowledged, delete the TCB, * enter the closed state and return. */ case TCPS_LAST_ACK: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tp = tcp_close(tp); goto drop; } break; } } step6: INP_WLOCK_ASSERT(tp->t_inpcb); /* * Update window information. * Don't look at window if no ACK: TAC's send garbage on first SYN. */ if ((thflags & TH_ACK) && (SEQ_LT(tp->snd_wl1, th->th_seq) || (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { /* keep track of pure window updates */ if (tlen == 0 && tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) TCPSTAT_INC(tcps_rcvwinupd); tp->snd_wnd = tiwin; tp->snd_wl1 = th->th_seq; tp->snd_wl2 = th->th_ack; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; needoutput = 1; } /* * Process segments with URG. */ if ((thflags & TH_URG) && th->th_urp && TCPS_HAVERCVDFIN(tp->t_state) == 0) { /* * This is a kludge, but if we receive and accept * random urgent pointers, we'll crash in * soreceive. It's hard to imagine someone * actually wanting to send this much urgent data. */ SOCKBUF_LOCK(&so->so_rcv); if (th->th_urp + sbavail(&so->so_rcv) > sb_max) { th->th_urp = 0; /* XXX */ thflags &= ~TH_URG; /* XXX */ SOCKBUF_UNLOCK(&so->so_rcv); /* XXX */ goto dodata; /* XXX */ } /* * If this segment advances the known urgent pointer, * then mark the data stream. This should not happen * in CLOSE_WAIT, CLOSING, LAST_ACK or TIME_WAIT STATES since * a FIN has been received from the remote side. * In these states we ignore the URG. * * According to RFC961 (Assigned Protocols), * the urgent pointer points to the last octet * of urgent data. We continue, however, * to consider it to indicate the first octet * of data past the urgent section as the original * spec states (in one of two places). */ if (SEQ_GT(th->th_seq+th->th_urp, tp->rcv_up)) { tp->rcv_up = th->th_seq + th->th_urp; so->so_oobmark = sbavail(&so->so_rcv) + (tp->rcv_up - tp->rcv_nxt) - 1; if (so->so_oobmark == 0) so->so_rcv.sb_state |= SBS_RCVATMARK; sohasoutofband(so); tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA); } SOCKBUF_UNLOCK(&so->so_rcv); /* * Remove out of band data so doesn't get presented to user. * This can happen independent of advancing the URG pointer, * but if two URG's are pending at once, some out-of-band * data may creep in... ick. */ if (th->th_urp <= (uint32_t)tlen && !(so->so_options & SO_OOBINLINE)) { /* hdr drop is delayed */ tcp_pulloutofband(so, th, m, drop_hdrlen); } } else { /* * If no out of band data is expected, * pull receive urgent pointer along * with the receive window. */ if (SEQ_GT(tp->rcv_nxt, tp->rcv_up)) tp->rcv_up = tp->rcv_nxt; } dodata: /* XXX */ INP_WLOCK_ASSERT(tp->t_inpcb); /* * Process the segment text, merging it into the TCP sequencing queue, * and arranging for acknowledgment of receipt if necessary. * This process logically involves adjusting tp->rcv_wnd as data * is presented to the user (this happens in tcp_usrreq.c, * case PRU_RCVD). If a FIN has already been received on this * connection then we just ignore the text. */ if ((tlen || (thflags & TH_FIN)) && TCPS_HAVERCVDFIN(tp->t_state) == 0) { tcp_seq save_start = th->th_seq; m_adj(m, drop_hdrlen); /* delayed header drop */ /* * Insert segment which includes th into TCP reassembly queue * with control block tp. Set thflags to whether reassembly now * includes a segment with FIN. This handles the common case * inline (segment is the next to be received on an established * connection, and the queue is empty), avoiding linkage into * and removal from the queue and repetition of various * conversions. * Set DELACK for segments received in order, but ack * immediately when segments are out of order (so * fast retransmit can work). */ if (th->th_seq == tp->rcv_nxt && LIST_EMPTY(&tp->t_segq) && TCPS_HAVEESTABLISHED(tp->t_state)) { if (DELAY_ACK(tp, tlen)) tp->t_flags |= TF_DELACK; else tp->t_flags |= TF_ACKNOW; tp->rcv_nxt += tlen; thflags = th->th_flags & TH_FIN; TCPSTAT_INC(tcps_rcvpack); TCPSTAT_ADD(tcps_rcvbyte, tlen); SOCKBUF_LOCK(&so->so_rcv); if (so->so_rcv.sb_state & SBS_CANTRCVMORE) m_freem(m); else sbappendstream_locked(&so->so_rcv, m, 0); /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); } else { /* * XXX: Due to the header drop above "th" is * theoretically invalid by now. Fortunately * m_adj() doesn't actually frees any mbufs * when trimming from the head. */ thflags = tcp_reass(tp, th, &tlen, m); tp->t_flags |= TF_ACKNOW; } if (tlen > 0 && (tp->t_flags & TF_SACK_PERMIT)) tcp_update_sack_list(tp, save_start, save_start + tlen); #if 0 /* * Note the amount of data that peer has sent into * our window, in order to estimate the sender's * buffer size. * XXX: Unused. */ if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) len = so->so_rcv.sb_hiwat - (tp->rcv_adv - tp->rcv_nxt); else len = so->so_rcv.sb_hiwat; #endif } else { m_freem(m); thflags &= ~TH_FIN; } /* * If FIN is received ACK the FIN and let the user know * that the connection is closing. */ if (thflags & TH_FIN) { if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { socantrcvmore(so); /* * If connection is half-synchronized * (ie NEEDSYN flag on) then delay ACK, * so it may be piggybacked when SYN is sent. * Otherwise, since we received a FIN then no * more input can be expected, send ACK now. */ if (tp->t_flags & TF_NEEDSYN) tp->t_flags |= TF_DELACK; else tp->t_flags |= TF_ACKNOW; tp->rcv_nxt++; } switch (tp->t_state) { /* * In SYN_RECEIVED and ESTABLISHED STATES * enter the CLOSE_WAIT state. */ case TCPS_SYN_RECEIVED: tp->t_starttime = ticks; /* FALLTHROUGH */ case TCPS_ESTABLISHED: tcp_state_change(tp, TCPS_CLOSE_WAIT); break; /* * If still in FIN_WAIT_1 STATE FIN has not been acked so * enter the CLOSING state. */ case TCPS_FIN_WAIT_1: tcp_state_change(tp, TCPS_CLOSING); break; /* * In FIN_WAIT_2 state enter the TIME_WAIT state, * starting the time-wait timer, turning off the other * standard timers. */ case TCPS_FIN_WAIT_2: INP_INFO_RLOCK_ASSERT(&V_tcbinfo); KASSERT(ti_locked == TI_RLOCKED, ("%s: dodata " "TCP_FIN_WAIT_2 ti_locked: %d", __func__, ti_locked)); tcp_twstart(tp); INP_INFO_RUNLOCK(&V_tcbinfo); return; } } if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); /* * Return any desired output. */ if (needoutput || (tp->t_flags & TF_ACKNOW)) (void) tp->t_fb->tfb_tcp_output(tp); KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { tp->t_flags &= ~TF_DELACK; tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } INP_WUNLOCK(tp->t_inpcb); return; dropafterack: /* * Generate an ACK dropping incoming segment if it occupies * sequence space, where the ACK reflects our state. * * We can now skip the test for the RST flag since all * paths to this code happen after packets containing * RST have been dropped. * * In the SYN-RECEIVED state, don't send an ACK unless the * segment we received passes the SYN-RECEIVED ACK test. * If it fails send a RST. This breaks the loop in the * "LAND" DoS attack, and also prevents an ACK storm * between two listening ports that have been sent forged * SYN segments, each with the source address of the other. */ if (tp->t_state == TCPS_SYN_RECEIVED && (thflags & TH_ACK) && (SEQ_GT(tp->snd_una, th->th_ack) || SEQ_GT(th->th_ack, tp->snd_max)) ) { rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__drop, tp, th, m); if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; tp->t_flags |= TF_ACKNOW; (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; dropwithreset: if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; if (tp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(tp->t_inpcb); } else tcp_dropwithreset(m, th, NULL, tlen, rstreason); return; drop: if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS else INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); #endif /* * Drop space held by incoming segment and return. */ #ifdef TCPDEBUG if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__drop, tp, th, m); if (tp != NULL) INP_WUNLOCK(tp->t_inpcb); m_freem(m); } /* * Do fast slow is a combination of the original * tcp_dosegment and a split fastpath, one function * for the fast-ack which also includes allowing fastpath * for window advanced in sequence acks. And also a * sub-function that handles the insequence data. */ void tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, int ti_locked) { int thflags; uint32_t tiwin; char *s; uint16_t nsegs; int can_enter; struct in_conninfo *inc; struct tcpopt to; thflags = th->th_flags; inc = &tp->t_inpcb->inp_inc; nsegs = max(1, m->m_pkthdr.lro_nsegs); /* * If this is either a state-changing packet or current state isn't * established, we require a write lock on tcbinfo. Otherwise, we * allow the tcbinfo to be in either alocked or unlocked, as the * caller may have unnecessarily acquired a write lock due to a race. */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { #ifdef INVARIANTS if (ti_locked == TI_RLOCKED) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " "ti_locked: %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", __func__)); KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", __func__)); if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: " "SYN|FIN segment ignored (based on " "sysctl setting)\n", s, __func__); free(s, M_TCPLOG); } if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; } /* * If a segment with the ACK-bit set arrives in the SYN-SENT state * check SEQ.ACK first. */ if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { tcp_dropwithreset(m, th, tp, tlen, BANDLIM_UNLIMITED); if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } INP_WUNLOCK(tp->t_inpcb); return; } tp->sackhint.last_sack_ack = 0; /* * Segment received on connection. * Reset idle time and keep-alive timer. * XXX: This should be done after segment * validation to ignore broken/spoofed segs. */ tp->t_rcvtime = ticks; /* * Unscale the window into a 32-bit value. * For the SYN_SENT state the scale is zero. */ tiwin = th->th_win << tp->snd_scale; /* * TCP ECN processing. */ if (tp->t_flags & TF_ECN_PERMIT) { if (thflags & TH_CWR) tp->t_flags &= ~TF_ECN_SND_ECE; switch (iptos & IPTOS_ECN_MASK) { case IPTOS_ECN_CE: tp->t_flags |= TF_ECN_SND_ECE; TCPSTAT_INC(tcps_ecn_ce); break; case IPTOS_ECN_ECT0: TCPSTAT_INC(tcps_ecn_ect0); break; case IPTOS_ECN_ECT1: TCPSTAT_INC(tcps_ecn_ect1); break; } /* Congestion experienced. */ if (thflags & TH_ECE) { cc_cong_signal(tp, th, CC_ECN); } } /* * Parse options on any incoming segment. */ tcp_dooptions(&to, (u_char *)(th + 1), (th->th_off << 2) - sizeof(struct tcphdr), (thflags & TH_SYN) ? TO_SYN : 0); /* * If echoed timestamp is later than the current time, * fall back to non RFC1323 RTT calculation. Normalize * timestamp if syncookies were used when this connection * was established. */ if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) { to.to_tsecr -= tp->ts_offset; if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks())) to.to_tsecr = 0; } /* * If timestamps were negotiated during SYN/ACK they should * appear on every segment during this session and vice versa. */ if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp missing, " "no action\n", s, __func__); free(s, M_TCPLOG); } } if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp not expected, " "no action\n", s, __func__); free(s, M_TCPLOG); } } /* * Process options only when we get SYN/ACK back. The SYN case * for incoming connections is handled in tcp_syncache. * According to RFC1323 the window field in a SYN (i.e., a * or ) segment itself is never scaled. * XXX this is traditional behavior, may need to be cleaned up. */ if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { if ((to.to_flags & TOF_SCALE) && (tp->t_flags & TF_REQ_SCALE)) { tp->t_flags |= TF_RCVD_SCALE; tp->snd_scale = to.to_wscale; } /* * Initial send window. It will be updated with * the next incoming segment to the scaled value. */ tp->snd_wnd = th->th_win; if (to.to_flags & TOF_TS) { tp->t_flags |= TF_RCVD_TSTMP; tp->ts_recent = to.to_tsval; tp->ts_recent_age = tcp_ts_getticks(); } if (to.to_flags & TOF_MSS) tcp_mss(tp, to.to_mss); if ((tp->t_flags & TF_SACK_PERMIT) && (to.to_flags & TOF_SACKPERM) == 0) tp->t_flags &= ~TF_SACK_PERMIT; } can_enter = 0; if (__predict_true((tlen == 0))) { /* * The ack moved forward and we have a window (non-zero) * * The ack did not move forward, but the window increased. */ if (__predict_true((SEQ_GT(th->th_ack, tp->snd_una) && tiwin) || ((th->th_ack == tp->snd_una) && tiwin && (tiwin > tp->snd_wnd)))) { can_enter = 1; } } else { /* * Data incoming, use the old entry criteria * for fast-path with data. */ if ((tiwin && tiwin == tp->snd_wnd)) { can_enter = 1; } } /* * Header prediction: check for the two common cases * of a uni-directional data xfer. If the packet has * no control flags, is in-sequence, the window didn't * change and we're not retransmitting, it's a * candidate. If the length is zero and the ack moved * forward, we're the sender side of the xfer. Just * free the data acked & wake any higher level process * that was blocked waiting for space. If the length * is non-zero and the ack didn't move, we're the * receiver side. If we're getting packets in-order * (the reassembly queue is empty), add the data to * the socket buffer and note that we need a delayed ack. * Make sure that the hidden state-flags are also off. * Since we check for TCPS_ESTABLISHED first, it can only * be TH_NEEDSYN. */ if (__predict_true(tp->t_state == TCPS_ESTABLISHED && th->th_seq == tp->rcv_nxt && (thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK && tp->snd_nxt == tp->snd_max && can_enter && ((tp->t_flags & (TF_NEEDSYN|TF_NEEDFIN)) == 0) && LIST_EMPTY(&tp->t_segq) && ((to.to_flags & TOF_TS) == 0 || TSTMP_GEQ(to.to_tsval, tp->ts_recent)))) { if (__predict_true((tlen == 0) && (SEQ_LEQ(th->th_ack, tp->snd_max) && !IN_RECOVERY(tp->t_flags) && (to.to_flags & TOF_SACK) == 0 && TAILQ_EMPTY(&tp->snd_holes)))) { /* We are done */ tcp_do_fastack(m, th, so, tp, &to, drop_hdrlen, tlen, ti_locked, tiwin); return; } else if ((tlen) && (th->th_ack == tp->snd_una && tlen <= sbspace(&so->so_rcv))) { tcp_do_fastnewdata(m, th, so, tp, &to, drop_hdrlen, tlen, ti_locked, tiwin); /* We are done */ return; } } tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen, ti_locked, tiwin, thflags); } /* * This subfunction is used to try to highly optimize the * fast path. We again allow window updates that are * in sequence to remain in the fast-path. We also add * in the __predict's to attempt to help the compiler. * Note that if we return a 0, then we can *not* process * it and the caller should push the packet into the * slow-path. */ static int tcp_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, int ti_locked, uint32_t tiwin) { int acked; uint16_t nsegs; int winup_only=0; nsegs = max(1, m->m_pkthdr.lro_nsegs); #ifdef TCPDEBUG /* * The size of tcp_saveipgen must be the size of the max ip header, * now IPv6. */ u_char tcp_saveipgen[IP6_HDR_LEN]; struct tcphdr tcp_savetcp; short ostate = 0; #endif if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { /* Old ack, behind (or duplicate to) the last one rcv'd */ return (0); } if (__predict_false(th->th_ack == tp->snd_una) && __predict_false(tiwin <= tp->snd_wnd)) { /* duplicate ack a shrinking dup ack with shrinking window */ return (0); } if (__predict_false(tiwin == 0)) { /* zero window */ return (0); } if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) { /* Above what we have sent? */ return (0); } if (__predict_false(tp->snd_nxt != tp->snd_max)) { /* We are retransmitting */ return (0); } if (__predict_false(tp->t_flags & (TF_NEEDSYN|TF_NEEDFIN))) { /* We need a SYN or a FIN, unlikely.. */ return (0); } if((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) { /* Timestamp is behind .. old ack with seq wrap? */ return (0); } if (__predict_false(IN_RECOVERY(tp->t_flags))) { /* Still recovering */ return (0); } if (__predict_false(to->to_flags & TOF_SACK)) { /* Sack included in the ack.. */ return (0); } if (!TAILQ_EMPTY(&tp->snd_holes)) { /* We have sack holes on our scoreboard */ return (0); } /* Ok if we reach here, we can process a fast-ack */ /* Did the window get updated? */ if (tiwin != tp->snd_wnd) { /* keep track of pure window updates */ if (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) { winup_only = 1; TCPSTAT_INC(tcps_rcvwinupd); } tp->snd_wnd = tiwin; tp->snd_wl1 = th->th_seq; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; } /* * Pull snd_wl2 up to prevent seq wrap relative * to th_ack. */ tp->snd_wl2 = th->th_ack; /* * If last ACK falls within this segment's sequence numbers, * record the timestamp. * NOTE that the test is modified according to the latest * proposal of the tcplw@cray.com list (Braden 1993/04/26). */ if ((to->to_flags & TOF_TS) != 0 && SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { tp->ts_recent_age = tcp_ts_getticks(); tp->ts_recent = to->to_tsval; } /* * This is a pure ack for outstanding data. */ if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } ti_locked = TI_UNLOCKED; TCPSTAT_INC(tcps_predack); /* * "bad retransmit" recovery. */ if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) { cc_cong_signal(tp, th, CC_RTO_ERR); } /* * Recalculate the transmit timer / rtt. * * Some boxes send broken timestamp replies * during the SYN+ACK phase, ignore * timestamps of 0 or we could calculate a * huge RTT and blow up the retransmit timer. */ if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) { uint32_t t; t = tcp_ts_getticks() - to->to_tsecr; if (!tp->t_rttlow || tp->t_rttlow > t) tp->t_rttlow = t; tcp_xmit_timer(tp, TCP_TS_TO_TICKS(t) + 1); } else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) { if (!tp->t_rttlow || tp->t_rttlow > ticks - tp->t_rtttime) tp->t_rttlow = ticks - tp->t_rtttime; tcp_xmit_timer(tp, ticks - tp->t_rtttime); } if (winup_only == 0) { acked = BYTES_THIS_ACK(tp, th); #ifdef TCP_HHOOK /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ hhook_run_tcp_est_in(tp, th, to); #endif TCPSTAT_ADD(tcps_rcvackbyte, acked); sbdrop(&so->so_snd, acked); if (SEQ_GT(tp->snd_una, tp->snd_recover) && SEQ_LEQ(th->th_ack, tp->snd_recover)) tp->snd_recover = th->th_ack - 1; /* * Let the congestion control algorithm update * congestion control related information. This * typically means increasing the congestion * window. */ cc_ack_received(tp, th, nsegs, CC_ACK); tp->snd_una = th->th_ack; tp->t_dupacks = 0; /* * If all outstanding data are acked, stop * retransmit timer, otherwise restart timer * using current (possibly backed-off) value. * If process is waiting for space, * wakeup/selwakeup/signal. If data * are ready to send, let tcp_output * decide between more output or persist. */ #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); m_freem(m); if (tp->snd_una == tp->snd_max) tcp_timer_activate(tp, TT_REXMT, 0); else if (!tcp_timer_active(tp, TT_PERSIST)) tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); /* Wake up the socket if we have room to write more */ sowwakeup(so); } else { /* * Window update only, just free the mbufs and * send out whatever we can. */ m_freem(m); } if (sbavail(&so->so_snd)) (void) tcp_output(tp); KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { tp->t_flags &= ~TF_DELACK; tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); } INP_WUNLOCK(tp->t_inpcb); return (1); } /* * This tcp-do-segment concentrates on making the fastest * ack processing path. It does not have a fast-path for * data (it possibly could which would then eliminate the * need for fast-slow above). For a content distributor having * large outgoing elephants and very very little coming in * having no fastpath for data does not really help (since you * don't get much data in). The most important thing is * processing ack's quickly and getting the rest of the data * output to the peer as quickly as possible. This routine * seems to be about an overall 3% faster then the old * tcp_do_segment and keeps us in the fast-path for packets * much more (by allowing window updates to also stay in the fastpath). */ void tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, int ti_locked) { int thflags; uint32_t tiwin; char *s; struct in_conninfo *inc; struct tcpopt to; thflags = th->th_flags; inc = &tp->t_inpcb->inp_inc; /* * If this is either a state-changing packet or current state isn't * established, we require a write lock on tcbinfo. Otherwise, we * allow the tcbinfo to be in either alocked or unlocked, as the * caller may have unnecessarily acquired a write lock due to a race. */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { #ifdef INVARIANTS if (ti_locked == TI_RLOCKED) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); } else { KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " "ti_locked: %d", __func__, ti_locked)); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); } #endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", __func__)); KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", __func__)); if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: " "SYN|FIN segment ignored (based on " "sysctl setting)\n", s, __func__); free(s, M_TCPLOG); } if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; } /* * If a segment with the ACK-bit set arrives in the SYN-SENT state * check SEQ.ACK first. */ if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { tcp_dropwithreset(m, th, tp, tlen, BANDLIM_UNLIMITED); if (ti_locked == TI_RLOCKED) { INP_INFO_RUNLOCK(&V_tcbinfo); } INP_WUNLOCK(tp->t_inpcb); return; } tp->sackhint.last_sack_ack = 0; /* * Segment received on connection. * Reset idle time and keep-alive timer. * XXX: This should be done after segment * validation to ignore broken/spoofed segs. */ tp->t_rcvtime = ticks; /* * Unscale the window into a 32-bit value. * For the SYN_SENT state the scale is zero. */ tiwin = th->th_win << tp->snd_scale; /* * TCP ECN processing. */ if (tp->t_flags & TF_ECN_PERMIT) { if (thflags & TH_CWR) tp->t_flags &= ~TF_ECN_SND_ECE; switch (iptos & IPTOS_ECN_MASK) { case IPTOS_ECN_CE: tp->t_flags |= TF_ECN_SND_ECE; TCPSTAT_INC(tcps_ecn_ce); break; case IPTOS_ECN_ECT0: TCPSTAT_INC(tcps_ecn_ect0); break; case IPTOS_ECN_ECT1: TCPSTAT_INC(tcps_ecn_ect1); break; } /* Congestion experienced. */ if (thflags & TH_ECE) { cc_cong_signal(tp, th, CC_ECN); } } /* * Parse options on any incoming segment. */ tcp_dooptions(&to, (u_char *)(th + 1), (th->th_off << 2) - sizeof(struct tcphdr), (thflags & TH_SYN) ? TO_SYN : 0); /* * If echoed timestamp is later than the current time, * fall back to non RFC1323 RTT calculation. Normalize * timestamp if syncookies were used when this connection * was established. */ if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) { to.to_tsecr -= tp->ts_offset; if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks())) to.to_tsecr = 0; } /* * If timestamps were negotiated during SYN/ACK they should * appear on every segment during this session and vice versa. */ if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp missing, " "no action\n", s, __func__); free(s, M_TCPLOG); } } if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { log(LOG_DEBUG, "%s; %s: Timestamp not expected, " "no action\n", s, __func__); free(s, M_TCPLOG); } } /* * Process options only when we get SYN/ACK back. The SYN case * for incoming connections is handled in tcp_syncache. * According to RFC1323 the window field in a SYN (i.e., a * or ) segment itself is never scaled. * XXX this is traditional behavior, may need to be cleaned up. */ if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { if ((to.to_flags & TOF_SCALE) && (tp->t_flags & TF_REQ_SCALE)) { tp->t_flags |= TF_RCVD_SCALE; tp->snd_scale = to.to_wscale; } /* * Initial send window. It will be updated with * the next incoming segment to the scaled value. */ tp->snd_wnd = th->th_win; if (to.to_flags & TOF_TS) { tp->t_flags |= TF_RCVD_TSTMP; tp->ts_recent = to.to_tsval; tp->ts_recent_age = tcp_ts_getticks(); } if (to.to_flags & TOF_MSS) tcp_mss(tp, to.to_mss); if ((tp->t_flags & TF_SACK_PERMIT) && (to.to_flags & TOF_SACKPERM) == 0) tp->t_flags &= ~TF_SACK_PERMIT; } /* * Header prediction: check for the two common cases * of a uni-directional data xfer. If the packet has * no control flags, is in-sequence, the window didn't * change and we're not retransmitting, it's a * candidate. If the length is zero and the ack moved * forward, we're the sender side of the xfer. Just * free the data acked & wake any higher level process * that was blocked waiting for space. If the length * is non-zero and the ack didn't move, we're the * receiver side. If we're getting packets in-order * (the reassembly queue is empty), add the data to * the socket buffer and note that we need a delayed ack. * Make sure that the hidden state-flags are also off. * Since we check for TCPS_ESTABLISHED first, it can only * be TH_NEEDSYN. */ if (__predict_true(tp->t_state == TCPS_ESTABLISHED) && __predict_true(((to.to_flags & TOF_SACK) == 0)) && __predict_true(tlen == 0) && __predict_true((thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK) && __predict_true(LIST_EMPTY(&tp->t_segq)) && __predict_true(th->th_seq == tp->rcv_nxt)) { if (tcp_fastack(m, th, so, tp, &to, drop_hdrlen, tlen, ti_locked, tiwin)) { return; } } tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen, ti_locked, tiwin, thflags); } struct tcp_function_block __tcp_fastslow = { .tfb_tcp_block_name = "fastslow", .tfb_tcp_output = tcp_output, .tfb_tcp_do_segment = tcp_do_segment_fastslow, .tfb_tcp_ctloutput = tcp_default_ctloutput, }; struct tcp_function_block __tcp_fastack = { .tfb_tcp_block_name = "fastack", .tfb_tcp_output = tcp_output, .tfb_tcp_do_segment = tcp_do_segment_fastack, .tfb_tcp_ctloutput = tcp_default_ctloutput }; static int tcp_addfastpaths(module_t mod, int type, void *data) { int err=0; switch (type) { case MOD_LOAD: err = register_tcp_functions(&__tcp_fastack, M_WAITOK); if (err) { printf("Failed to register fastack module -- err:%d\n", err); return(err); } err = register_tcp_functions(&__tcp_fastslow, M_WAITOK); if (err) { printf("Failed to register fastslow module -- err:%d\n", err); deregister_tcp_functions(&__tcp_fastack); return(err); } break; case MOD_QUIESCE: if ((__tcp_fastslow.tfb_refcnt) ||( __tcp_fastack.tfb_refcnt)) { return(EBUSY); } break; case MOD_UNLOAD: err = deregister_tcp_functions(&__tcp_fastack); if (err == EBUSY) break; err = deregister_tcp_functions(&__tcp_fastslow); if (err == EBUSY) break; err = 0; break; default: return (EOPNOTSUPP); } return (err); } static moduledata_t new_tcp_fastpaths = { .name = "tcp_fastpaths", .evhand = tcp_addfastpaths, .priv = 0 }; MODULE_VERSION(kern_tcpfastpaths, 1); DECLARE_MODULE(kern_tcpfastpaths, new_tcp_fastpaths, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY); Index: projects/runtime-coverage/sys/sys/pctrie.h =================================================================== --- projects/runtime-coverage/sys/sys/pctrie.h (revision 321584) +++ projects/runtime-coverage/sys/sys/pctrie.h (revision 321585) @@ -1,137 +1,137 @@ /* * Copyright (c) 2013 EMC Corp. * Copyright (c) 2011 Jeffrey Roberson * Copyright (c) 2008 Mayur Shardul * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_PCTRIE_H_ #define _SYS_PCTRIE_H_ #include #ifdef _KERNEL #define PCTRIE_DEFINE(name, type, field, allocfn, freefn) \ \ CTASSERT(sizeof(((struct type *)0)->field) == sizeof(uint64_t)); \ /* \ * XXX This assert protects flag bits, it does not enforce natural \ * alignment. 32bit architectures do not naturally align 64bit fields. \ */ \ CTASSERT((__offsetof(struct type, field) & (sizeof(uint32_t) - 1)) == 0); \ \ static __inline struct type * \ name##_PCTRIE_VAL2PTR(uint64_t *val) \ { \ \ if (val == NULL) \ return (NULL); \ return (struct type *) \ ((uintptr_t)val - __offsetof(struct type, field)); \ } \ \ static __inline uint64_t * \ name##_PCTRIE_PTR2VAL(struct type *ptr) \ { \ \ return &ptr->field; \ } \ \ static __inline int \ name##_PCTRIE_INSERT(struct pctrie *ptree, struct type *ptr) \ { \ \ return pctrie_insert(ptree, name##_PCTRIE_PTR2VAL(ptr), \ allocfn); \ } \ \ static __inline struct type * \ name##_PCTRIE_LOOKUP(struct pctrie *ptree, uint64_t key) \ { \ \ return name##_PCTRIE_VAL2PTR(pctrie_lookup(ptree, key)); \ } \ \ -static __inline struct type * \ +static __inline __unused struct type * \ name##_PCTRIE_LOOKUP_LE(struct pctrie *ptree, uint64_t key) \ { \ \ return name##_PCTRIE_VAL2PTR(pctrie_lookup_le(ptree, key)); \ } \ \ static __inline __unused struct type * \ name##_PCTRIE_LOOKUP_GE(struct pctrie *ptree, uint64_t key) \ { \ \ return name##_PCTRIE_VAL2PTR(pctrie_lookup_ge(ptree, key)); \ } \ \ static __inline __unused void \ name##_PCTRIE_RECLAIM(struct pctrie *ptree) \ { \ \ pctrie_reclaim_allnodes(ptree, freefn); \ } \ \ static __inline void \ name##_PCTRIE_REMOVE(struct pctrie *ptree, uint64_t key) \ { \ \ pctrie_remove(ptree, key, freefn); \ } typedef void *(*pctrie_alloc_t)(struct pctrie *ptree); typedef void (*pctrie_free_t)(struct pctrie *ptree, void *node); int pctrie_insert(struct pctrie *ptree, uint64_t *val, pctrie_alloc_t allocfn); uint64_t *pctrie_lookup(struct pctrie *ptree, uint64_t key); uint64_t *pctrie_lookup_ge(struct pctrie *ptree, uint64_t key); uint64_t *pctrie_lookup_le(struct pctrie *ptree, uint64_t key); void pctrie_reclaim_allnodes(struct pctrie *ptree, pctrie_free_t freefn); void pctrie_remove(struct pctrie *ptree, uint64_t key, pctrie_free_t freefn); size_t pctrie_node_size(void); int pctrie_zone_init(void *mem, int size, int flags); static __inline void pctrie_init(struct pctrie *ptree) { ptree->pt_root = 0; } static __inline boolean_t pctrie_is_empty(struct pctrie *ptree) { return (ptree->pt_root == 0); } #endif /* _KERNEL */ #endif /* !_SYS_PCTRIE_H_ */ Index: projects/runtime-coverage/sys/vm/vnode_pager.c =================================================================== --- projects/runtime-coverage/sys/vm/vnode_pager.c (revision 321584) +++ projects/runtime-coverage/sys/vm/vnode_pager.c (revision 321585) @@ -1,1418 +1,1460 @@ /*- * Copyright (c) 1990 University of Utah. * Copyright (c) 1991 The Regents of the University of California. * All rights reserved. * Copyright (c) 1993, 1994 John S. Dyson * Copyright (c) 1995, David Greenman * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 */ /* * Page to/from files (vnodes). */ /* * TODO: * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will * greatly re-simplify the vnode_pager. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, daddr_t *rtaddress, int *run); static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); static int vnode_pager_input_old(vm_object_t object, vm_page_t m); static void vnode_pager_dealloc(vm_object_t); static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int *, int *); static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, int *, vop_getpages_iodone_t, void *); static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, int, int *); static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *cred); static int vnode_pager_generic_getpages_done(struct buf *); static void vnode_pager_generic_getpages_done_async(struct buf *); struct pagerops vnodepagerops = { .pgo_alloc = vnode_pager_alloc, .pgo_dealloc = vnode_pager_dealloc, .pgo_getpages = vnode_pager_getpages, .pgo_getpages_async = vnode_pager_getpages_async, .pgo_putpages = vnode_pager_putpages, .pgo_haspage = vnode_pager_haspage, }; int vnode_pbuf_freecnt; int vnode_async_pbuf_freecnt; /* Create the VM system backing object for this vnode */ int vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td) { vm_object_t object; vm_ooffset_t size = isize; struct vattr va; if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE) return (0); while ((object = vp->v_object) != NULL) { VM_OBJECT_WLOCK(object); if (!(object->flags & OBJ_DEAD)) { VM_OBJECT_WUNLOCK(object); return (0); } VOP_UNLOCK(vp, 0); vm_object_set_flag(object, OBJ_DISCONNECTWNT); VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } if (size == 0) { if (vn_isdisk(vp, NULL)) { size = IDX_TO_OFF(INT_MAX); } else { if (VOP_GETATTR(vp, &va, td->td_ucred)) return (0); size = va.va_size; } } object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred); /* * Dereference the reference we just created. This assumes * that the object is associated with the vp. */ VM_OBJECT_WLOCK(object); object->ref_count--; VM_OBJECT_WUNLOCK(object); vrele(vp); KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); return (0); } void vnode_destroy_vobject(struct vnode *vp) { struct vm_object *obj; obj = vp->v_object; if (obj == NULL) return; ASSERT_VOP_ELOCKED(vp, "vnode_destroy_vobject"); VM_OBJECT_WLOCK(obj); umtx_shm_object_terminated(obj); if (obj->ref_count == 0) { /* * don't double-terminate the object */ if ((obj->flags & OBJ_DEAD) == 0) { vm_object_terminate(obj); } else { /* * Waiters were already handled during object * termination. The exclusive vnode lock hopefully * prevented new waiters from referencing the dying * object. */ KASSERT((obj->flags & OBJ_DISCONNECTWNT) == 0, ("OBJ_DISCONNECTWNT set obj %p flags %x", obj, obj->flags)); vp->v_object = NULL; VM_OBJECT_WUNLOCK(obj); } } else { /* * Woe to the process that tries to page now :-). */ vm_pager_deallocate(obj); VM_OBJECT_WUNLOCK(obj); } KASSERT(vp->v_object == NULL, ("vp %p obj %p", vp, vp->v_object)); } /* * Allocate (or lookup) pager for a vnode. * Handle is a vnode pointer. * * MPSAFE */ vm_object_t vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t offset, struct ucred *cred) { vm_object_t object; struct vnode *vp; /* * Pageout to vnode, no can do yet. */ if (handle == NULL) return (NULL); vp = (struct vnode *) handle; /* * If the object is being terminated, wait for it to * go away. */ retry: while ((object = vp->v_object) != NULL) { VM_OBJECT_WLOCK(object); if ((object->flags & OBJ_DEAD) == 0) break; vm_object_set_flag(object, OBJ_DISCONNECTWNT); VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0); } KASSERT(vp->v_usecount != 0, ("vnode_pager_alloc: no vnode reference")); if (object == NULL) { /* * Add an object of the appropriate size */ object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); object->un_pager.vnp.vnp_size = size; object->un_pager.vnp.writemappings = 0; object->handle = handle; VI_LOCK(vp); if (vp->v_object != NULL) { /* * Object has been created while we were sleeping */ VI_UNLOCK(vp); VM_OBJECT_WLOCK(object); KASSERT(object->ref_count == 1, ("leaked ref %p %d", object, object->ref_count)); object->type = OBJT_DEAD; object->ref_count = 0; VM_OBJECT_WUNLOCK(object); vm_object_destroy(object); goto retry; } vp->v_object = object; VI_UNLOCK(vp); } else { object->ref_count++; #if VM_NRESERVLEVEL > 0 vm_object_color(object, 0); #endif VM_OBJECT_WUNLOCK(object); } vrefact(vp); return (object); } /* * The object must be locked. */ static void vnode_pager_dealloc(vm_object_t object) { struct vnode *vp; int refs; vp = object->handle; if (vp == NULL) panic("vnode_pager_dealloc: pager already dealloced"); VM_OBJECT_ASSERT_WLOCKED(object); vm_object_pip_wait(object, "vnpdea"); refs = object->ref_count; object->handle = NULL; object->type = OBJT_DEAD; if (object->flags & OBJ_DISCONNECTWNT) { vm_object_clear_flag(object, OBJ_DISCONNECTWNT); wakeup(object); } ASSERT_VOP_ELOCKED(vp, "vnode_pager_dealloc"); if (object->un_pager.vnp.writemappings > 0) { object->un_pager.vnp.writemappings = 0; VOP_ADD_WRITECOUNT(vp, -1); CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", __func__, vp, vp->v_writecount); } vp->v_object = NULL; VOP_UNSET_TEXT(vp); VM_OBJECT_WUNLOCK(object); while (refs-- > 0) vunref(vp); VM_OBJECT_WLOCK(object); } static boolean_t vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { struct vnode *vp = object->handle; daddr_t bn; int err; daddr_t reqblock; int poff; int bsize; int pagesperblock, blocksperpage; VM_OBJECT_ASSERT_WLOCKED(object); /* * If no vp or vp is doomed or marked transparent to VM, we do not * have the page. */ if (vp == NULL || vp->v_iflag & VI_DOOMED) return FALSE; /* * If the offset is beyond end of file we do * not have the page. */ if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size) return FALSE; bsize = vp->v_mount->mnt_stat.f_iosize; pagesperblock = bsize / PAGE_SIZE; blocksperpage = 0; if (pagesperblock > 0) { reqblock = pindex / pagesperblock; } else { blocksperpage = (PAGE_SIZE / bsize); reqblock = pindex * blocksperpage; } VM_OBJECT_WUNLOCK(object); err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); VM_OBJECT_WLOCK(object); if (err) return TRUE; if (bn == -1) return FALSE; if (pagesperblock > 0) { poff = pindex - (reqblock * pagesperblock); if (before) { *before *= pagesperblock; *before += poff; } if (after) { /* * The BMAP vop can report a partial block in the * 'after', but must not report blocks after EOF. * Assert the latter, and truncate 'after' in case * of the former. */ KASSERT((reqblock + *after) * pagesperblock < roundup2(object->size, pagesperblock), ("%s: reqblock %jd after %d size %ju", __func__, (intmax_t )reqblock, *after, (uintmax_t )object->size)); *after *= pagesperblock; *after += pagesperblock - (poff + 1); if (pindex + *after >= object->size) *after = object->size - 1 - pindex; } } else { if (before) { *before /= blocksperpage; } if (after) { *after /= blocksperpage; } } return TRUE; } /* * Lets the VM system know about a change in size for a file. * We adjust our own internal size and flush any cached pages in * the associated object that are affected by the size change. * * Note: this routine may be invoked as a result of a pager put * operation (possibly at object termination time), so we must be careful. */ void vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) { vm_object_t object; vm_page_t m; vm_pindex_t nobjsize; if ((object = vp->v_object) == NULL) return; /* ASSERT_VOP_ELOCKED(vp, "vnode_pager_setsize and not locked vnode"); */ VM_OBJECT_WLOCK(object); if (object->type == OBJT_DEAD) { VM_OBJECT_WUNLOCK(object); return; } KASSERT(object->type == OBJT_VNODE, ("not vnode-backed object %p", object)); if (nsize == object->un_pager.vnp.vnp_size) { /* * Hasn't changed size */ VM_OBJECT_WUNLOCK(object); return; } nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); if (nsize < object->un_pager.vnp.vnp_size) { /* * File has shrunk. Toss any cached pages beyond the new EOF. */ if (nobjsize < object->size) vm_object_page_remove(object, nobjsize, object->size, 0); /* * this gets rid of garbage at the end of a page that is now * only partially backed by the vnode. * * XXX for some reason (I don't know yet), if we take a * completely invalid page and mark it partially valid * it can screw up NFS reads, so we don't allow the case. */ if ((nsize & PAGE_MASK) && (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && m->valid != 0) { int base = (int)nsize & PAGE_MASK; int size = PAGE_SIZE - base; /* * Clear out partial-page garbage in case * the page has been mapped. */ pmap_zero_page_area(m, base, size); /* * Update the valid bits to reflect the blocks that * have been zeroed. Some of these valid bits may * have already been set. */ vm_page_set_valid_range(m, base, size); /* * Round "base" to the next block boundary so that the * dirty bit for a partially zeroed block is not * cleared. */ base = roundup2(base, DEV_BSIZE); /* * Clear out partial-page dirty bits. * * note that we do not clear out the valid * bits. This would prevent bogus_page * replacement from working properly. */ vm_page_clear_dirty(m, base, PAGE_SIZE - base); } } object->un_pager.vnp.vnp_size = nsize; object->size = nobjsize; VM_OBJECT_WUNLOCK(object); } /* * calculate the linear (byte) disk address of specified virtual * file address */ static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, daddr_t *rtaddress, int *run) { int bsize; int err; daddr_t vblock; daddr_t voffset; if (address < 0) return -1; if (vp->v_iflag & VI_DOOMED) return -1; bsize = vp->v_mount->mnt_stat.f_iosize; vblock = address / bsize; voffset = address % bsize; err = VOP_BMAP(vp, vblock, NULL, rtaddress, run, NULL); if (err == 0) { if (*rtaddress != -1) *rtaddress += voffset / DEV_BSIZE; if (run) { *run += 1; *run *= bsize/PAGE_SIZE; *run -= voffset/PAGE_SIZE; } } return (err); } /* * small block filesystem vnode pager input */ static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m) { struct vnode *vp; struct bufobj *bo; struct buf *bp; struct sf_buf *sf; daddr_t fileaddr; vm_offset_t bsize; vm_page_bits_t bits; int error, i; error = 0; vp = object->handle; if (vp->v_iflag & VI_DOOMED) return VM_PAGER_BAD; bsize = vp->v_mount->mnt_stat.f_iosize; VOP_BMAP(vp, 0, &bo, 0, NULL, NULL); sf = sf_buf_alloc(m, 0); for (i = 0; i < PAGE_SIZE / bsize; i++) { vm_ooffset_t address; bits = vm_page_bits(i * bsize, bsize); if (m->valid & bits) continue; address = IDX_TO_OFF(m->pindex) + i * bsize; if (address >= object->un_pager.vnp.vnp_size) { fileaddr = -1; } else { error = vnode_pager_addr(vp, address, &fileaddr, NULL); if (error) break; } if (fileaddr != -1) { bp = getpbuf(&vnode_pbuf_freecnt); /* build a minimal buffer header */ bp->b_iocmd = BIO_READ; bp->b_iodone = bdone; KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); bp->b_rcred = crhold(curthread->td_ucred); bp->b_wcred = crhold(curthread->td_ucred); bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize; bp->b_blkno = fileaddr; pbgetbo(bo, bp); bp->b_vp = vp; bp->b_bcount = bsize; bp->b_bufsize = bsize; bp->b_runningbufspace = bp->b_bufsize; atomic_add_long(&runningbufspace, bp->b_runningbufspace); /* do the input */ bp->b_iooffset = dbtob(bp->b_blkno); bstrategy(bp); bwait(bp, PVM, "vnsrd"); if ((bp->b_ioflags & BIO_ERROR) != 0) error = EIO; /* * free the buffer header back to the swap buffer pool */ bp->b_vp = NULL; pbrelbo(bp); relpbuf(bp, &vnode_pbuf_freecnt); if (error) break; } else bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); KASSERT((m->dirty & bits) == 0, ("vnode_pager_input_smlfs: page %p is dirty", m)); VM_OBJECT_WLOCK(object); m->valid |= bits; VM_OBJECT_WUNLOCK(object); } sf_buf_free(sf); if (error) { return VM_PAGER_ERROR; } return VM_PAGER_OK; } /* * old style vnode pager input routine */ static int vnode_pager_input_old(vm_object_t object, vm_page_t m) { struct uio auio; struct iovec aiov; int error; int size; struct sf_buf *sf; struct vnode *vp; VM_OBJECT_ASSERT_WLOCKED(object); error = 0; /* * Return failure if beyond current EOF */ if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { return VM_PAGER_BAD; } else { size = PAGE_SIZE; if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); vp = object->handle; VM_OBJECT_WUNLOCK(object); /* * Allocate a kernel virtual address and initialize so that * we can use VOP_READ/WRITE routines. */ sf = sf_buf_alloc(m, 0); aiov.iov_base = (caddr_t)sf_buf_kva(sf); aiov.iov_len = size; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = IDX_TO_OFF(m->pindex); auio.uio_segflg = UIO_SYSSPACE; auio.uio_rw = UIO_READ; auio.uio_resid = size; auio.uio_td = curthread; error = VOP_READ(vp, &auio, 0, curthread->td_ucred); if (!error) { int count = size - auio.uio_resid; if (count == 0) error = EINVAL; else if (count != PAGE_SIZE) bzero((caddr_t)sf_buf_kva(sf) + count, PAGE_SIZE - count); } sf_buf_free(sf); VM_OBJECT_WLOCK(object); } KASSERT(m->dirty == 0, ("vnode_pager_input_old: page %p is dirty", m)); if (!error) m->valid = VM_PAGE_BITS_ALL; return error ? VM_PAGER_ERROR : VM_PAGER_OK; } /* * generic vnode pager input routine */ /* * Local media VFS's that do not implement their own VOP_GETPAGES * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() * to implement the previous behaviour. * * All other FS's should use the bypass to get to the local media * backing vp's VOP_GETPAGES. */ static int vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead) { struct vnode *vp; int rtval; vp = object->handle; VM_OBJECT_WUNLOCK(object); rtval = VOP_GETPAGES(vp, m, count, rbehind, rahead); KASSERT(rtval != EOPNOTSUPP, ("vnode_pager: FS getpages not implemented\n")); VM_OBJECT_WLOCK(object); return rtval; } static int vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead, vop_getpages_iodone_t iodone, void *arg) { struct vnode *vp; int rtval; vp = object->handle; VM_OBJECT_WUNLOCK(object); rtval = VOP_GETPAGES_ASYNC(vp, m, count, rbehind, rahead, iodone, arg); KASSERT(rtval != EOPNOTSUPP, ("vnode_pager: FS getpages_async not implemented\n")); VM_OBJECT_WLOCK(object); return (rtval); } /* * The implementation of VOP_GETPAGES() and VOP_GETPAGES_ASYNC() for * local filesystems, where partially valid pages can only occur at * the end of file. */ int vnode_pager_local_getpages(struct vop_getpages_args *ap) { return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, ap->a_rahead, NULL, NULL)); } int vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap) { return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg)); } /* * This is now called from local media FS's to operate against their * own vnodes if they fail to implement VOP_GETPAGES. */ int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, int *a_rbehind, int *a_rahead, vop_getpages_iodone_t iodone, void *arg) { vm_object_t object; struct bufobj *bo; struct buf *bp; off_t foff; #ifdef INVARIANTS off_t blkno0; #endif int bsize, pagesperblock, *freecnt; int error, before, after, rbehind, rahead, poff, i; int bytecount, secmask; KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, ("%s does not support devices", __func__)); if (vp->v_iflag & VI_DOOMED) return (VM_PAGER_BAD); object = vp->v_object; foff = IDX_TO_OFF(m[0]->pindex); bsize = vp->v_mount->mnt_stat.f_iosize; pagesperblock = bsize / PAGE_SIZE; KASSERT(foff < object->un_pager.vnp.vnp_size, ("%s: page %p offset beyond vp %p size", __func__, m[0], vp)); KASSERT(count <= sizeof(bp->b_pages), ("%s: requested %d pages", __func__, count)); /* * The last page has valid blocks. Invalid part can only * exist at the end of file, and the page is made fully valid * by zeroing in vm_pager_get_pages(). */ if (m[count - 1]->valid != 0 && --count == 0) { if (iodone != NULL) iodone(arg, m, 1, 0); return (VM_PAGER_OK); } /* * Synchronous and asynchronous paging operations use different * free pbuf counters. This is done to avoid asynchronous requests * to consume all pbufs. * Allocate the pbuf at the very beginning of the function, so that * if we are low on certain kind of pbufs don't even proceed to BMAP, * but sleep. */ freecnt = iodone != NULL ? &vnode_async_pbuf_freecnt : &vnode_pbuf_freecnt; bp = getpbuf(freecnt); /* * Get the underlying device blocks for the file with VOP_BMAP(). * If the file system doesn't support VOP_BMAP, use old way of * getting pages via VOP_READ. */ error = VOP_BMAP(vp, foff / bsize, &bo, &bp->b_blkno, &after, &before); if (error == EOPNOTSUPP) { relpbuf(bp, freecnt); VM_OBJECT_WLOCK(object); for (i = 0; i < count; i++) { VM_CNT_INC(v_vnodein); VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_old(object, m[i]); if (error) break; } VM_OBJECT_WUNLOCK(object); return (error); } else if (error != 0) { relpbuf(bp, freecnt); return (VM_PAGER_ERROR); } /* * If the file system supports BMAP, but blocksize is smaller * than a page size, then use special small filesystem code. */ if (pagesperblock == 0) { relpbuf(bp, freecnt); for (i = 0; i < count; i++) { VM_CNT_INC(v_vnodein); VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_smlfs(object, m[i]); if (error) break; } return (error); } /* * A sparse file can be encountered only for a single page request, * which may not be preceded by call to vm_pager_haspage(). */ if (bp->b_blkno == -1) { KASSERT(count == 1, ("%s: array[%d] request to a sparse file %p", __func__, count, vp)); relpbuf(bp, freecnt); pmap_zero_page(m[0]); KASSERT(m[0]->dirty == 0, ("%s: page %p is dirty", __func__, m[0])); VM_OBJECT_WLOCK(object); m[0]->valid = VM_PAGE_BITS_ALL; VM_OBJECT_WUNLOCK(object); return (VM_PAGER_OK); } #ifdef INVARIANTS blkno0 = bp->b_blkno; #endif bp->b_blkno += (foff % bsize) / DEV_BSIZE; /* Recalculate blocks available after/before to pages. */ poff = (foff % bsize) / PAGE_SIZE; before *= pagesperblock; before += poff; after *= pagesperblock; after += pagesperblock - (poff + 1); if (m[0]->pindex + after >= object->size) after = object->size - 1 - m[0]->pindex; KASSERT(count <= after + 1, ("%s: %d pages asked, can do only %d", __func__, count, after + 1)); after -= count - 1; /* Trim requested rbehind/rahead to possible values. */ rbehind = a_rbehind ? *a_rbehind : 0; rahead = a_rahead ? *a_rahead : 0; rbehind = min(rbehind, before); rbehind = min(rbehind, m[0]->pindex); rahead = min(rahead, after); rahead = min(rahead, object->size - m[count - 1]->pindex); /* * Check that total amount of pages fit into buf. Trim rbehind and * rahead evenly if not. */ if (rbehind + rahead + count > nitems(bp->b_pages)) { int trim, sum; trim = rbehind + rahead + count - nitems(bp->b_pages) + 1; sum = rbehind + rahead; if (rbehind == before) { /* Roundup rbehind trim to block size. */ rbehind -= roundup(trim * rbehind / sum, pagesperblock); if (rbehind < 0) rbehind = 0; } else rbehind -= trim * rbehind / sum; rahead -= trim * rahead / sum; } KASSERT(rbehind + rahead + count <= nitems(bp->b_pages), ("%s: behind %d ahead %d count %d", __func__, rbehind, rahead, count)); /* * Fill in the bp->b_pages[] array with requested and optional * read behind or read ahead pages. Read behind pages are looked * up in a backward direction, down to a first cached page. Same * for read ahead pages, but there is no need to shift the array * in case of encountering a cached page. */ i = bp->b_npages = 0; if (rbehind) { vm_pindex_t startpindex, tpindex; vm_page_t p; VM_OBJECT_WLOCK(object); startpindex = m[0]->pindex - rbehind; if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL && p->pindex >= startpindex) startpindex = p->pindex + 1; /* tpindex is unsigned; beware of numeric underflow. */ for (tpindex = m[0]->pindex - 1; tpindex >= startpindex && tpindex < m[0]->pindex; tpindex--, i++) { p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); if (p == NULL) { /* Shift the array. */ for (int j = 0; j < i; j++) bp->b_pages[j] = bp->b_pages[j + tpindex + 1 - startpindex]; break; } bp->b_pages[tpindex - startpindex] = p; } bp->b_pgbefore = i; bp->b_npages += i; bp->b_blkno -= IDX_TO_OFF(i) / DEV_BSIZE; } else bp->b_pgbefore = 0; /* Requested pages. */ for (int j = 0; j < count; j++, i++) bp->b_pages[i] = m[j]; bp->b_npages += count; if (rahead) { vm_pindex_t endpindex, tpindex; vm_page_t p; if (!VM_OBJECT_WOWNED(object)) VM_OBJECT_WLOCK(object); endpindex = m[count - 1]->pindex + rahead + 1; if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL && p->pindex < endpindex) endpindex = p->pindex; if (endpindex > object->size) endpindex = object->size; for (tpindex = m[count - 1]->pindex + 1; tpindex < endpindex; i++, tpindex++) { p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); if (p == NULL) break; bp->b_pages[i] = p; } bp->b_pgafter = i - bp->b_npages; bp->b_npages = i; } else bp->b_pgafter = 0; if (VM_OBJECT_WOWNED(object)) VM_OBJECT_WUNLOCK(object); /* Report back actual behind/ahead read. */ if (a_rbehind) *a_rbehind = bp->b_pgbefore; if (a_rahead) *a_rahead = bp->b_pgafter; #ifdef INVARIANTS KASSERT(bp->b_npages <= nitems(bp->b_pages), ("%s: buf %p overflowed", __func__, bp)); for (int j = 1, prev = 0; j < bp->b_npages; j++) { if (bp->b_pages[j] == bogus_page) continue; KASSERT(bp->b_pages[j]->pindex - bp->b_pages[prev]->pindex == j - prev, ("%s: pages array not consecutive, bp %p", __func__, bp)); prev = j; } #endif /* * Recalculate first offset and bytecount with regards to read behind. * Truncate bytecount to vnode real size and round up physical size * for real devices. */ foff = IDX_TO_OFF(bp->b_pages[0]->pindex); bytecount = bp->b_npages << PAGE_SHIFT; if ((foff + bytecount) > object->un_pager.vnp.vnp_size) bytecount = object->un_pager.vnp.vnp_size - foff; secmask = bo->bo_bsize - 1; KASSERT(secmask < PAGE_SIZE && secmask > 0, ("%s: sector size %d too large", __func__, secmask + 1)); bytecount = (bytecount + secmask) & ~secmask; /* * And map the pages to be read into the kva, if the filesystem * requires mapped buffers. */ if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 && unmapped_buf_allowed) { bp->b_data = unmapped_buf; bp->b_offset = 0; } else { bp->b_data = bp->b_kvabase; pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages); } /* Build a minimal buffer header. */ bp->b_iocmd = BIO_READ; KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); bp->b_rcred = crhold(curthread->td_ucred); bp->b_wcred = crhold(curthread->td_ucred); pbgetbo(bo, bp); bp->b_vp = vp; bp->b_bcount = bp->b_bufsize = bp->b_runningbufspace = bytecount; bp->b_iooffset = dbtob(bp->b_blkno); KASSERT(IDX_TO_OFF(m[0]->pindex - bp->b_pages[0]->pindex) == (blkno0 - bp->b_blkno) * DEV_BSIZE + IDX_TO_OFF(m[0]->pindex) % bsize, ("wrong offsets bsize %d m[0] %ju b_pages[0] %ju " "blkno0 %ju b_blkno %ju", bsize, (uintmax_t)m[0]->pindex, (uintmax_t)bp->b_pages[0]->pindex, (uintmax_t)blkno0, (uintmax_t)bp->b_blkno)); atomic_add_long(&runningbufspace, bp->b_runningbufspace); VM_CNT_INC(v_vnodein); VM_CNT_ADD(v_vnodepgsin, bp->b_npages); if (iodone != NULL) { /* async */ bp->b_pgiodone = iodone; bp->b_caller1 = arg; bp->b_iodone = vnode_pager_generic_getpages_done_async; bp->b_flags |= B_ASYNC; BUF_KERNPROC(bp); bstrategy(bp); return (VM_PAGER_OK); } else { bp->b_iodone = bdone; bstrategy(bp); bwait(bp, PVM, "vnread"); error = vnode_pager_generic_getpages_done(bp); for (i = 0; i < bp->b_npages; i++) bp->b_pages[i] = NULL; bp->b_vp = NULL; pbrelbo(bp); relpbuf(bp, &vnode_pbuf_freecnt); return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); } } static void vnode_pager_generic_getpages_done_async(struct buf *bp) { int error; error = vnode_pager_generic_getpages_done(bp); /* Run the iodone upon the requested range. */ bp->b_pgiodone(bp->b_caller1, bp->b_pages + bp->b_pgbefore, bp->b_npages - bp->b_pgbefore - bp->b_pgafter, error); for (int i = 0; i < bp->b_npages; i++) bp->b_pages[i] = NULL; bp->b_vp = NULL; pbrelbo(bp); relpbuf(bp, &vnode_async_pbuf_freecnt); } static int vnode_pager_generic_getpages_done(struct buf *bp) { vm_object_t object; off_t tfoff, nextoff; int i, error; error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0; object = bp->b_vp->v_object; if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) { if (!buf_mapped(bp)) { bp->b_data = bp->b_kvabase; pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages); } bzero(bp->b_data + bp->b_bcount, PAGE_SIZE * bp->b_npages - bp->b_bcount); } if (buf_mapped(bp)) { pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages); bp->b_data = unmapped_buf; } VM_OBJECT_WLOCK(object); for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex); i < bp->b_npages; i++, tfoff = nextoff) { vm_page_t mt; nextoff = tfoff + PAGE_SIZE; mt = bp->b_pages[i]; if (nextoff <= object->un_pager.vnp.vnp_size) { /* * Read filled up entire page. */ mt->valid = VM_PAGE_BITS_ALL; KASSERT(mt->dirty == 0, ("%s: page %p is dirty", __func__, mt)); KASSERT(!pmap_page_is_mapped(mt), ("%s: page %p is mapped", __func__, mt)); } else { /* * Read did not fill up entire page. * * Currently we do not set the entire page valid, * we just try to clear the piece that we couldn't * read. */ vm_page_set_valid_range(mt, 0, object->un_pager.vnp.vnp_size - tfoff); KASSERT((mt->dirty & vm_page_bits(0, object->un_pager.vnp.vnp_size - tfoff)) == 0, ("%s: page %p is dirty", __func__, mt)); } if (i < bp->b_pgbefore || i >= bp->b_npages - bp->b_pgafter) vm_page_readahead_finish(mt); } VM_OBJECT_WUNLOCK(object); if (error != 0) printf("%s: I/O read error %d\n", __func__, error); return (error); } /* * EOPNOTSUPP is no longer legal. For local media VFS's that do not * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to * vnode_pager_generic_putpages() to implement the previous behaviour. * * All other FS's should use the bypass to get to the local media * backing vp's VOP_PUTPAGES. */ static void vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, int *rtvals) { int rtval; struct vnode *vp; int bytes = count * PAGE_SIZE; /* * Force synchronous operation if we are extremely low on memory * to prevent a low-memory deadlock. VOP operations often need to * allocate more memory to initiate the I/O ( i.e. do a BMAP * operation ). The swapper handles the case by limiting the amount * of asynchronous I/O, but that sort of solution doesn't scale well * for the vnode pager without a lot of work. * * Also, the backing vnode's iodone routine may not wake the pageout * daemon up. This should be probably be addressed XXX. */ if (vm_cnt.v_free_count < vm_cnt.v_pageout_free_min) flags |= VM_PAGER_PUT_SYNC; /* * Call device-specific putpages function */ vp = object->handle; VM_OBJECT_WUNLOCK(object); rtval = VOP_PUTPAGES(vp, m, bytes, flags, rtvals); KASSERT(rtval != EOPNOTSUPP, ("vnode_pager: stale FS putpages\n")); VM_OBJECT_WLOCK(object); } /* * This is now called from local media FS's to operate against their * own vnodes if they fail to implement VOP_PUTPAGES. * * This is typically called indirectly via the pageout daemon and * clustering has already typically occurred, so in general we ask the * underlying filesystem to write the data out asynchronously rather * then delayed. */ int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount, int flags, int *rtvals) { vm_object_t object; vm_page_t m; vm_ooffset_t poffset; struct uio auio; struct iovec aiov; int count, error, i, maxsize, ncount, pgoff, ppscheck; static struct timeval lastfail; static int curfail; object = vp->v_object; count = bytecount / PAGE_SIZE; for (i = 0; i < count; i++) rtvals[i] = VM_PAGER_ERROR; if ((int64_t)ma[0]->pindex < 0) { printf("vnode_pager_generic_putpages: " "attempt to write meta-data 0x%jx(%lx)\n", (uintmax_t)ma[0]->pindex, (u_long)ma[0]->dirty); rtvals[0] = VM_PAGER_BAD; return (VM_PAGER_BAD); } maxsize = count * PAGE_SIZE; ncount = count; poffset = IDX_TO_OFF(ma[0]->pindex); /* * If the page-aligned write is larger then the actual file we * have to invalidate pages occurring beyond the file EOF. However, * there is an edge case where a file may not be page-aligned where * the last page is partially invalid. In this case the filesystem * may not properly clear the dirty bits for the entire page (which * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). * With the page locked we are free to fix-up the dirty bits here. * * We do not under any circumstances truncate the valid bits, as * this will screw up bogus page replacement. */ VM_OBJECT_WLOCK(object); if (maxsize + poffset > object->un_pager.vnp.vnp_size) { if (object->un_pager.vnp.vnp_size > poffset) { maxsize = object->un_pager.vnp.vnp_size - poffset; ncount = btoc(maxsize); if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { /* * If the object is locked and the following * conditions hold, then the page's dirty * field cannot be concurrently changed by a * pmap operation. */ m = ma[ncount - 1]; vm_page_assert_sbusied(m); KASSERT(!pmap_page_is_write_mapped(m), ("vnode_pager_generic_putpages: page %p is not read-only", m)); MPASS(m->dirty != 0); vm_page_clear_dirty(m, pgoff, PAGE_SIZE - pgoff); } } else { maxsize = 0; ncount = 0; } for (i = ncount; i < count; i++) rtvals[i] = VM_PAGER_BAD; } for (i = 0; i < ncount - ((btoc(maxsize) & PAGE_MASK) != 0); i++) MPASS(ma[i]->dirty == VM_PAGE_BITS_ALL); VM_OBJECT_WUNLOCK(object); aiov.iov_base = NULL; aiov.iov_len = maxsize; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = poffset; auio.uio_segflg = UIO_NOCOPY; auio.uio_rw = UIO_WRITE; auio.uio_resid = maxsize; auio.uio_td = NULL; error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags), curthread->td_ucred); VM_CNT_INC(v_vnodeout); VM_CNT_ADD(v_vnodepgsout, ncount); ppscheck = 0; if (error != 0 && (ppscheck = ppsratecheck(&lastfail, &curfail, 1)) != 0) printf("vnode_pager_putpages: I/O error %d\n", error); if (auio.uio_resid != 0 && (ppscheck != 0 || ppsratecheck(&lastfail, &curfail, 1) != 0)) printf("vnode_pager_putpages: residual I/O %zd at %ju\n", auio.uio_resid, (uintmax_t)ma[0]->pindex); for (i = 0; i < ncount; i++) rtvals[i] = VM_PAGER_OK; return (rtvals[0]); } int vnode_pager_putpages_ioflags(int pager_flags) { int ioflags; /* * Pageouts are already clustered, use IO_ASYNC to force a * bawrite() rather then a bdwrite() to prevent paging I/O * from saturating the buffer cache. Dummy-up the sequential * heuristic to cause large ranges to cluster. If neither * IO_SYNC or IO_ASYNC is set, the system decides how to * cluster. */ ioflags = IO_VMIO; if ((pager_flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) != 0) ioflags |= IO_SYNC; else if ((pager_flags & VM_PAGER_CLUSTER_OK) == 0) ioflags |= IO_ASYNC; ioflags |= (pager_flags & VM_PAGER_PUT_INVAL) != 0 ? IO_INVAL: 0; ioflags |= (pager_flags & VM_PAGER_PUT_NOREUSE) != 0 ? IO_NOREUSE : 0; ioflags |= IO_SEQMAX << IO_SEQSHIFT; return (ioflags); } +/* + * vnode_pager_undirty_pages(). + * + * A helper to mark pages as clean after pageout that was possibly + * done with a short write. The lpos argument specifies the page run + * length in bytes, and the written argument specifies how many bytes + * were actually written. eof is the offset past the last valid byte + * in the vnode using the absolute file position of the first byte in + * the run as the base from which it is computed. + */ void -vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written) +vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, off_t eof, + int lpos) { vm_object_t obj; - int i, pos; + int i, pos, pos_devb; - if (written == 0) + if (written == 0 && eof >= lpos) return; obj = ma[0]->object; VM_OBJECT_WLOCK(obj); for (i = 0, pos = 0; pos < written; i++, pos += PAGE_SIZE) { if (pos < trunc_page(written)) { rtvals[i] = VM_PAGER_OK; vm_page_undirty(ma[i]); } else { /* Partially written page. */ rtvals[i] = VM_PAGER_AGAIN; vm_page_clear_dirty(ma[i], 0, written & PAGE_MASK); } } + if (eof >= lpos) /* avoid truncation */ + goto done; + for (pos = eof, i = OFF_TO_IDX(trunc_page(pos)); pos < lpos; i++) { + if (pos != trunc_page(pos)) { + /* + * The page contains the last valid byte in + * the vnode, mark the rest of the page as + * clean, potentially making the whole page + * clean. + */ + pos_devb = roundup2(pos & PAGE_MASK, DEV_BSIZE); + vm_page_clear_dirty(ma[i], pos_devb, PAGE_SIZE - + pos_devb); + + /* + * If the page was cleaned, report the pageout + * on it as successful. msync() no longer + * needs to write out the page, endlessly + * creating write requests and dirty buffers. + */ + if (ma[i]->dirty == 0) + rtvals[i] = VM_PAGER_OK; + + pos = round_page(pos); + } else { + /* vm_pageout_flush() clears dirty */ + rtvals[i] = VM_PAGER_BAD; + pos += PAGE_SIZE; + } + } +done: VM_OBJECT_WUNLOCK(obj); } void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { struct vnode *vp; vm_ooffset_t old_wm; VM_OBJECT_WLOCK(object); if (object->type != OBJT_VNODE) { VM_OBJECT_WUNLOCK(object); return; } old_wm = object->un_pager.vnp.writemappings; object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start; vp = object->handle; if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) { ASSERT_VOP_ELOCKED(vp, "v_writecount inc"); VOP_ADD_WRITECOUNT(vp, 1); CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d", __func__, vp, vp->v_writecount); } else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) { ASSERT_VOP_ELOCKED(vp, "v_writecount dec"); VOP_ADD_WRITECOUNT(vp, -1); CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", __func__, vp, vp->v_writecount); } VM_OBJECT_WUNLOCK(object); } void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { struct vnode *vp; struct mount *mp; vm_offset_t inc; VM_OBJECT_WLOCK(object); /* * First, recheck the object type to account for the race when * the vnode is reclaimed. */ if (object->type != OBJT_VNODE) { VM_OBJECT_WUNLOCK(object); return; } /* * Optimize for the case when writemappings is not going to * zero. */ inc = end - start; if (object->un_pager.vnp.writemappings != inc) { object->un_pager.vnp.writemappings -= inc; VM_OBJECT_WUNLOCK(object); return; } vp = object->handle; vhold(vp); VM_OBJECT_WUNLOCK(object); mp = NULL; vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); /* * Decrement the object's writemappings, by swapping the start * and end arguments for vnode_pager_update_writecount(). If * there was not a race with vnode reclaimation, then the * vnode's v_writecount is decremented. */ vnode_pager_update_writecount(object, end, start); VOP_UNLOCK(vp, 0); vdrop(vp); if (mp != NULL) vn_finished_write(mp); } Index: projects/runtime-coverage/sys/vm/vnode_pager.h =================================================================== --- projects/runtime-coverage/sys/vm/vnode_pager.h (revision 321584) +++ projects/runtime-coverage/sys/vm/vnode_pager.h (revision 321585) @@ -1,58 +1,59 @@ /*- * Copyright (c) 1990 University of Utah. * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)vnode_pager.h 8.1 (Berkeley) 6/11/93 * $FreeBSD$ */ #ifndef _VNODE_PAGER_ #define _VNODE_PAGER_ 1 #ifdef _KERNEL int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, int *rbehind, int *rahead, vop_getpages_iodone_t iodone, void *arg); int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int count, int flags, int *rtvals); int vnode_pager_local_getpages(struct vop_getpages_args *ap); int vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap); int vnode_pager_putpages_ioflags(int pager_flags); void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end); -void vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written); +void vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, + off_t eof, int lpos); void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end); #endif /* _KERNEL */ #endif /* _VNODE_PAGER_ */ Index: projects/runtime-coverage/usr.bin/indent/args.c =================================================================== --- projects/runtime-coverage/usr.bin/indent/args.c (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/args.c (revision 321585) @@ -1,355 +1,356 @@ /*- * Copyright (c) 1985 Sun Microsystems, Inc. * Copyright (c) 1980, 1993 * The Regents of the University of California. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if 0 #ifndef lint static char sccsid[] = "@(#)args.c 8.1 (Berkeley) 6/6/93"; #endif /* not lint */ #endif #include __FBSDID("$FreeBSD$"); /* * Argument scanning and profile reading code. Default parameters are set * here as well. */ #include #include #include #include #include #include #include "indent_globs.h" #include "indent.h" /* profile types */ #define PRO_SPECIAL 1 /* special case */ #define PRO_BOOL 2 /* boolean */ #define PRO_INT 3 /* integer */ #define PRO_FONT 4 /* troff font */ /* profile specials for booleans */ #define ON 1 /* turn it on */ #define OFF 0 /* turn it off */ /* profile specials for specials */ #define IGN 1 /* ignore it */ #define CLI 2 /* case label indent (float) */ #define STDIN 3 /* use stdin */ #define KEY 4 /* type (keyword) */ static void scan_profile(FILE *); #define KEY_FILE 5 /* only used for args */ const char *option_source = "?"; void add_typedefs_from_file(const char *str); /* * N.B.: because of the way the table here is scanned, options whose names are * substrings of other options must occur later; that is, with -lp vs -l, -lp * must be first. Also, while (most) booleans occur more than once, the last * default value is the one actually assigned. */ struct pro { const char *p_name; /* name, e.g. -bl, -cli */ int p_type; /* type (int, bool, special) */ int p_default; /* the default value (if int) */ int p_special; /* depends on type */ int *p_obj; /* the associated variable */ } pro[] = { {"T", PRO_SPECIAL, 0, KEY, 0}, {"U", PRO_SPECIAL, 0, KEY_FILE, 0}, {"P", PRO_SPECIAL, 0, IGN, 0}, {"bacc", PRO_BOOL, false, ON, &blanklines_around_conditional_compilation}, {"badp", PRO_BOOL, false, ON, &blanklines_after_declarations_at_proctop}, {"bad", PRO_BOOL, false, ON, &blanklines_after_declarations}, {"bap", PRO_BOOL, false, ON, &blanklines_after_procs}, {"bbb", PRO_BOOL, false, ON, &blanklines_before_blockcomments}, {"bc", PRO_BOOL, true, OFF, &ps.leave_comma}, {"bl", PRO_BOOL, true, OFF, &btype_2}, {"br", PRO_BOOL, true, ON, &btype_2}, {"bs", PRO_BOOL, false, ON, &Bill_Shannon}, {"cdb", PRO_BOOL, true, ON, &comment_delimiter_on_blankline}, {"cd", PRO_INT, 0, 0, &ps.decl_com_ind}, {"ce", PRO_BOOL, true, ON, &cuddle_else}, {"ci", PRO_INT, 0, 0, &continuation_indent}, {"cli", PRO_SPECIAL, 0, CLI, 0}, {"c", PRO_INT, 33, 0, &ps.com_ind}, {"di", PRO_INT, 16, 0, &ps.decl_indent}, {"dj", PRO_BOOL, false, ON, &ps.ljust_decl}, {"d", PRO_INT, 0, 0, &ps.unindent_displace}, {"eei", PRO_BOOL, false, ON, &extra_expression_indent}, {"ei", PRO_BOOL, true, ON, &ps.else_if}, {"fbc", PRO_FONT, 0, 0, (int *) &blkcomf}, {"fbs", PRO_BOOL, true, ON, &function_brace_split}, {"fbx", PRO_FONT, 0, 0, (int *) &boxcomf}, {"fb", PRO_FONT, 0, 0, (int *) &bodyf}, {"fc1", PRO_BOOL, true, ON, &format_col1_comments}, {"fcb", PRO_BOOL, true, ON, &format_block_comments}, {"fc", PRO_FONT, 0, 0, (int *) &scomf}, {"fk", PRO_FONT, 0, 0, (int *) &keywordf}, {"fs", PRO_FONT, 0, 0, (int *) &stringf}, {"ip", PRO_BOOL, true, ON, &ps.indent_parameters}, {"i", PRO_INT, 8, 0, &ps.ind_size}, {"lc", PRO_INT, 0, 0, &block_comment_max_col}, {"ldi", PRO_INT, -1, 0, &ps.local_decl_indent}, {"lp", PRO_BOOL, true, ON, &lineup_to_parens}, {"l", PRO_INT, 78, 0, &max_col}, {"nbacc", PRO_BOOL, false, OFF, &blanklines_around_conditional_compilation}, {"nbadp", PRO_BOOL, false, OFF, &blanklines_after_declarations_at_proctop}, {"nbad", PRO_BOOL, false, OFF, &blanklines_after_declarations}, {"nbap", PRO_BOOL, false, OFF, &blanklines_after_procs}, {"nbbb", PRO_BOOL, false, OFF, &blanklines_before_blockcomments}, {"nbc", PRO_BOOL, true, ON, &ps.leave_comma}, {"nbs", PRO_BOOL, false, OFF, &Bill_Shannon}, {"ncdb", PRO_BOOL, true, OFF, &comment_delimiter_on_blankline}, {"nce", PRO_BOOL, true, OFF, &cuddle_else}, {"ndj", PRO_BOOL, false, OFF, &ps.ljust_decl}, {"neei", PRO_BOOL, false, OFF, &extra_expression_indent}, {"nei", PRO_BOOL, true, OFF, &ps.else_if}, {"nfbs", PRO_BOOL, true, OFF, &function_brace_split}, {"nfc1", PRO_BOOL, true, OFF, &format_col1_comments}, {"nfcb", PRO_BOOL, true, OFF, &format_block_comments}, {"nip", PRO_BOOL, true, OFF, &ps.indent_parameters}, {"nlp", PRO_BOOL, true, OFF, &lineup_to_parens}, {"npcs", PRO_BOOL, false, OFF, &proc_calls_space}, {"npro", PRO_SPECIAL, 0, IGN, 0}, {"npsl", PRO_BOOL, true, OFF, &procnames_start_line}, {"nps", PRO_BOOL, false, OFF, &pointer_as_binop}, {"nsac", PRO_BOOL, false, OFF, &space_after_cast}, {"nsc", PRO_BOOL, true, OFF, &star_comment_cont}, {"nsob", PRO_BOOL, false, OFF, &swallow_optional_blanklines}, {"nut", PRO_BOOL, true, OFF, &use_tabs}, {"nv", PRO_BOOL, false, OFF, &verbose}, {"pcs", PRO_BOOL, false, ON, &proc_calls_space}, {"psl", PRO_BOOL, true, ON, &procnames_start_line}, {"ps", PRO_BOOL, false, ON, &pointer_as_binop}, {"sac", PRO_BOOL, false, ON, &space_after_cast}, {"sc", PRO_BOOL, true, ON, &star_comment_cont}, {"sob", PRO_BOOL, false, ON, &swallow_optional_blanklines}, {"st", PRO_SPECIAL, 0, STDIN, 0}, {"ta", PRO_BOOL, false, ON, &auto_typedefs}, + {"ts", PRO_INT, 8, 0, &tabsize}, {"troff", PRO_BOOL, false, ON, &troff}, {"ut", PRO_BOOL, true, ON, &use_tabs}, {"v", PRO_BOOL, false, ON, &verbose}, /* whew! */ {0, 0, 0, 0, 0} }; /* * set_profile reads $HOME/.indent.pro and ./.indent.pro and handles arguments * given in these files. */ void set_profile(const char *profile_name) { FILE *f; char fname[PATH_MAX]; static char prof[] = ".indent.pro"; if (profile_name == NULL) snprintf(fname, sizeof(fname), "%s/%s", getenv("HOME"), prof); else snprintf(fname, sizeof(fname), "%s", profile_name + 2); if ((f = fopen(option_source = fname, "r")) != NULL) { scan_profile(f); (void) fclose(f); } if ((f = fopen(option_source = prof, "r")) != NULL) { scan_profile(f); (void) fclose(f); } option_source = "Command line"; } static void scan_profile(FILE *f) { int comment, i; char *p; char buf[BUFSIZ]; while (1) { p = buf; comment = 0; while ((i = getc(f)) != EOF) { if (i == '*' && !comment && p > buf && p[-1] == '/') { comment = p - buf; *p++ = i; } else if (i == '/' && comment && p > buf && p[-1] == '*') { p = buf + comment - 1; comment = 0; } else if (isspace(i)) { if (p > buf && !comment) break; } else { *p++ = i; } } if (p != buf) { *p++ = 0; if (verbose) printf("profile: %s\n", buf); set_option(buf); } else if (i == EOF) return; } } static const char * eqin(const char *s1, const char *s2) { while (*s1) { if (*s1++ != *s2++) return (NULL); } return (s2); } /* * Set the defaults. */ void set_defaults(void) { struct pro *p; /* * Because ps.case_indent is a float, we can't initialize it from the * table: */ ps.case_indent = 0.0; /* -cli0.0 */ for (p = pro; p->p_name; p++) if (p->p_type != PRO_SPECIAL && p->p_type != PRO_FONT) *p->p_obj = p->p_default; } void set_option(char *arg) { struct pro *p; const char *param_start; arg++; /* ignore leading "-" */ for (p = pro; p->p_name; p++) if (*p->p_name == *arg && (param_start = eqin(p->p_name, arg)) != NULL) goto found; errx(1, "%s: unknown parameter \"%s\"", option_source, arg - 1); found: switch (p->p_type) { case PRO_SPECIAL: switch (p->p_special) { case IGN: break; case CLI: if (*param_start == 0) goto need_param; ps.case_indent = atof(param_start); break; case STDIN: if (input == NULL) input = stdin; if (output == NULL) output = stdout; break; case KEY: if (*param_start == 0) goto need_param; add_typename(param_start); break; case KEY_FILE: if (*param_start == 0) goto need_param; add_typedefs_from_file(param_start); break; default: errx(1, "set_option: internal error: p_special %d", p->p_special); } break; case PRO_BOOL: if (p->p_special == OFF) *p->p_obj = false; else *p->p_obj = true; break; case PRO_INT: if (!isdigit(*param_start)) { need_param: errx(1, "%s: ``%s'' requires a parameter", option_source, arg - 1); } *p->p_obj = atoi(param_start); break; case PRO_FONT: parsefont((struct fstate *) p->p_obj, param_start); break; default: errx(1, "set_option: internal error: p_type %d", p->p_type); } } void add_typedefs_from_file(const char *str) { FILE *file; char line[BUFSIZ]; if ((file = fopen(str, "r")) == NULL) { fprintf(stderr, "indent: cannot open file %s\n", str); exit(1); } while ((fgets(line, BUFSIZ, file)) != NULL) { /* Remove trailing whitespace */ line[strcspn(line, " \t\n\r")] = '\0'; add_typename(line); } fclose(file); } Index: projects/runtime-coverage/usr.bin/indent/indent.1 =================================================================== --- projects/runtime-coverage/usr.bin/indent/indent.1 (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/indent.1 (revision 321585) @@ -1,590 +1,593 @@ .\" Copyright (c) 1980, 1990, 1993 .\" The Regents of the University of California. All rights reserved. .\" Copyright (c) 1976 Board of Trustees of the University of Illinois. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. Neither the name of the University nor the names of its contributors .\" may be used to endorse or promote products derived from this software .\" without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" @(#)indent.1 8.1 (Berkeley) 7/1/93 .\" $FreeBSD$ .\" -.Dd January 2, 2017 +.Dd July 25, 2017 .Dt INDENT 1 .Os .Sh NAME .Nm indent .Nd indent and format C program source .Sh SYNOPSIS .Nm .Op Ar input-file Op Ar output-file .Op Fl bacc | Fl nbacc .Op Fl bad | Fl nbad .Op Fl badp | Fl nbadp .Op Fl bap | Fl nbap .Bk -words .Op Fl ei | Fl ei .Op Fl eei | Fl eei .Op Fl bbb | Fl nbbb .Ek .Op Fl \&bc | Fl nbc .Op Fl \&bl .Op Fl \&br .Op Fl bs | Fl nbs .Op Fl c Ns Ar n .Op Fl \&cd Ns Ar n .Bk -words .Op Fl cdb | Fl ncdb .Ek .Op Fl \&ce | Fl nce .Op Fl \&ci Ns Ar n .Op Fl cli Ns Ar n .Op Fl d Ns Ar n .Op Fl \&di Ns Ar n .Bk -words .Op Fl fbs | Fl nfbs .Op Fl fc1 | Fl nfc1 .Op Fl fcb | Fl nfcb .Ek .Op Fl i Ns Ar n .Op Fl \&ip | Fl nip .Op Fl l Ns Ar n .Op Fl \&lc Ns Ar n .Op Fl \&ldi Ns Ar n .Op Fl \&lp | Fl nlp .Op Fl npro .Op Fl P Ns Ar file .Op Fl pcs | Fl npcs .Op Fl psl | Fl npsl .Op Fl sac | Fl nsac .Op Fl \&sc | Fl nsc .Bk -words .Op Fl sob | Fl nsob .Ek .Op Fl \&st .Op Fl \&ta .Op Fl troff +.Op Fl ts Ns Ar n .Op Fl U Ns Ar file .Op Fl ut | Fl nut .Op Fl v | Fl \&nv .Sh DESCRIPTION The .Nm utility is a .Em C program formatter. It reformats the .Em C program in the .Ar input-file according to the switches. The switches which can be specified are described below. They may appear before or after the file names. .Pp .Sy NOTE : If you only specify an .Ar input-file , the formatting is done `in-place', that is, the formatted file is written back into .Ar input-file and a backup copy of .Ar input-file is written in the current directory. If .Ar input-file is named .Sq Pa /blah/blah/file , the backup file is named .Sq Pa file.BAK . .Pp If .Ar output-file is specified, .Nm checks to make sure that it is different from .Ar input-file . .Pp The options listed below control the formatting style imposed by .Nm . .Bl -tag -width Op .It Fl bacc , nbacc If .Fl bacc is specified, a blank line is forced around every conditional compilation block. For example, in front of every #ifdef and after every #endif. Other blank lines surrounding such blocks will be swallowed. Default: .Fl nbacc . .It Fl bad , nbad If .Fl bad is specified, a blank line is forced after every block of declarations. Default: .Fl nbad . .It Fl badp , nbadp This is vaguely similar to .Fl bad except that it only applies to the first set of declarations in a procedure (just after the first `{') and it causes a blank line to be generated even if there are no declarations. The default is .Fl nbadp. .It Fl bap , nbap If .Fl bap is specified, a blank line is forced after every procedure body. Default: .Fl nbap . .It Fl bbb , nbbb If .Fl bbb is specified, a blank line is forced before every block comment. Default: .Fl nbbb . .It Fl \&bc , nbc If .Fl \&bc is specified, then a newline is forced after each comma in a declaration. .Fl nbc turns off this option. Default: .Fl \&nbc . .It Fl \&br , \&bl Specifying .Fl \&bl lines-up compound statements like this: .Bd -literal -offset indent if (...) { code } .Ed .Pp Specifying .Fl \&br (the default) makes them look like this: .Bd -literal -offset indent if (...) { code } .Ed .It Fl bs , nbs Whether a blank should always be inserted after sizeof. The default is .Fl nbs. .It Fl c Ns Ar n The column in which comments on code start. The default is 33. .It Fl cd Ns Ar n The column in which comments on declarations start. The default is for these comments to start in the same column as those on code. .It Fl cdb , ncdb Enables (disables) the placement of comment delimiters on blank lines. With this option enabled, comments look like this: .Bd -literal -offset indent /* * this is a comment */ .Ed .Pp Rather than like this: .Bd -literal -offset indent /* this is a comment */ .Ed .Pp This only affects block comments, not comments to the right of code. The default is .Fl cdb . .It Fl ce , nce Enables (disables) forcing of `else's to cuddle up to the immediately preceding `}'. The default is .Fl \&ce . .It Fl \&ci Ns Ar n Sets the continuation indent to be .Ar n . Continuation lines will be indented that far from the beginning of the first line of the statement. Parenthesized expressions have extra indentation added to indicate the nesting, unless .Fl \&lp is in effect or the continuation indent is exactly half of the main indent. .Fl \&ci defaults to the same value as .Fl i . .It Fl cli Ns Ar n Causes case labels to be indented .Ar n tab stops to the right of the containing .Ic switch statement. .Fl cli0.5 causes case labels to be indented half a tab stop. The default is .Fl cli0 . .It Fl d Ns Ar n Controls the placement of comments which are not to the right of code. For example, .Fl \&d\&1 means that such comments are placed one indentation level to the left of code. Specifying the default .Fl \&d\&0 lines-up these comments with the code. See the section on comment indentation below. .It Fl \&di Ns Ar n Specifies the indentation, in character positions, of global variable names and all struct/union member names relative to the beginning of their type declaration. The default is .Fl di16 . .It Fl dj , ndj .Fl \&dj left justifies declarations. .Fl ndj indents declarations the same as code. The default is .Fl ndj . .It Fl \&ei , nei Enables (disables) special .Ic else-if processing. If it is enabled, an .Ic if following an .Ic else will have the same indentation as the preceding .Ic \&if statement. The default is .Fl ei . .It Fl eei , neei Enables (disables) extra indentation on continuation lines of the expression part of .Ic if and .Ic while statements. These continuation lines will be indented one extra level. The default is .Fl neei . .It Fl fbs , nfbs Enables (disables) splitting the function declaration and opening brace across two lines. The default is .Fl fbs . .It Fl fc1 , nfc1 Enables (disables) the formatting of comments that start in column 1. Often, comments whose leading `/' is in column 1 have been carefully hand formatted by the programmer. In such cases, .Fl nfc1 should be used. The default is .Fl fc1 . .It Fl fcb , nfcb Enables (disables) the formatting of block comments (ones that begin with `/*\\n'). Often, block comments have been not so carefully hand formatted by the programmer, but reformatting that would just change the line breaks is not wanted. In such cases, .Fl nfcb should be used. Block comments are then handled like box comments. The default is .Fl fcb . .It Fl i Ns Ar n The number of spaces for one indentation level. The default is 8. .It Fl \&ip , nip Enables (disables) the indentation of parameter declarations from the left margin. The default is .Fl \&ip . .It Fl l Ns Ar n Maximum length of an output line. The default is 78. .It Fl lc Ns Ar n Maximum length of an output line in a block comment. The default is 0, which means to limit block comment lines in accordance with .Fl l. .It Fl \&ldi Ns Ar n Specifies the indentation, in character positions, of local variable names relative to the beginning of their type declaration. The default is for local variable names to be indented by the same amount as global ones. .It Fl \&lp , nlp Lines-up code surrounded by parenthesis in continuation lines. If a line has a left paren which is not closed on that line, then continuation lines will be lined up to start at the character position just after the left paren. For example, here is how a piece of continued code looks with .Fl nlp in effect: .Bd -literal -offset indent p1 = first_procedure(second_procedure(p2, p3), \ \ third_procedure(p4, p5)); .Ed .Pp With .Fl lp in effect (the default) the code looks somewhat clearer: .Bd -literal -offset indent p1\ =\ first_procedure(second_procedure(p2,\ p3), \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ third_procedure(p4,\ p5)); .Ed .Pp Inserting two more newlines we get: .Bd -literal -offset indent p1\ =\ first_procedure(second_procedure(p2, \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ p3), \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ third_procedure(p4, \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ p5)); .Ed .It Fl npro Causes the profile files, .Sq Pa ./.indent.pro and .Sq Pa ~/.indent.pro , to be ignored. .It Fl P Ns Ar file Read profile from .Ar file . .It Fl pcs , npcs If true .Pq Fl pcs all procedure calls will have a space inserted between the name and the `('. The default is .Fl npcs . .It Fl psl , npsl If true .Pq Fl psl the names of procedures being defined are placed in column 1 \- their types, if any, will be left on the previous lines. The default is .Fl psl . .It Fl sac , nsac Control whether parenthesized type names in casts are followed by a space or not. The default is .Fl nsac . .It Fl \&sc , nsc Enables (disables) the placement of asterisks (`*'s) at the left edge of all comments. The default is .Fl sc . .It Fl sob , nsob If .Fl sob is specified, indent will swallow optional blank lines. You can use this to get rid of blank lines after declarations. Default: .Fl nsob . .It Fl \&st Causes .Nm to take its input from stdin and put its output to stdout. .It Fl ta Automatically add all identifiers ending in "_t" to the list of type keywords. .It Fl T Ns Ar typename Adds .Ar typename to the list of type keywords. Names accumulate: .Fl T can be specified more than once. You need to specify all the typenames that appear in your program that are defined by .Ic typedef \- nothing will be harmed if you miss a few, but the program will not be formatted as nicely as it should. This sounds like a painful thing to have to do, but it is really a symptom of a problem in C: .Ic typedef causes a syntactic change in the language and .Nm cannot find all instances of .Ic typedef . .It Fl troff Causes .Nm to format the program for processing by .Xr troff 1 . It will produce a fancy listing in much the same spirit as .Xr vgrind 1 . If the output file is not specified, the default is standard output, rather than formatting in place. +.It Fl ts Ns Ar n +Assumed distance between tab stops. +The default is 8. .It Fl U Ns Ar file Adds type names from .Ar file to the list of type keywords. .It Fl ut , nut Enables (disables) the use of tab characters in the output. -Tabs are assumed to be aligned on columns divisible by 8. The default is .Fl ut . .It Fl v , \&nv .Fl v turns on `verbose' mode; .Fl \&nv turns it off. When in verbose mode, .Nm reports when it splits one line of input into two or more lines of output, and gives some size statistics at completion. The default is .Fl \&nv . .El .Pp You may set up your own `profile' of defaults to .Nm by creating a file called .Pa .indent.pro in your login directory and/or the current directory and including whatever switches you like. A `.indent.pro' in the current directory takes precedence over the one in your login directory. If .Nm is run and a profile file exists, then it is read to set up the program's defaults. Switches on the command line, though, always override profile switches. The switches should be separated by spaces, tabs or newlines. .Pp .Ss Comments .Sq Em Box .Em comments . The .Nm utility assumes that any comment with a dash or star immediately after the start of comment (that is, `/*\-' or `/**') is a comment surrounded by a box of stars. Each line of such a comment is left unchanged, except that its indentation may be adjusted to account for the change in indentation of the first line of the comment. .Pp .Em Straight text . All other comments are treated as straight text. The .Nm utility fits as many words (separated by blanks, tabs, or newlines) on a line as possible. Blank lines break paragraphs. .Ss Comment indentation If a comment is on a line with code it is started in the `comment column', which is set by the .Fl c Ns Ns Ar n command line parameter. Otherwise, the comment is started at .Ar n indentation levels less than where code is currently being placed, where .Ar n is specified by the .Fl d Ns Ns Ar n command line parameter. If the code on a line extends past the comment column, the comment starts further to the right, and the right margin may be automatically extended in extreme cases. .Ss Preprocessor lines In general, .Nm leaves preprocessor lines alone. The only reformatting that it will do is to straighten up trailing comments. It leaves embedded comments alone. Conditional compilation .Pq Ic #ifdef...#endif is recognized and .Nm attempts to correctly compensate for the syntactic peculiarities introduced. .Ss C syntax The .Nm utility understands a substantial amount about the syntax of C, but it has a `forgiving' parser. It attempts to cope with the usual sorts of incomplete and malformed syntax. In particular, the use of macros like: .Pp .Dl #define forever for(;;) .Pp is handled properly. .Sh ENVIRONMENT The .Nm utility uses the .Ev HOME environment variable. .Sh FILES .Bl -tag -width "./.indent.pro" -compact .It Pa ./.indent.pro profile file .It Pa ~/.indent.pro profile file .El .Sh HISTORY The .Nm command appeared in .Bx 4.2 . .Sh BUGS The .Nm utility has even more switches than .Xr ls 1 . .Pp A common mistake is to try to indent all the .Em C programs in a directory by typing: .Pp .Dl indent *.c .Pp This is probably a bug, not a feature. Index: projects/runtime-coverage/usr.bin/indent/indent.c =================================================================== --- projects/runtime-coverage/usr.bin/indent/indent.c (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/indent.c (revision 321585) @@ -1,1291 +1,1294 @@ /*- * Copyright (c) 1985 Sun Microsystems, Inc. * Copyright (c) 1976 Board of Trustees of the University of Illinois. * Copyright (c) 1980, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef lint static const char copyright[] = "@(#) Copyright (c) 1985 Sun Microsystems, Inc.\n\ @(#) Copyright (c) 1976 Board of Trustees of the University of Illinois.\n\ @(#) Copyright (c) 1980, 1993\n\ The Regents of the University of California. All rights reserved.\n"; #endif /* not lint */ #if 0 #ifndef lint static char sccsid[] = "@(#)indent.c 5.17 (Berkeley) 6/7/93"; #endif /* not lint */ #endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "indent_globs.h" #include "indent_codes.h" #include "indent.h" static void bakcopy(void); static void indent_declaration(int, int); const char *in_name = "Standard Input"; /* will always point to name of input * file */ const char *out_name = "Standard Output"; /* will always point to name * of output file */ char bakfile[MAXPATHLEN] = ""; int main(int argc, char **argv) { cap_rights_t rights; int dec_ind; /* current indentation for declarations */ int di_stack[20]; /* a stack of structure indentation levels */ int flushed_nl; /* used when buffering up comments to remember * that a newline was passed over */ int force_nl; /* when true, code must be broken */ int hd_type = 0; /* used to store type of stmt for if (...), * for (...), etc */ int i; /* local loop counter */ int scase; /* set to true when we see a case, so we will * know what to do with the following colon */ int sp_sw; /* when true, we are in the expression of * if(...), while(...), etc. */ int squest; /* when this is positive, we have seen a ? * without the matching : in a ?: * construct */ const char *t_ptr; /* used for copying tokens */ int tabs_to_var; /* true if using tabs to indent to var name */ int type_code; /* the type of token, returned by lexi */ int last_else = 0; /* true iff last keyword was an else */ const char *profile_name = NULL; /*-----------------------------------------------*\ | INITIALIZATION | \*-----------------------------------------------*/ found_err = 0; ps.p_stack[0] = stmt; /* this is the parser's stack */ ps.last_nl = true; /* this is true if the last thing scanned was * a newline */ ps.last_token = semicolon; combuf = (char *) malloc(bufsize); if (combuf == NULL) err(1, NULL); labbuf = (char *) malloc(bufsize); if (labbuf == NULL) err(1, NULL); codebuf = (char *) malloc(bufsize); if (codebuf == NULL) err(1, NULL); tokenbuf = (char *) malloc(bufsize); if (tokenbuf == NULL) err(1, NULL); alloc_typenames(); l_com = combuf + bufsize - 5; l_lab = labbuf + bufsize - 5; l_code = codebuf + bufsize - 5; l_token = tokenbuf + bufsize - 5; combuf[0] = codebuf[0] = labbuf[0] = ' '; /* set up code, label, and * comment buffers */ combuf[1] = codebuf[1] = labbuf[1] = '\0'; ps.else_if = 1; /* Default else-if special processing to on */ s_lab = e_lab = labbuf + 1; s_code = e_code = codebuf + 1; s_com = e_com = combuf + 1; s_token = e_token = tokenbuf + 1; in_buffer = (char *) malloc(10); if (in_buffer == NULL) err(1, NULL); in_buffer_limit = in_buffer + 8; buf_ptr = buf_end = in_buffer; line_no = 1; had_eof = ps.in_decl = ps.decl_on_line = break_comma = false; sp_sw = force_nl = false; ps.in_or_st = false; ps.bl_line = true; dec_ind = 0; di_stack[ps.dec_nest = 0] = 0; ps.want_blank = ps.in_stmt = ps.ind_stmt = false; scase = ps.pcase = false; squest = 0; sc_end = NULL; bp_save = NULL; be_save = NULL; output = NULL; tabs_to_var = 0; /*--------------------------------------------------*\ | COMMAND LINE SCAN | \*--------------------------------------------------*/ #ifdef undef max_col = 78; /* -l78 */ lineup_to_parens = 1; /* -lp */ ps.ljust_decl = 0; /* -ndj */ ps.com_ind = 33; /* -c33 */ star_comment_cont = 1; /* -sc */ ps.ind_size = 8; /* -i8 */ verbose = 0; ps.decl_indent = 16; /* -di16 */ ps.local_decl_indent = -1; /* if this is not set to some nonnegative value * by an arg, we will set this equal to * ps.decl_ind */ ps.indent_parameters = 1; /* -ip */ ps.decl_com_ind = 0; /* if this is not set to some positive value * by an arg, we will set this equal to * ps.com_ind */ btype_2 = 1; /* -br */ cuddle_else = 1; /* -ce */ ps.unindent_displace = 0; /* -d0 */ ps.case_indent = 0; /* -cli0 */ format_block_comments = 1; /* -fcb */ format_col1_comments = 1; /* -fc1 */ procnames_start_line = 1; /* -psl */ proc_calls_space = 0; /* -npcs */ comment_delimiter_on_blankline = 1; /* -cdb */ ps.leave_comma = 1; /* -nbc */ #endif for (i = 1; i < argc; ++i) if (strcmp(argv[i], "-npro") == 0) break; else if (argv[i][0] == '-' && argv[i][1] == 'P' && argv[i][2] != '\0') profile_name = argv[i]; /* non-empty -P (set profile) */ set_defaults(); if (i >= argc) set_profile(profile_name); for (i = 1; i < argc; ++i) { /* * look thru args (if any) for changes to defaults */ if (argv[i][0] != '-') {/* no flag on parameter */ if (input == NULL) { /* we must have the input file */ in_name = argv[i]; /* remember name of input file */ input = fopen(in_name, "r"); if (input == NULL) /* check for open error */ err(1, "%s", in_name); continue; } else if (output == NULL) { /* we have the output file */ out_name = argv[i]; /* remember name of output file */ if (strcmp(in_name, out_name) == 0) { /* attempt to overwrite * the file */ errx(1, "input and output files must be different"); } output = fopen(out_name, "w"); if (output == NULL) /* check for create error */ err(1, "%s", out_name); continue; } errx(1, "unknown parameter: %s", argv[i]); } else set_option(argv[i]); } /* end of for */ if (input == NULL) input = stdin; if (output == NULL) { if (troff || input == stdin) output = stdout; else { out_name = in_name; bakcopy(); } } /* Restrict input/output descriptors and enter Capsicum sandbox. */ cap_rights_init(&rights, CAP_FSTAT, CAP_WRITE); if (cap_rights_limit(fileno(output), &rights) < 0 && errno != ENOSYS) err(EXIT_FAILURE, "unable to limit rights for %s", out_name); cap_rights_init(&rights, CAP_FSTAT, CAP_READ); if (cap_rights_limit(fileno(input), &rights) < 0 && errno != ENOSYS) err(EXIT_FAILURE, "unable to limit rights for %s", in_name); if (cap_enter() < 0 && errno != ENOSYS) err(EXIT_FAILURE, "unable to enter capability mode"); if (ps.com_ind <= 1) ps.com_ind = 2; /* dont put normal comments before column 2 */ if (troff) { if (bodyf.font[0] == 0) parsefont(&bodyf, "R"); if (scomf.font[0] == 0) parsefont(&scomf, "I"); if (blkcomf.font[0] == 0) blkcomf = scomf, blkcomf.size += 2; if (boxcomf.font[0] == 0) boxcomf = blkcomf; if (stringf.font[0] == 0) parsefont(&stringf, "L"); if (keywordf.font[0] == 0) parsefont(&keywordf, "B"); writefdef(&bodyf, 'B'); writefdef(&scomf, 'C'); writefdef(&blkcomf, 'L'); writefdef(&boxcomf, 'X'); writefdef(&stringf, 'S'); writefdef(&keywordf, 'K'); } if (block_comment_max_col <= 0) block_comment_max_col = max_col; if (ps.local_decl_indent < 0) /* if not specified by user, set this */ ps.local_decl_indent = ps.decl_indent; if (ps.decl_com_ind <= 0) /* if not specified by user, set this */ ps.decl_com_ind = ps.ljust_decl ? (ps.com_ind <= 10 ? 2 : ps.com_ind - 8) : ps.com_ind; if (continuation_indent == 0) continuation_indent = ps.ind_size; fill_buffer(); /* get first batch of stuff into input buffer */ parse(semicolon); { char *p = buf_ptr; int col = 1; while (1) { if (*p == ' ') col++; else if (*p == '\t') - col = ((col - 1) & ~7) + 9; + col = tabsize * (1 + (col - 1) / tabsize) + 1; else break; p++; } if (col > ps.ind_size) ps.ind_level = ps.i_l_follow = col / ps.ind_size; } if (troff) { const char *p = in_name, *beg = in_name; while (*p) if (*p++ == '/') beg = p; fprintf(output, ".Fn \"%s\"\n", beg); } /* * START OF MAIN LOOP */ while (1) { /* this is the main loop. it will go until we * reach eof */ int is_procname; type_code = lexi(); /* lexi reads one token. The actual * characters read are stored in "token". lexi * returns a code indicating the type of token */ is_procname = ps.procname[0]; /* * The following code moves everything following an if (), while (), * else, etc. up to the start of the following stmt to a buffer. This * allows proper handling of both kinds of brace placement. */ flushed_nl = false; while (ps.search_brace) { /* if we scanned an if(), while(), * etc., we might need to copy stuff * into a buffer we must loop, copying * stuff into save_com, until we find * the start of the stmt which follows * the if, or whatever */ switch (type_code) { case newline: ++line_no; if (sc_end != NULL) { /* dump comment, if any */ *sc_end++ = '\n'; /* newlines are needed in this case */ goto sw_buffer; } flushed_nl = true; case form_feed: break; /* form feeds and newlines found here will be * ignored */ case lbrace: /* this is a brace that starts the compound * stmt */ if (sc_end == NULL) { /* ignore buffering if a comment wasn't * stored up */ ps.search_brace = false; goto check_type; } if (btype_2) { save_com[0] = '{'; /* we either want to put the brace * right after the if */ goto sw_buffer; /* go to common code to get out of * this loop */ } case comment: /* we have a comment, so we must copy it into * the buffer */ if (!flushed_nl || sc_end != NULL) { if (sc_end == NULL) { /* if this is the first comment, we * must set up the buffer */ save_com[0] = save_com[1] = ' '; sc_end = &(save_com[2]); } else { *sc_end++ = '\n'; /* add newline between * comments */ *sc_end++ = ' '; --line_no; } *sc_end++ = '/'; /* copy in start of comment */ *sc_end++ = '*'; for (;;) { /* loop until we get to the end of the comment */ *sc_end = *buf_ptr++; if (buf_ptr >= buf_end) fill_buffer(); if (*sc_end++ == '*' && *buf_ptr == '/') break; /* we are at end of comment */ if (sc_end >= &(save_com[sc_size])) { /* check for temp buffer * overflow */ diag2(1, "Internal buffer overflow - Move big comment from right after if, while, or whatever"); fflush(output); exit(1); } } *sc_end++ = '/'; /* add ending slash */ if (++buf_ptr >= buf_end) /* get past / in buffer */ fill_buffer(); break; } default: /* it is the start of a normal statement */ if (flushed_nl) /* if we flushed a newline, make sure it is * put back */ force_nl = true; if ((type_code == sp_paren && *token == 'i' && last_else && ps.else_if) || (type_code == sp_nparen && *token == 'e' && e_code != s_code && e_code[-1] == '}')) force_nl = false; if (sc_end == NULL) { /* ignore buffering if comment wasn't * saved up */ ps.search_brace = false; goto check_type; } if (force_nl) { /* if we should insert a nl here, put it into * the buffer */ force_nl = false; --line_no; /* this will be re-increased when the nl is * read from the buffer */ *sc_end++ = '\n'; *sc_end++ = ' '; if (verbose && !flushed_nl) /* print error msg if the line * was not already broken */ diag2(0, "Line broken"); flushed_nl = false; } for (t_ptr = token; *t_ptr; ++t_ptr) *sc_end++ = *t_ptr; /* copy token into temp buffer */ ps.procname[0] = 0; sw_buffer: ps.search_brace = false; /* stop looking for start of * stmt */ bp_save = buf_ptr; /* save current input buffer */ be_save = buf_end; buf_ptr = save_com; /* fix so that subsequent calls to * lexi will take tokens out of * save_com */ *sc_end++ = ' ';/* add trailing blank, just in case */ buf_end = sc_end; sc_end = NULL; break; } /* end of switch */ if (type_code != 0) /* we must make this check, just in case there * was an unexpected EOF */ type_code = lexi(); /* read another token */ /* if (ps.search_brace) ps.procname[0] = 0; */ if ((is_procname = ps.procname[0]) && flushed_nl && !procnames_start_line && ps.in_decl && type_code == ident) flushed_nl = 0; } /* end of while (search_brace) */ last_else = 0; check_type: if (type_code == 0) { /* we got eof */ if (s_lab != e_lab || s_code != e_code || s_com != e_com) /* must dump end of line */ dump_line(); if (ps.tos > 1) /* check for balanced braces */ diag2(1, "Stuff missing from end of file"); if (verbose) { printf("There were %d output lines and %d comments\n", ps.out_lines, ps.out_coms); printf("(Lines with comments)/(Lines with code): %6.3f\n", (1.0 * ps.com_lines) / code_lines); } fflush(output); exit(found_err); } if ( (type_code != comment) && (type_code != newline) && (type_code != preesc) && (type_code != form_feed)) { if (force_nl && (type_code != semicolon) && (type_code != lbrace || !btype_2)) { /* we should force a broken line here */ if (verbose && !flushed_nl) diag2(0, "Line broken"); flushed_nl = false; dump_line(); ps.want_blank = false; /* dont insert blank at line start */ force_nl = false; } ps.in_stmt = true; /* turn on flag which causes an extra level of * indentation. this is turned off by a ; or * '}' */ if (s_com != e_com) { /* the turkey has embedded a comment * in a line. fix it */ *e_code++ = ' '; for (t_ptr = s_com; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = *t_ptr; } *e_code++ = ' '; *e_code = '\0'; /* null terminate code sect */ ps.want_blank = false; e_com = s_com; } } else if (type_code != comment) /* preserve force_nl thru a comment */ force_nl = false; /* cancel forced newline after newline, form * feed, etc */ /*-----------------------------------------------------*\ | do switch on type of token scanned | \*-----------------------------------------------------*/ CHECK_SIZE_CODE; switch (type_code) { /* now, decide what to do with the token */ case form_feed: /* found a form feed in line */ ps.use_ff = true; /* a form feed is treated much like a newline */ dump_line(); ps.want_blank = false; break; case newline: if (ps.last_token != comma || ps.p_l_follow > 0 || !ps.leave_comma || ps.block_init || !break_comma || s_com != e_com) { dump_line(); ps.want_blank = false; } ++line_no; /* keep track of input line number */ break; case lparen: /* got a '(' or '[' */ /* count parens to make Healy happy */ if (++ps.p_l_follow == nitems(ps.paren_indents)) { diag3(0, "Reached internal limit of %d unclosed parens", nitems(ps.paren_indents)); ps.p_l_follow--; } if (ps.want_blank && *token != '[' && ((ps.last_token != ident && ps.last_token != funcname) || proc_calls_space || /* offsetof (1) is never allowed a space; sizeof (2) gets * one iff -bs; all other keywords (>2) always get a space * before lparen */ ps.keyword + Bill_Shannon > 2)) *e_code++ = ' '; ps.want_blank = false; if (ps.in_decl && !ps.block_init && !ps.dumped_decl_indent && !is_procname) { /* function pointer declarations */ if (troff) { sprintf(e_code, "\n.Du %dp+\200p \"%s\"\n", dec_ind * 7, token); e_code += strlen(e_code); } else { indent_declaration(dec_ind, tabs_to_var); } ps.dumped_decl_indent = true; } if (!troff) *e_code++ = token[0]; ps.paren_indents[ps.p_l_follow - 1] = e_code - s_code; if (sp_sw && ps.p_l_follow == 1 && extra_expression_indent && ps.paren_indents[0] < 2 * ps.ind_size) ps.paren_indents[0] = 2 * ps.ind_size; if (ps.in_or_st && *token == '(' && ps.tos <= 2) { /* * this is a kluge to make sure that declarations will be * aligned right if proc decl has an explicit type on it, i.e. * "int a(x) {..." */ parse(semicolon); /* I said this was a kluge... */ ps.in_or_st = false; /* turn off flag for structure decl or * initialization */ } /* parenthesized type following sizeof or offsetof is not a cast */ if (ps.keyword == 1 || ps.keyword == 2) ps.not_cast_mask |= 1 << ps.p_l_follow; break; case rparen: /* got a ')' or ']' */ if (ps.cast_mask & (1 << ps.p_l_follow) & ~ps.not_cast_mask) { ps.last_u_d = true; ps.cast_mask &= (1 << ps.p_l_follow) - 1; ps.want_blank = space_after_cast; } else ps.want_blank = true; ps.not_cast_mask &= (1 << ps.p_l_follow) - 1; if (--ps.p_l_follow < 0) { ps.p_l_follow = 0; diag3(0, "Extra %c", *token); } if (e_code == s_code) /* if the paren starts the line */ ps.paren_level = ps.p_l_follow; /* then indent it */ *e_code++ = token[0]; if (sp_sw && (ps.p_l_follow == 0)) { /* check for end of if * (...), or some such */ sp_sw = false; force_nl = true;/* must force newline after if */ ps.last_u_d = true; /* inform lexi that a following * operator is unary */ ps.in_stmt = false; /* dont use stmt continuation * indentation */ parse(hd_type); /* let parser worry about if, or whatever */ } ps.search_brace = btype_2; /* this should insure that constructs * such as main(){...} and int[]{...} * have their braces put in the right * place */ break; case unary_op: /* this could be any unary operation */ if (!ps.dumped_decl_indent && ps.in_decl && !is_procname && !ps.block_init) { /* pointer declarations */ if (troff) { if (ps.want_blank) *e_code++ = ' '; sprintf(e_code, "\n.Du %dp+\200p \"%s\"\n", dec_ind * 7, token); e_code += strlen(e_code); } else { /* if this is a unary op in a declaration, we should * indent this token */ for (i = 0; token[i]; ++i) /* find length of token */; indent_declaration(dec_ind - i, tabs_to_var); } ps.dumped_decl_indent = true; } else if (ps.want_blank) *e_code++ = ' '; { const char *res = token; if (troff && token[0] == '-' && token[1] == '>') res = "\\(->"; for (t_ptr = res; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = *t_ptr; } } ps.want_blank = false; break; case binary_op: /* any binary operation */ if (ps.want_blank) *e_code++ = ' '; { const char *res = token; if (troff) switch (token[0]) { case '<': if (token[1] == '=') res = "\\(<="; break; case '>': if (token[1] == '=') res = "\\(>="; break; case '!': if (token[1] == '=') res = "\\(!="; break; case '|': if (token[1] == '|') res = "\\(br\\(br"; else if (token[1] == 0) res = "\\(br"; break; } for (t_ptr = res; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = *t_ptr; /* move the operator */ } } ps.want_blank = true; break; case postop: /* got a trailing ++ or -- */ *e_code++ = token[0]; *e_code++ = token[1]; ps.want_blank = true; break; case question: /* got a ? */ squest++; /* this will be used when a later colon * appears so we can distinguish the * ?: construct */ if (ps.want_blank) *e_code++ = ' '; *e_code++ = '?'; ps.want_blank = true; break; case casestmt: /* got word 'case' or 'default' */ scase = true; /* so we can process the later colon properly */ goto copy_id; case colon: /* got a ':' */ if (squest > 0) { /* it is part of the ?: construct */ --squest; if (ps.want_blank) *e_code++ = ' '; *e_code++ = ':'; ps.want_blank = true; break; } if (ps.in_or_st) { *e_code++ = ':'; ps.want_blank = false; break; } ps.in_stmt = false; /* seeing a label does not imply we are in a * stmt */ for (t_ptr = s_code; *t_ptr; ++t_ptr) *e_lab++ = *t_ptr; /* turn everything so far into a label */ e_code = s_code; *e_lab++ = ':'; *e_lab++ = ' '; *e_lab = '\0'; force_nl = ps.pcase = scase; /* ps.pcase will be used by * dump_line to decide how to * indent the label. force_nl * will force a case n: to be * on a line by itself */ scase = false; ps.want_blank = false; break; case semicolon: /* got a ';' */ if (ps.dec_nest == 0) ps.in_or_st = false;/* we are not in an initialization or * structure declaration */ scase = false; /* these will only need resetting in an error */ squest = 0; if (ps.last_token == rparen) ps.in_parameter_declaration = 0; ps.cast_mask = 0; ps.not_cast_mask = 0; ps.block_init = 0; ps.block_init_level = 0; ps.just_saw_decl--; if (ps.in_decl && s_code == e_code && !ps.block_init && !ps.dumped_decl_indent) { /* indent stray semicolons in declarations */ indent_declaration(dec_ind - 1, tabs_to_var); ps.dumped_decl_indent = true; } ps.in_decl = (ps.dec_nest > 0); /* if we were in a first level * structure declaration, we * arent any more */ if ((!sp_sw || hd_type != forstmt) && ps.p_l_follow > 0) { /* * This should be true iff there were unbalanced parens in the * stmt. It is a bit complicated, because the semicolon might * be in a for stmt */ diag2(1, "Unbalanced parens"); ps.p_l_follow = 0; if (sp_sw) { /* this is a check for an if, while, etc. with * unbalanced parens */ sp_sw = false; parse(hd_type); /* dont lose the if, or whatever */ } } *e_code++ = ';'; ps.want_blank = true; ps.in_stmt = (ps.p_l_follow > 0); /* we are no longer in the * middle of a stmt */ if (!sp_sw) { /* if not if for (;;) */ parse(semicolon); /* let parser know about end of stmt */ force_nl = true;/* force newline after an end of stmt */ } break; case lbrace: /* got a '{' */ ps.in_stmt = false; /* dont indent the {} */ if (!ps.block_init) force_nl = true;/* force other stuff on same line as '{' onto * new line */ else if (ps.block_init_level <= 0) ps.block_init_level = 1; else ps.block_init_level++; if (s_code != e_code && !ps.block_init) { if (!btype_2) { dump_line(); ps.want_blank = false; } else if (ps.in_parameter_declaration && !ps.in_or_st) { ps.i_l_follow = 0; if (function_brace_split) { /* dump the line prior to the * brace ... */ dump_line(); ps.want_blank = false; } else /* add a space between the decl and brace */ ps.want_blank = true; } } if (ps.in_parameter_declaration) prefix_blankline_requested = 0; if (ps.p_l_follow > 0) { /* check for preceding unbalanced * parens */ diag2(1, "Unbalanced parens"); ps.p_l_follow = 0; if (sp_sw) { /* check for unclosed if, for, etc. */ sp_sw = false; parse(hd_type); ps.ind_level = ps.i_l_follow; } } if (s_code == e_code) ps.ind_stmt = false; /* dont put extra indentation on line * with '{' */ if (ps.in_decl && ps.in_or_st) { /* this is either a structure * declaration or an init */ di_stack[ps.dec_nest++] = dec_ind; /* ? dec_ind = 0; */ } else { ps.decl_on_line = false; /* we can't be in the middle of * a declaration, so don't do * special indentation of * comments */ if (blanklines_after_declarations_at_proctop && ps.in_parameter_declaration) postfix_blankline_requested = 1; ps.in_parameter_declaration = 0; ps.in_decl = false; } dec_ind = 0; parse(lbrace); /* let parser know about this */ if (ps.want_blank) /* put a blank before '{' if '{' is not at * start of line */ *e_code++ = ' '; ps.want_blank = false; *e_code++ = '{'; ps.just_saw_decl = 0; break; case rbrace: /* got a '}' */ if (ps.p_stack[ps.tos] == decl && !ps.block_init) /* semicolons can be * omitted in * declarations */ parse(semicolon); if (ps.p_l_follow) {/* check for unclosed if, for, else. */ diag2(1, "Unbalanced parens"); ps.p_l_follow = 0; sp_sw = false; } ps.just_saw_decl = 0; ps.block_init_level--; if (s_code != e_code && !ps.block_init) { /* '}' must be first on * line */ if (verbose) diag2(0, "Line broken"); dump_line(); } *e_code++ = '}'; ps.want_blank = true; ps.in_stmt = ps.ind_stmt = false; if (ps.dec_nest > 0) { /* we are in multi-level structure * declaration */ dec_ind = di_stack[--ps.dec_nest]; if (ps.dec_nest == 0 && !ps.in_parameter_declaration) ps.just_saw_decl = 2; ps.in_decl = true; } prefix_blankline_requested = 0; parse(rbrace); /* let parser know about this */ ps.search_brace = cuddle_else && ps.p_stack[ps.tos] == ifhead && ps.il[ps.tos] >= ps.ind_level; if (ps.tos <= 1 && blanklines_after_procs && ps.dec_nest <= 0) postfix_blankline_requested = 1; break; case swstmt: /* got keyword "switch" */ sp_sw = true; hd_type = swstmt; /* keep this for when we have seen the * expression */ goto copy_id; /* go move the token into buffer */ case sp_paren: /* token is if, while, for */ sp_sw = true; /* the interesting stuff is done after the * expression is scanned */ hd_type = (*token == 'i' ? ifstmt : (*token == 'w' ? whilestmt : forstmt)); /* * remember the type of header for later use by parser */ goto copy_id; /* copy the token into line */ case sp_nparen: /* got else, do */ ps.in_stmt = false; if (*token == 'e') { if (e_code != s_code && (!cuddle_else || e_code[-1] != '}')) { if (verbose) diag2(0, "Line broken"); dump_line();/* make sure this starts a line */ ps.want_blank = false; } force_nl = true;/* also, following stuff must go onto new line */ last_else = 1; parse(elselit); } else { if (e_code != s_code) { /* make sure this starts a line */ if (verbose) diag2(0, "Line broken"); dump_line(); ps.want_blank = false; } force_nl = true;/* also, following stuff must go onto new line */ last_else = 0; parse(dolit); } goto copy_id; /* move the token into line */ case storage: prefix_blankline_requested = 0; goto copy_id; case decl: /* we have a declaration type (int, etc.) */ parse(decl); /* let parser worry about indentation */ if (ps.last_token == rparen && ps.tos <= 1) { if (s_code != e_code) { dump_line(); ps.want_blank = 0; } } if (ps.in_parameter_declaration && ps.indent_parameters && ps.dec_nest == 0) { ps.ind_level = ps.i_l_follow = 1; ps.ind_stmt = 0; } ps.in_or_st = true; /* this might be a structure or initialization * declaration */ ps.in_decl = ps.decl_on_line = true; if ( /* !ps.in_or_st && */ ps.dec_nest <= 0) ps.just_saw_decl = 2; prefix_blankline_requested = 0; for (i = 0; token[i++];); /* get length of token */ if (ps.ind_level == 0 || ps.dec_nest > 0) { /* global variable or struct member in local variable */ dec_ind = ps.decl_indent > 0 ? ps.decl_indent : i; tabs_to_var = (use_tabs ? ps.decl_indent > 0 : 0); } else { /* local variable */ dec_ind = ps.local_decl_indent > 0 ? ps.local_decl_indent : i; tabs_to_var = (use_tabs ? ps.local_decl_indent > 0 : 0); } goto copy_id; case funcname: case ident: /* got an identifier or constant */ if (ps.in_decl) { /* if we are in a declaration, we must indent * identifier */ if (type_code != funcname || !procnames_start_line) { if (!ps.block_init && !ps.dumped_decl_indent) { if (troff) { if (ps.want_blank) *e_code++ = ' '; sprintf(e_code, "\n.De %dp+\200p\n", dec_ind * 7); e_code += strlen(e_code); } else indent_declaration(dec_ind, tabs_to_var); ps.dumped_decl_indent = true; ps.want_blank = false; } } else { if (ps.want_blank && !(procnames_start_line && type_code == funcname)) *e_code++ = ' '; ps.want_blank = false; if (dec_ind && s_code != e_code) { *e_code = '\0'; dump_line(); } dec_ind = 0; } } else if (sp_sw && ps.p_l_follow == 0) { sp_sw = false; force_nl = true; ps.last_u_d = true; ps.in_stmt = false; parse(hd_type); } copy_id: if (ps.want_blank) *e_code++ = ' '; if (troff && ps.keyword) { e_code = chfont(&bodyf, &keywordf, e_code); for (t_ptr = token; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = keywordf.allcaps && islower(*t_ptr) ? toupper(*t_ptr) : *t_ptr; } e_code = chfont(&keywordf, &bodyf, e_code); } else for (t_ptr = token; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = *t_ptr; } if (type_code != funcname) ps.want_blank = true; break; case strpfx: if (ps.want_blank) *e_code++ = ' '; for (t_ptr = token; *t_ptr; ++t_ptr) { CHECK_SIZE_CODE; *e_code++ = *t_ptr; } ps.want_blank = false; break; case period: /* treat a period kind of like a binary * operation */ *e_code++ = '.'; /* move the period into line */ ps.want_blank = false; /* dont put a blank after a period */ break; case comma: ps.want_blank = (s_code != e_code); /* only put blank after comma * if comma does not start the * line */ if (ps.in_decl && is_procname == 0 && !ps.block_init && !ps.dumped_decl_indent) { /* indent leading commas and not the actual identifiers */ indent_declaration(dec_ind - 1, tabs_to_var); ps.dumped_decl_indent = true; } *e_code++ = ','; if (ps.p_l_follow == 0) { if (ps.block_init_level <= 0) ps.block_init = 0; - if (break_comma && (!ps.leave_comma || compute_code_target() + (e_code - s_code) > max_col - 8)) + if (break_comma && (!ps.leave_comma || compute_code_target() + (e_code - s_code) > max_col - tabsize)) force_nl = true; } break; case preesc: /* got the character '#' */ if ((s_com != e_com) || (s_lab != e_lab) || (s_code != e_code)) dump_line(); *e_lab++ = '#'; /* move whole line to 'label' buffer */ { int in_comment = 0; int com_start = 0; char quote = 0; int com_end = 0; while (*buf_ptr == ' ' || *buf_ptr == '\t') { buf_ptr++; if (buf_ptr >= buf_end) fill_buffer(); } while (*buf_ptr != '\n' || (in_comment && !had_eof)) { CHECK_SIZE_LAB; *e_lab = *buf_ptr++; if (buf_ptr >= buf_end) fill_buffer(); switch (*e_lab++) { case BACKSLASH: if (troff) *e_lab++ = BACKSLASH; if (!in_comment) { *e_lab++ = *buf_ptr++; if (buf_ptr >= buf_end) fill_buffer(); } break; case '/': if (*buf_ptr == '*' && !in_comment && !quote) { in_comment = 1; *e_lab++ = *buf_ptr++; com_start = e_lab - s_lab - 2; } break; case '"': if (quote == '"') quote = 0; break; case '\'': if (quote == '\'') quote = 0; break; case '*': if (*buf_ptr == '/' && in_comment) { in_comment = 0; *e_lab++ = *buf_ptr++; com_end = e_lab - s_lab; } break; } } while (e_lab > s_lab && (e_lab[-1] == ' ' || e_lab[-1] == '\t')) e_lab--; if (e_lab - s_lab == com_end && bp_save == NULL) { /* comment on preprocessor line */ if (sc_end == NULL) /* if this is the first comment, we * must set up the buffer */ sc_end = &(save_com[0]); else { *sc_end++ = '\n'; /* add newline between * comments */ *sc_end++ = ' '; --line_no; } bcopy(s_lab + com_start, sc_end, com_end - com_start); sc_end += com_end - com_start; if (sc_end >= &save_com[sc_size]) abort(); e_lab = s_lab + com_start; while (e_lab > s_lab && (e_lab[-1] == ' ' || e_lab[-1] == '\t')) e_lab--; bp_save = buf_ptr; /* save current input buffer */ be_save = buf_end; buf_ptr = save_com; /* fix so that subsequent calls to * lexi will take tokens out of * save_com */ *sc_end++ = ' '; /* add trailing blank, just in case */ buf_end = sc_end; sc_end = NULL; } *e_lab = '\0'; /* null terminate line */ ps.pcase = false; } if (strncmp(s_lab, "#if", 3) == 0) { /* also ifdef, ifndef */ if ((size_t)ifdef_level < nitems(state_stack)) { match_state[ifdef_level].tos = -1; state_stack[ifdef_level++] = ps; } else diag2(1, "#if stack overflow"); } else if (strncmp(s_lab, "#el", 3) == 0) { /* else, elif */ if (ifdef_level <= 0) diag2(1, s_lab[3] == 'i' ? "Unmatched #elif" : "Unmatched #else"); else { match_state[ifdef_level - 1] = ps; ps = state_stack[ifdef_level - 1]; } } else if (strncmp(s_lab, "#endif", 6) == 0) { if (ifdef_level <= 0) diag2(1, "Unmatched #endif"); else ifdef_level--; } else { struct directives { int size; const char *string; } recognized[] = { {7, "include"}, {6, "define"}, {5, "undef"}, {4, "line"}, {5, "error"}, {6, "pragma"} }; int d = nitems(recognized); while (--d >= 0) if (strncmp(s_lab + 1, recognized[d].string, recognized[d].size) == 0) break; if (d < 0) { diag2(1, "Unrecognized cpp directive"); break; } } if (blanklines_around_conditional_compilation) { postfix_blankline_requested++; n_real_blanklines = 0; } else { postfix_blankline_requested = 0; prefix_blankline_requested = 0; } break; /* subsequent processing of the newline * character will cause the line to be printed */ case comment: /* we have gotten a / followed by * this is a biggie */ if (flushed_nl) { /* we should force a broken line here */ dump_line(); ps.want_blank = false; /* dont insert blank at line start */ force_nl = false; } pr_comment(); break; } /* end of big switch stmt */ *e_code = '\0'; /* make sure code section is null terminated */ if (type_code != comment && type_code != newline && type_code != preesc) ps.last_token = type_code; } /* end of main while (1) loop */ } /* * copy input file to backup file if in_name is /blah/blah/blah/file, then * backup file will be ".Bfile" then make the backup file the input and * original input file the output */ static void bakcopy(void) { int n, bakchn; char buff[8 * 1024]; const char *p; /* construct file name .Bfile */ for (p = in_name; *p; p++); /* skip to end of string */ while (p > in_name && *p != '/') /* find last '/' */ p--; if (*p == '/') p++; sprintf(bakfile, "%s.BAK", p); /* copy in_name to backup file */ bakchn = creat(bakfile, 0600); if (bakchn < 0) err(1, "%s", bakfile); while ((n = read(fileno(input), buff, sizeof(buff))) > 0) if (write(bakchn, buff, n) != n) err(1, "%s", bakfile); if (n < 0) err(1, "%s", in_name); close(bakchn); fclose(input); /* re-open backup file as the input file */ input = fopen(bakfile, "r"); if (input == NULL) err(1, "%s", bakfile); /* now the original input file will be the output */ output = fopen(in_name, "w"); if (output == NULL) { unlink(bakfile); err(1, "%s", in_name); } } static void indent_declaration(int cur_dec_ind, int tabs_to_var) { int pos = e_code - s_code; char *startpos = e_code; /* - * get the tab math right for indentations that are not multiples of 8 + * get the tab math right for indentations that are not multiples of tabsize */ - if ((ps.ind_level * ps.ind_size) % 8 != 0) { - pos += (ps.ind_level * ps.ind_size) % 8; - cur_dec_ind += (ps.ind_level * ps.ind_size) % 8; + if ((ps.ind_level * ps.ind_size) % tabsize != 0) { + pos += (ps.ind_level * ps.ind_size) % tabsize; + cur_dec_ind += (ps.ind_level * ps.ind_size) % tabsize; } - if (tabs_to_var) - while ((pos & ~7) + 8 <= cur_dec_ind) { + if (tabs_to_var) { + int tpos; + + while ((tpos = tabsize * (1 + pos / tabsize)) <= cur_dec_ind) { CHECK_SIZE_CODE; *e_code++ = '\t'; - pos = (pos & ~7) + 8; + pos = tpos; } + } while (pos < cur_dec_ind) { CHECK_SIZE_CODE; *e_code++ = ' '; pos++; } if (e_code == startpos && ps.want_blank) { *e_code++ = ' '; ps.want_blank = false; } } Index: projects/runtime-coverage/usr.bin/indent/indent_globs.h =================================================================== --- projects/runtime-coverage/usr.bin/indent/indent_globs.h (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/indent_globs.h (revision 321585) @@ -1,334 +1,331 @@ /*- * Copyright (c) 1985 Sun Microsystems, Inc. * Copyright (c) 1980, 1993 * The Regents of the University of California. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)indent_globs.h 8.1 (Berkeley) 6/6/93 * $FreeBSD$ */ #define BACKSLASH '\\' #define bufsize 200 /* size of internal buffers */ #define sc_size 5000 /* size of save_com buffer */ #define label_offset 2 /* number of levels a label is placed to left * of code */ -#define tabsize 8 /* the size of a tab */ -#define tabmask 0177770 /* mask used when figuring length of lines - * with tabs */ - #define false 0 #define true 1 FILE *input; /* the fid for the input file */ FILE *output; /* the output file */ #define CHECK_SIZE_CODE \ if (e_code >= l_code) { \ int nsize = l_code-s_code+400; \ int code_len = e_code-s_code; \ codebuf = (char *) realloc(codebuf, nsize); \ if (codebuf == NULL) \ err(1, NULL); \ e_code = codebuf + code_len + 1; \ l_code = codebuf + nsize - 5; \ s_code = codebuf + 1; \ } #define CHECK_SIZE_COM \ if (e_com >= l_com) { \ int nsize = l_com-s_com+400; \ int com_len = e_com - s_com; \ int blank_pos = last_bl - s_com; \ combuf = (char *) realloc(combuf, nsize); \ if (combuf == NULL) \ err(1, NULL); \ e_com = combuf + com_len + 1; \ last_bl = combuf + blank_pos + 1; \ l_com = combuf + nsize - 5; \ s_com = combuf + 1; \ } #define CHECK_SIZE_LAB \ if (e_lab >= l_lab) { \ int nsize = l_lab-s_lab+400; \ int label_len = e_lab - s_lab; \ labbuf = (char *) realloc(labbuf, nsize); \ if (labbuf == NULL) \ err(1, NULL); \ e_lab = labbuf + label_len + 1; \ l_lab = labbuf + nsize - 5; \ s_lab = labbuf + 1; \ } #define CHECK_SIZE_TOKEN \ if (e_token >= l_token) { \ int nsize = l_token-s_token+400; \ int token_len = e_token - s_token; \ tokenbuf = (char *) realloc(tokenbuf, nsize); \ if (tokenbuf == NULL) \ err(1, NULL); \ e_token = tokenbuf + token_len + 1; \ l_token = tokenbuf + nsize - 5; \ s_token = tokenbuf + 1; \ } char *labbuf; /* buffer for label */ char *s_lab; /* start ... */ char *e_lab; /* .. and end of stored label */ char *l_lab; /* limit of label buffer */ char *codebuf; /* buffer for code section */ char *s_code; /* start ... */ char *e_code; /* .. and end of stored code */ char *l_code; /* limit of code section */ char *combuf; /* buffer for comments */ char *s_com; /* start ... */ char *e_com; /* ... and end of stored comments */ char *l_com; /* limit of comment buffer */ #define token s_token char *tokenbuf; /* the last token scanned */ char *s_token; char *e_token; char *l_token; char *in_buffer; /* input buffer */ char *in_buffer_limit; /* the end of the input buffer */ char *buf_ptr; /* ptr to next character to be taken from * in_buffer */ char *buf_end; /* ptr to first after last char in in_buffer */ char save_com[sc_size]; /* input text is saved here when looking for * the brace after an if, while, etc */ char *sc_end; /* pointer into save_com buffer */ char *bp_save; /* saved value of buf_ptr when taking input * from save_com */ char *be_save; /* similarly saved value of buf_end */ int found_err; int pointer_as_binop; int blanklines_after_declarations; int blanklines_before_blockcomments; int blanklines_after_procs; int blanklines_around_conditional_compilation; int swallow_optional_blanklines; int n_real_blanklines; int prefix_blankline_requested; int postfix_blankline_requested; int break_comma; /* when true and not in parens, break after a * comma */ int btype_2; /* when true, brace should be on same line as * if, while, etc */ float case_ind; /* indentation level to be used for a "case * n:" */ int code_lines; /* count of lines with code */ int had_eof; /* set to true when input is exhausted */ int line_no; /* the current line number. */ int max_col; /* the maximum allowable line length */ int verbose; /* when true, non-essential error messages are * printed */ int cuddle_else; /* true if else should cuddle up to '}' */ int star_comment_cont; /* true iff comment continuation lines should * have stars at the beginning of each line. */ int comment_delimiter_on_blankline; int troff; /* true iff were generating troff input */ int procnames_start_line; /* if true, the names of procedures * being defined get placed in column * 1 (ie. a newline is placed between * the type of the procedure and its * name) */ int proc_calls_space; /* If true, procedure calls look like: * foo(bar) rather than foo (bar) */ int format_block_comments; /* true if comments beginning with * `/ * \n' are to be reformatted */ int format_col1_comments; /* If comments which start in column 1 * are to be magically reformatted * (just like comments that begin in * later columns) */ int inhibit_formatting; /* true if INDENT OFF is in effect */ int suppress_blanklines;/* set iff following blanklines should be * suppressed */ int continuation_indent;/* set to the indentation between the edge of * code and continuation lines */ int lineup_to_parens; /* if true, continued code within parens will * be lined up to the open paren */ int Bill_Shannon; /* true iff a blank should always be inserted * after sizeof */ int blanklines_after_declarations_at_proctop; /* This is vaguely * similar to * blanklines_after_decla * rations except that * it only applies to * the first set of * declarations in a * procedure (just after * the first '{') and it * causes a blank line * to be generated even * if there are no * declarations */ int block_comment_max_col; int extra_expression_indent; /* true if continuation lines from the * expression part of "if(e)", * "while(e)", "for(e;e;e)" should be * indented an extra tab stop so that * they don't conflict with the code * that follows */ int function_brace_split; /* split function declaration and * brace onto separate lines */ int use_tabs; /* set true to use tabs for spacing, * false uses all spaces */ int auto_typedefs; /* set true to recognize identifiers * ending in "_t" like typedefs */ int space_after_cast; /* "b = (int) a" vs "b = (int)a" */ +int tabsize; /* the size of a tab */ /* -troff font state information */ struct fstate { char font[4]; char size; int allcaps:1; } __aligned(sizeof(int)); char *chfont(struct fstate *, struct fstate *, char *); struct fstate keywordf, /* keyword font */ stringf, /* string font */ boxcomf, /* Box comment font */ blkcomf, /* Block comment font */ scomf, /* Same line comment font */ bodyf; /* major body font */ #define STACKSIZE 256 struct parser_state { int last_token; struct fstate cfont; /* Current font */ int p_stack[STACKSIZE]; /* this is the parsers stack */ int il[STACKSIZE]; /* this stack stores indentation levels */ float cstk[STACKSIZE];/* used to store case stmt indentation levels */ int box_com; /* set to true when we are in a "boxed" * comment. In that case, the first non-blank * char should be lined up with the / in / followed by * */ int comment_delta, n_comment_delta; int cast_mask; /* indicates which close parens potentially * close off casts */ int not_cast_mask; /* indicates which close parens definitely * close off something else than casts */ int block_init; /* true iff inside a block initialization */ int block_init_level; /* The level of brace nesting in an * initialization */ int last_nl; /* this is true if the last thing scanned was * a newline */ int in_or_st; /* Will be true iff there has been a * declarator (e.g. int or char) and no left * paren since the last semicolon. When true, * a '{' is starting a structure definition or * an initialization list */ int bl_line; /* set to 1 by dump_line if the line is blank */ int col_1; /* set to true if the last token started in * column 1 */ int com_col; /* this is the column in which the current * comment should start */ int com_ind; /* the column in which comments to the right * of code should start */ int com_lines; /* the number of lines with comments, set by * dump_line */ int dec_nest; /* current nesting level for structure or init */ int decl_com_ind; /* the column in which comments after * declarations should be put */ int decl_on_line; /* set to true if this line of code has part * of a declaration on it */ int i_l_follow; /* the level to which ind_level should be set * after the current line is printed */ int in_decl; /* set to true when we are in a declaration * stmt. The processing of braces is then * slightly different */ int in_stmt; /* set to 1 while in a stmt */ int ind_level; /* the current indentation level */ int ind_size; /* the size of one indentation level */ int ind_stmt; /* set to 1 if next line should have an extra * indentation level because we are in the * middle of a stmt */ int last_u_d; /* set to true after scanning a token which * forces a following operator to be unary */ int leave_comma; /* if true, never break declarations after * commas */ int ljust_decl; /* true if declarations should be left * justified */ int out_coms; /* the number of comments processed, set by * pr_comment */ int out_lines; /* the number of lines written, set by * dump_line */ int p_l_follow; /* used to remember how to indent following * statement */ int paren_level; /* parenthesization level. used to indent * within statements */ short paren_indents[20]; /* column positions of each paren */ int pcase; /* set to 1 if the current line label is a * case. It is printed differently from a * regular label */ int search_brace; /* set to true by parse when it is necessary * to buffer up all info up to the start of a * stmt after an if, while, etc */ int unindent_displace; /* comments not to the right of code * will be placed this many * indentation levels to the left of * code */ int use_ff; /* set to one if the current line should be * terminated with a form feed */ int want_blank; /* set to true when the following token should * be prefixed by a blank. (Said prefixing is * ignored in some cases.) */ int else_if; /* True iff else if pairs should be handled * specially */ int decl_indent; /* column to indent declared identifiers to */ int local_decl_indent; /* like decl_indent but for locals */ int keyword; /* the type of a keyword or 0 */ int dumped_decl_indent; float case_indent; /* The distance to indent case labels from the * switch statement */ int in_parameter_declaration; int indent_parameters; int tos; /* pointer to top of stack */ char procname[100]; /* The name of the current procedure */ int just_saw_decl; } ps; int ifdef_level; struct parser_state state_stack[5]; struct parser_state match_state[5]; Index: projects/runtime-coverage/usr.bin/indent/io.c =================================================================== --- projects/runtime-coverage/usr.bin/indent/io.c (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/io.c (revision 321585) @@ -1,665 +1,667 @@ /*- * Copyright (c) 1985 Sun Microsystems, Inc. * Copyright (c) 1980, 1993 * The Regents of the University of California. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if 0 #ifndef lint static char sccsid[] = "@(#)io.c 8.1 (Berkeley) 6/6/93"; #endif /* not lint */ #endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "indent_globs.h" #include "indent.h" int comment_open; static int paren_target; static int pad_output(int current, int target); void dump_line(void) { /* dump_line is the routine that actually * effects the printing of the new source. It * prints the label section, followed by the * code section with the appropriate nesting * level, followed by any comments */ int cur_col, target_col = 1; static int not_first_line; if (ps.procname[0]) { if (troff) { if (comment_open) { comment_open = 0; fprintf(output, ".*/\n"); } fprintf(output, ".Pr \"%s\"\n", ps.procname); } ps.ind_level = 0; ps.procname[0] = 0; } if (s_code == e_code && s_lab == e_lab && s_com == e_com) { if (suppress_blanklines > 0) suppress_blanklines--; else { ps.bl_line = true; n_real_blanklines++; } } else if (!inhibit_formatting) { suppress_blanklines = 0; ps.bl_line = false; if (prefix_blankline_requested && not_first_line) { if (swallow_optional_blanklines) { if (n_real_blanklines == 1) n_real_blanklines = 0; } else { if (n_real_blanklines == 0) n_real_blanklines = 1; } } while (--n_real_blanklines >= 0) putc('\n', output); n_real_blanklines = 0; if (ps.ind_level == 0) ps.ind_stmt = 0; /* this is a class A kludge. dont do * additional statement indentation if we are * at bracket level 0 */ if (e_lab != s_lab || e_code != s_code) ++code_lines; /* keep count of lines with code */ if (e_lab != s_lab) { /* print lab, if any */ if (comment_open) { comment_open = 0; fprintf(output, ".*/\n"); } while (e_lab > s_lab && (e_lab[-1] == ' ' || e_lab[-1] == '\t')) e_lab--; *e_lab = '\0'; cur_col = pad_output(1, compute_label_target()); if (s_lab[0] == '#' && (strncmp(s_lab, "#else", 5) == 0 || strncmp(s_lab, "#endif", 6) == 0)) { char *s = s_lab; if (e_lab[-1] == '\n') e_lab--; do putc(*s++, output); while (s < e_lab && 'a' <= *s && *s<='z'); while ((*s == ' ' || *s == '\t') && s < e_lab) s++; if (s < e_lab) fprintf(output, s[0]=='/' && s[1]=='*' ? "\t%.*s" : "\t/* %.*s */", (int)(e_lab - s), s); } else fprintf(output, "%.*s", (int)(e_lab - s_lab), s_lab); cur_col = count_spaces(cur_col, s_lab); } else cur_col = 1; /* there is no label section */ ps.pcase = false; if (s_code != e_code) { /* print code section, if any */ char *p; if (comment_open) { comment_open = 0; fprintf(output, ".*/\n"); } target_col = compute_code_target(); { int i; for (i = 0; i < ps.p_l_follow; i++) if (ps.paren_indents[i] >= 0) ps.paren_indents[i] = -(ps.paren_indents[i] + target_col); } cur_col = pad_output(cur_col, target_col); for (p = s_code; p < e_code; p++) if (*p == (char) 0200) fprintf(output, "%d", target_col * 7); else putc(*p, output); cur_col = count_spaces(cur_col, s_code); } if (s_com != e_com) { if (troff) { int all_here = 0; char *p; if (e_com[-1] == '/' && e_com[-2] == '*') e_com -= 2, all_here++; while (e_com > s_com && e_com[-1] == ' ') e_com--; *e_com = 0; p = s_com; while (*p == ' ') p++; if (p[0] == '/' && p[1] == '*') p += 2, all_here++; else if (p[0] == '*') p += p[1] == '/' ? 2 : 1; while (*p == ' ') p++; if (*p == 0) goto inhibit_newline; if (comment_open < 2 && ps.box_com) { comment_open = 0; fprintf(output, ".*/\n"); } if (comment_open == 0) { if ('a' <= *p && *p <= 'z') *p = *p + 'A' - 'a'; if (e_com - p < 50 && all_here == 2) { char *follow = p; fprintf(output, "\n.nr C! \\w\1"); while (follow < e_com) { switch (*follow) { case '\n': putc(' ', output); case 1: break; case '\\': putc('\\', output); /* add a backslash to escape the '\' */ default: putc(*follow, output); } follow++; } putc(1, output); } fprintf(output, "\n./* %dp %d %dp\n", ps.com_col * 7, (s_code != e_code || s_lab != e_lab) - ps.box_com, target_col * 7); } comment_open = 1 + ps.box_com; while (*p) { if (*p == BACKSLASH) putc(BACKSLASH, output); putc(*p++, output); } } else { /* print comment, if any */ int target = ps.com_col; char *com_st = s_com; target += ps.comment_delta; while (*com_st == '\t') /* consider original indentation in * case this is a box comment */ - com_st++, target += 8; + com_st++, target += tabsize; while (target <= 0) if (*com_st == ' ') target++, com_st++; else if (*com_st == '\t') - target = ((target - 1) & ~7) + 9, com_st++; + target = tabsize * (1 + (target - 1) / tabsize) + 1, com_st++; else target = 1; if (cur_col > target) { /* if comment can't fit on this line, * put it on next line */ putc('\n', output); cur_col = 1; ++ps.out_lines; } while (e_com > com_st && isspace(e_com[-1])) e_com--; (void)pad_output(cur_col, target); fwrite(com_st, e_com - com_st, 1, output); ps.comment_delta = ps.n_comment_delta; ++ps.com_lines; /* count lines with comments */ } } if (ps.use_ff) putc('\014', output); else putc('\n', output); inhibit_newline: ++ps.out_lines; if (ps.just_saw_decl == 1 && blanklines_after_declarations) { prefix_blankline_requested = 1; ps.just_saw_decl = 0; } else prefix_blankline_requested = postfix_blankline_requested; postfix_blankline_requested = 0; } ps.decl_on_line = ps.in_decl; /* if we are in the middle of a * declaration, remember that fact for * proper comment indentation */ ps.ind_stmt = ps.in_stmt & ~ps.in_decl; /* next line should be * indented if we have not * completed this stmt and if * we are not in the middle of * a declaration */ ps.use_ff = false; ps.dumped_decl_indent = 0; *(e_lab = s_lab) = '\0'; /* reset buffers */ *(e_code = s_code) = '\0'; *(e_com = s_com = combuf + 1) = '\0'; ps.ind_level = ps.i_l_follow; ps.paren_level = ps.p_l_follow; if (ps.paren_level > 0) paren_target = -ps.paren_indents[ps.paren_level - 1]; not_first_line = 1; } int compute_code_target(void) { int target_col = ps.ind_size * ps.ind_level + 1; if (ps.paren_level) if (!lineup_to_parens) target_col += continuation_indent * (2 * continuation_indent == ps.ind_size ? 1 : ps.paren_level); else { int w; int t = paren_target; if ((w = count_spaces(t, s_code) - max_col) > 0 && count_spaces(target_col, s_code) <= max_col) { t -= w + 1; if (t > target_col) target_col = t; } else target_col = t; } else if (ps.ind_stmt) target_col += continuation_indent; return target_col; } int compute_label_target(void) { return ps.pcase ? (int) (case_ind * ps.ind_size) + 1 : *s_lab == '#' ? 1 : ps.ind_size * (ps.ind_level - label_offset) + 1; } /* * Copyright (C) 1976 by the Board of Trustees of the University of Illinois * * All rights reserved * * * NAME: fill_buffer * * FUNCTION: Reads one block of input into input_buffer * * HISTORY: initial coding November 1976 D A Willcox of CAC 1/7/77 A * Willcox of CAC Added check for switch back to partly full input * buffer from temporary buffer * */ void fill_buffer(void) { /* this routine reads stuff from the input */ char *p; int i; FILE *f = input; if (bp_save != NULL) { /* there is a partly filled input buffer left */ buf_ptr = bp_save; /* do not read anything, just switch buffers */ buf_end = be_save; bp_save = be_save = NULL; if (buf_ptr < buf_end) return; /* only return if there is really something in * this buffer */ } for (p = in_buffer;;) { if (p >= in_buffer_limit) { int size = (in_buffer_limit - in_buffer) * 2 + 10; int offset = p - in_buffer; in_buffer = realloc(in_buffer, size); if (in_buffer == NULL) errx(1, "input line too long"); p = in_buffer + offset; in_buffer_limit = in_buffer + size - 2; } if ((i = getc(f)) == EOF) { *p++ = ' '; *p++ = '\n'; had_eof = true; break; } *p++ = i; if (i == '\n') break; } buf_ptr = in_buffer; buf_end = p; if (p - in_buffer > 2 && p[-2] == '/' && p[-3] == '*') { if (in_buffer[3] == 'I' && strncmp(in_buffer, "/**INDENT**", 11) == 0) fill_buffer(); /* flush indent error message */ else { int com = 0; p = in_buffer; while (*p == ' ' || *p == '\t') p++; if (*p == '/' && p[1] == '*') { p += 2; while (*p == ' ' || *p == '\t') p++; if (p[0] == 'I' && p[1] == 'N' && p[2] == 'D' && p[3] == 'E' && p[4] == 'N' && p[5] == 'T') { p += 6; while (*p == ' ' || *p == '\t') p++; if (*p == '*') com = 1; else if (*p == 'O') { if (*++p == 'N') p++, com = 1; else if (*p == 'F' && *++p == 'F') p++, com = 2; } while (*p == ' ' || *p == '\t') p++; if (p[0] == '*' && p[1] == '/' && p[2] == '\n' && com) { if (s_com != e_com || s_lab != e_lab || s_code != e_code) dump_line(); if (!(inhibit_formatting = com - 1)) { n_real_blanklines = 0; postfix_blankline_requested = 0; prefix_blankline_requested = 0; suppress_blanklines = 1; } } } } } } if (inhibit_formatting) { p = in_buffer; do putc(*p, output); while (*p++ != '\n'); } } /* * Copyright (C) 1976 by the Board of Trustees of the University of Illinois * * All rights reserved * * * NAME: pad_output * * FUNCTION: Writes tabs and spaces to move the current column up to the desired * position. * * ALGORITHM: Put tabs and/or blanks into pobuf, then write pobuf. * * PARAMETERS: current integer The current column target * nteger The desired column * * RETURNS: Integer value of the new column. (If current >= target, no action is * taken, and current is returned. * * GLOBALS: None * * CALLS: write (sys) * * CALLED BY: dump_line * * HISTORY: initial coding November 1976 D A Willcox of CAC * */ static int pad_output(int current, int target) /* writes tabs and blanks (if necessary) to * get the current output position up to the * target column */ /* current: the current column value */ /* target: position we want it at */ { - int curr; /* internal column pointer */ - int tcur; if (troff) fprintf(output, "\\h'|%dp'", (target - 1) * 7); else { + int curr; /* internal column pointer */ + if (current >= target) return (current); /* line is already long enough */ curr = current; if (use_tabs) { - while ((tcur = ((curr - 1) & tabmask) + tabsize + 1) <= target) { + int tcur; + + while ((tcur = tabsize * (1 + (curr - 1) / tabsize) + 1) <= target) { putc('\t', output); curr = tcur; } } while (curr++ < target) putc(' ', output); /* pad with final blanks */ } return (target); } /* * Copyright (C) 1976 by the Board of Trustees of the University of Illinois * * All rights reserved * * * NAME: count_spaces * * FUNCTION: Find out where printing of a given string will leave the current * character position on output. * * ALGORITHM: Run thru input string and add appropriate values to current * position. * * RETURNS: Integer value of position after printing "buffer" starting in column * "current". * * HISTORY: initial coding November 1976 D A Willcox of CAC * */ int count_spaces_until(int cur, char *buffer, char *end) /* * this routine figures out where the character position will be after * printing the text in buffer starting at column "current" */ { char *buf; /* used to look thru buffer */ for (buf = buffer; *buf != '\0' && buf != end; ++buf) { switch (*buf) { case '\n': case 014: /* form feed */ cur = 1; break; case '\t': - cur = ((cur - 1) & tabmask) + tabsize + 1; + cur = tabsize * (1 + (cur - 1) / tabsize) + 1; break; case 010: /* backspace */ --cur; break; default: ++cur; break; } /* end of switch */ } /* end of for loop */ return (cur); } int count_spaces(int cur, char *buffer) { return (count_spaces_until(cur, buffer, NULL)); } void diag4(int level, const char *msg, int a, int b) { if (level) found_err = 1; if (output == stdout) { fprintf(stdout, "/**INDENT** %s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stdout, msg, a, b); fprintf(stdout, " */\n"); } else { fprintf(stderr, "%s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stderr, msg, a, b); fprintf(stderr, "\n"); } } void diag3(int level, const char *msg, int a) { if (level) found_err = 1; if (output == stdout) { fprintf(stdout, "/**INDENT** %s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stdout, msg, a); fprintf(stdout, " */\n"); } else { fprintf(stderr, "%s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stderr, msg, a); fprintf(stderr, "\n"); } } void diag2(int level, const char *msg) { if (level) found_err = 1; if (output == stdout) { fprintf(stdout, "/**INDENT** %s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stdout, "%s", msg); fprintf(stdout, " */\n"); } else { fprintf(stderr, "%s@%d: ", level == 0 ? "Warning" : "Error", line_no); fprintf(stderr, "%s", msg); fprintf(stderr, "\n"); } } void writefdef(struct fstate *f, int nm) { fprintf(output, ".ds f%c %s\n.nr s%c %d\n", nm, f->font, nm, f->size); } char * chfont(struct fstate *of, struct fstate *nf, char *s) { if (of->font[0] != nf->font[0] || of->font[1] != nf->font[1]) { *s++ = '\\'; *s++ = 'f'; if (nf->font[1]) { *s++ = '('; *s++ = nf->font[0]; *s++ = nf->font[1]; } else *s++ = nf->font[0]; } if (nf->size != of->size) { *s++ = '\\'; *s++ = 's'; if (nf->size < of->size) { *s++ = '-'; *s++ = '0' + of->size - nf->size; } else { *s++ = '+'; *s++ = '0' + nf->size - of->size; } } return s; } void parsefont(struct fstate *f, const char *s0) { const char *s = s0; int sizedelta = 0; memset(f, '\0', sizeof(*f)); while (*s) { if (isdigit(*s)) f->size = f->size * 10 + *s - '0'; else if (isupper(*s)) if (f->font[0]) f->font[1] = *s; else f->font[0] = *s; else if (*s == 'c') f->allcaps = 1; else if (*s == '+') sizedelta++; else if (*s == '-') sizedelta--; else { errx(1, "bad font specification: %s", s0); } s++; } if (f->font[0] == 0) f->font[0] = 'R'; if (bodyf.size == 0) bodyf.size = 11; if (f->size == 0) f->size = bodyf.size + sizedelta; else if (sizedelta > 0) f->size += bodyf.size; else f->size = bodyf.size - f->size; } Index: projects/runtime-coverage/usr.bin/indent/pr_comment.c =================================================================== --- projects/runtime-coverage/usr.bin/indent/pr_comment.c (revision 321584) +++ projects/runtime-coverage/usr.bin/indent/pr_comment.c (revision 321585) @@ -1,347 +1,347 @@ /*- * Copyright (c) 1985 Sun Microsystems, Inc. * Copyright (c) 1980, 1993 * The Regents of the University of California. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if 0 #ifndef lint static char sccsid[] = "@(#)pr_comment.c 8.1 (Berkeley) 6/6/93"; #endif /* not lint */ #endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include "indent_globs.h" #include "indent_codes.h" #include "indent.h" /* * NAME: * pr_comment * * FUNCTION: * This routine takes care of scanning and printing comments. * * ALGORITHM: * 1) Decide where the comment should be aligned, and if lines should * be broken. * 2) If lines should not be broken and filled, just copy up to end of * comment. * 3) If lines should be filled, then scan thru input_buffer copying * characters to com_buf. Remember where the last blank, tab, or * newline was. When line is filled, print up to last blank and * continue copying. * * HISTORY: * November 1976 D A Willcox of CAC Initial coding * 12/6/76 D A Willcox of CAC Modification to handle * UNIX-style comments * */ /* * this routine processes comments. It makes an attempt to keep comments from * going over the max line length. If a line is too long, it moves everything * from the last blank to the next comment line. Blanks and tabs from the * beginning of the input line are removed */ void pr_comment(void) { int now_col; /* column we are in now */ int adj_max_col; /* Adjusted max_col for when we decide to * spill comments over the right margin */ char *last_bl; /* points to the last blank in the output * buffer */ char *t_ptr; /* used for moving string */ int break_delim = comment_delimiter_on_blankline; int l_just_saw_decl = ps.just_saw_decl; adj_max_col = max_col; ps.just_saw_decl = 0; last_bl = NULL; /* no blanks found so far */ ps.box_com = false; /* at first, assume that we are not in * a boxed comment or some other * comment that should not be touched */ ++ps.out_coms; /* keep track of number of comments */ /* Figure where to align and how to treat the comment */ if (ps.col_1 && !format_col1_comments) { /* if comment starts in column * 1 it should not be touched */ ps.box_com = true; break_delim = false; ps.com_col = 1; } else { if (*buf_ptr == '-' || *buf_ptr == '*' || (*buf_ptr == '\n' && !format_block_comments)) { ps.box_com = true; /* A comment with a '-' or '*' immediately * after the /+* is assumed to be a boxed * comment. A comment with a newline * immediately after the /+* is assumed to * be a block comment and is treated as a * box comment unless format_block_comments * is nonzero (the default). */ break_delim = false; } if ( /* ps.bl_line && */ (s_lab == e_lab) && (s_code == e_code)) { /* klg: check only if this line is blank */ /* * If this (*and previous lines are*) blank, dont put comment way * out at left */ ps.com_col = (ps.ind_level - ps.unindent_displace) * ps.ind_size + 1; adj_max_col = block_comment_max_col; if (ps.com_col <= 1) ps.com_col = 1 + !format_col1_comments; } else { int target_col; break_delim = false; if (s_code != e_code) target_col = count_spaces(compute_code_target(), s_code); else { target_col = 1; if (s_lab != e_lab) target_col = count_spaces(compute_label_target(), s_lab); } ps.com_col = ps.decl_on_line || ps.ind_level == 0 ? ps.decl_com_ind : ps.com_ind; if (ps.com_col <= target_col) - ps.com_col = ((target_col + 7) & ~7) + 1; + ps.com_col = tabsize * (1 + (target_col - 1) / tabsize) + 1; if (ps.com_col + 24 > adj_max_col) adj_max_col = ps.com_col + 24; } } if (ps.box_com) { /* * Find out how much indentation there was originally, because that * much will have to be ignored by pad_output() in dump_line(). This * is a box comment, so nothing changes -- not even indentation. * * The comment we're about to read usually comes from in_buffer, * unless it has been copied into save_com. */ char *start = buf_ptr >= save_com && buf_ptr < save_com + sc_size ? bp_save : buf_ptr; ps.n_comment_delta = 1 - count_spaces_until(1, in_buffer, start - 2); } else { ps.n_comment_delta = 0; while (*buf_ptr == ' ' || *buf_ptr == '\t') buf_ptr++; } ps.comment_delta = 0; *e_com++ = '/'; /* put '/' followed by '*' into buffer */ *e_com++ = '*'; if (*buf_ptr != ' ' && !ps.box_com) *e_com++ = ' '; /* * Don't put a break delimiter if this is a one-liner that won't wrap. */ if (break_delim) for (t_ptr = buf_ptr; *t_ptr != '\0' && *t_ptr != '\n'; t_ptr++) { if (t_ptr >= buf_end) fill_buffer(); if (t_ptr[0] == '*' && t_ptr[1] == '/') { if (adj_max_col >= count_spaces_until(ps.com_col, buf_ptr, t_ptr + 2)) break_delim = false; break; } } if (break_delim) { char *t = e_com; e_com = s_com + 2; *e_com = 0; if (blanklines_before_blockcomments && ps.last_token != lbrace) prefix_blankline_requested = 1; dump_line(); e_com = s_com = t; if (!ps.box_com && star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; } if (troff) adj_max_col = 80; /* Start to copy the comment */ while (1) { /* this loop will go until the comment is * copied */ CHECK_SIZE_COM; switch (*buf_ptr) { /* this checks for various spcl cases */ case 014: /* check for a form feed */ if (!ps.box_com) { /* in a text comment, break the line here */ ps.use_ff = true; /* fix so dump_line uses a form feed */ dump_line(); last_bl = NULL; if (!ps.box_com && star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; while (*++buf_ptr == ' ' || *buf_ptr == '\t') ; } else { if (++buf_ptr >= buf_end) fill_buffer(); *e_com++ = 014; } break; case '\n': if (had_eof) { /* check for unexpected eof */ printf("Unterminated comment\n"); dump_line(); return; } last_bl = NULL; if (ps.box_com || ps.last_nl) { /* if this is a boxed comment, * we dont ignore the newline */ if (s_com == e_com) *e_com++ = ' '; if (!ps.box_com && e_com - s_com > 3) { dump_line(); if (star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; } dump_line(); if (!ps.box_com && star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; } else { ps.last_nl = 1; if (*(e_com - 1) == ' ' || *(e_com - 1) == '\t') last_bl = e_com - 1; /* * if there was a space at the end of the last line, remember * where it was */ else { /* otherwise, insert one */ last_bl = e_com; CHECK_SIZE_COM; *e_com++ = ' '; } } ++line_no; /* keep track of input line number */ if (!ps.box_com) { int nstar = 1; do { /* flush any blanks and/or tabs at start of * next line */ if (++buf_ptr >= buf_end) fill_buffer(); if (*buf_ptr == '*' && --nstar >= 0) { if (++buf_ptr >= buf_end) fill_buffer(); if (*buf_ptr == '/') goto end_of_comment; } } while (*buf_ptr == ' ' || *buf_ptr == '\t'); } else if (++buf_ptr >= buf_end) fill_buffer(); break; /* end of case for newline */ case '*': /* must check for possibility of being at end * of comment */ if (++buf_ptr >= buf_end) /* get to next char after * */ fill_buffer(); if (*buf_ptr == '/') { /* it is the end!!! */ end_of_comment: if (++buf_ptr >= buf_end) fill_buffer(); CHECK_SIZE_COM; if (break_delim) { if (e_com > s_com + 3) { dump_line(); } else s_com = e_com; *e_com++ = ' '; } if (e_com[-1] != ' ' && e_com[-1] != '\t' && !ps.box_com) *e_com++ = ' '; /* ensure blank before end */ *e_com++ = '*', *e_com++ = '/', *e_com = '\0'; ps.just_saw_decl = l_just_saw_decl; return; } else /* handle isolated '*' */ *e_com++ = '*'; break; default: /* we have a random char */ now_col = count_spaces_until(ps.com_col, s_com, e_com); do { *e_com = *buf_ptr++; if (buf_ptr >= buf_end) fill_buffer(); if (*e_com == ' ' || *e_com == '\t') last_bl = e_com; /* remember we saw a blank */ ++e_com; now_col++; } while (!memchr("*\n\r\b\t", *buf_ptr, 6) && (now_col <= adj_max_col || !last_bl)); ps.last_nl = false; if (now_col > adj_max_col && !ps.box_com && e_com[-1] > ' ') { /* * the comment is too long, it must be broken up */ if (last_bl == NULL) { dump_line(); if (!ps.box_com && star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; break; } *e_com = '\0'; e_com = last_bl; dump_line(); if (!ps.box_com && star_comment_cont) *e_com++ = ' ', *e_com++ = '*', *e_com++ = ' '; for (t_ptr = last_bl + 1; *t_ptr == ' ' || *t_ptr == '\t'; t_ptr++) ; last_bl = NULL; while (*t_ptr != '\0') { if (*t_ptr == ' ' || *t_ptr == '\t') last_bl = e_com; *e_com++ = *t_ptr++; } } break; } } } Index: projects/runtime-coverage/usr.bin/truss/syscalls.c =================================================================== --- projects/runtime-coverage/usr.bin/truss/syscalls.c (revision 321584) +++ projects/runtime-coverage/usr.bin/truss/syscalls.c (revision 321585) @@ -1,2427 +1,2414 @@ /* * Copyright 1997 Sean Eric Fagan * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Sean Eric Fagan * 4. Neither the name of the author may be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * This file has routines used to print out system calls and their * arguments. */ #include #include #include #include #include #include #include #include #define _WANT_FREEBSD11_STAT #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "truss.h" #include "extern.h" #include "syscall.h" /* * This should probably be in its own file, sorted alphabetically. */ static struct syscall decoded_syscalls[] = { /* Native ABI */ { .name = "__acl_aclcheck_fd", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_aclcheck_file", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_aclcheck_link", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_delete_fd", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Acltype, 1 } } }, { .name = "__acl_delete_file", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Acltype, 1 } } }, { .name = "__acl_delete_link", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Acltype, 1 } } }, { .name = "__acl_get_fd", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_get_file", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_get_link", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_set_fd", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_set_file", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__acl_set_link", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Acltype, 1 }, { Ptr, 2 } } }, { .name = "__cap_rights_get", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { CapRights | OUT, 2 } } }, { .name = "__getcwd", .ret_type = 1, .nargs = 2, .args = { { Name | OUT, 0 }, { Int, 1 } } }, { .name = "_umtx_op", .ret_type = 1, .nargs = 5, .args = { { Ptr, 0 }, { Umtxop, 1 }, { LongHex, 2 }, { Ptr, 3 }, { Ptr, 4 } } }, { .name = "accept", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Sockaddr | OUT, 1 }, { Ptr | OUT, 2 } } }, { .name = "access", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Accessmode, 1 } } }, { .name = "bind", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Sockaddr | IN, 1 }, { Socklent, 2 } } }, { .name = "bindat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Int, 1 }, { Sockaddr | IN, 2 }, { Int, 3 } } }, { .name = "break", .ret_type = 1, .nargs = 1, .args = { { Ptr, 0 } } }, { .name = "cap_fcntls_get", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CapFcntlRights | OUT, 1 } } }, { .name = "cap_fcntls_limit", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CapFcntlRights, 1 } } }, { .name = "cap_getmode", .ret_type = 1, .nargs = 1, .args = { { PUInt | OUT, 0 } } }, { .name = "cap_rights_limit", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CapRights, 1 } } }, { .name = "chdir", .ret_type = 1, .nargs = 1, .args = { { Name, 0 } } }, { .name = "chflags", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { FileFlags, 1 } } }, { .name = "chflagsat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { FileFlags, 2 }, { Atflags, 3 } } }, { .name = "chmod", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Octal, 1 } } }, { .name = "chown", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "chroot", .ret_type = 1, .nargs = 1, .args = { { Name, 0 } } }, { .name = "clock_gettime", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Timespec | OUT, 1 } } }, { .name = "close", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "compat11.fstat", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Stat11 | OUT, 1 } } }, { .name = "compat11.fstatat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Stat11 | OUT, 2 }, { Atflags, 3 } } }, { .name = "compat11.lstat", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Stat11 | OUT, 1 } } }, { .name = "compat11.stat", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Stat11 | OUT, 1 } } }, { .name = "connect", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Sockaddr | IN, 1 }, { Socklent, 2 } } }, { .name = "connectat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Int, 1 }, { Sockaddr | IN, 2 }, { Int, 3 } } }, { .name = "dup", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "dup2", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Int, 1 } } }, { .name = "eaccess", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Accessmode, 1 } } }, { .name = "execve", .ret_type = 1, .nargs = 3, .args = { { Name | IN, 0 }, { ExecArgs | IN, 1 }, { ExecEnv | IN, 2 } } }, { .name = "exit", .ret_type = 0, .nargs = 1, .args = { { Hex, 0 } } }, { .name = "extattr_delete_fd", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Extattrnamespace, 1 }, { Name, 2 } } }, { .name = "extattr_delete_file", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 } } }, { .name = "extattr_delete_link", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 } } }, { .name = "extattr_get_fd", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | OUT, 3 }, { Sizet, 4 } } }, { .name = "extattr_get_file", .ret_type = 1, .nargs = 5, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | OUT, 3 }, { Sizet, 4 } } }, { .name = "extattr_get_link", .ret_type = 1, .nargs = 5, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | OUT, 3 }, { Sizet, 4 } } }, { .name = "extattr_list_fd", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { Extattrnamespace, 1 }, { BinString | OUT, 2 }, { Sizet, 3 } } }, { .name = "extattr_list_file", .ret_type = 1, .nargs = 4, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { BinString | OUT, 2 }, { Sizet, 3 } } }, { .name = "extattr_list_link", .ret_type = 1, .nargs = 4, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { BinString | OUT, 2 }, { Sizet, 3 } } }, { .name = "extattr_set_fd", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | IN, 3 }, { Sizet, 4 } } }, { .name = "extattr_set_file", .ret_type = 1, .nargs = 5, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | IN, 3 }, { Sizet, 4 } } }, { .name = "extattr_set_link", .ret_type = 1, .nargs = 5, .args = { { Name, 0 }, { Extattrnamespace, 1 }, { Name, 2 }, { BinString | IN, 3 }, { Sizet, 4 } } }, { .name = "extattrctl", .ret_type = 1, .nargs = 5, .args = { { Name, 0 }, { Hex, 1 }, { Name, 2 }, { Extattrnamespace, 3 }, { Name, 4 } } }, { .name = "faccessat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Accessmode, 2 }, { Atflags, 3 } } }, { .name = "fchflags", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { FileFlags, 1 } } }, { .name = "fchmod", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Octal, 1 } } }, { .name = "fchmodat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name, 1 }, { Octal, 2 }, { Atflags, 3 } } }, { .name = "fchown", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "fchownat", .ret_type = 1, .nargs = 5, .args = { { Atfd, 0 }, { Name, 1 }, { Int, 2 }, { Int, 3 }, { Atflags, 4 } } }, { .name = "fcntl", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Fcntl, 1 }, { Fcntlflag, 2 } } }, { .name = "flock", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Flockop, 1 } } }, { .name = "fstat", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Stat | OUT, 1 } } }, { .name = "fstatat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Stat | OUT, 2 }, { Atflags, 3 } } }, { .name = "fstatfs", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { StatFs | OUT, 1 } } }, { .name = "ftruncate", .ret_type = 1, .nargs = 2, .args = { { Int | IN, 0 }, { QuadHex | IN, 1 } } }, { .name = "futimens", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Timespec2 | IN, 1 } } }, { .name = "futimes", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Timeval2 | IN, 1 } } }, { .name = "futimesat", .ret_type = 1, .nargs = 3, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Timeval2 | IN, 2 } } }, { .name = "getdirentries", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | OUT, 1 }, { Int, 2 }, { PQuadHex | OUT, 3 } } }, { .name = "getfsstat", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Long, 1 }, { Getfsstatmode, 2 } } }, { .name = "getitimer", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Itimerval | OUT, 2 } } }, { .name = "getpeername", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Sockaddr | OUT, 1 }, { Ptr | OUT, 2 } } }, { .name = "getpgid", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "getpriority", .ret_type = 1, .nargs = 2, .args = { { Priowhich, 0 }, { Int, 1 } } }, { .name = "getrlimit", .ret_type = 1, .nargs = 2, .args = { { Resource, 0 }, { Rlimit | OUT, 1 } } }, { .name = "getrusage", .ret_type = 1, .nargs = 2, .args = { { RusageWho, 0 }, { Rusage | OUT, 1 } } }, { .name = "getsid", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "getsockname", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Sockaddr | OUT, 1 }, { Ptr | OUT, 2 } } }, { .name = "getsockopt", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { Sockoptlevel, 1 }, { Sockoptname, 2 }, { Ptr | OUT, 3 }, { Ptr | OUT, 4 } } }, { .name = "gettimeofday", .ret_type = 1, .nargs = 2, .args = { { Timeval | OUT, 0 }, { Ptr, 1 } } }, { .name = "ioctl", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Ioctl, 1 }, { Ptr, 2 } } }, { .name = "kevent", .ret_type = 1, .nargs = 6, .args = { { Int, 0 }, { Kevent, 1 }, { Int, 2 }, { Kevent | OUT, 3 }, { Int, 4 }, { Timespec, 5 } } }, { .name = "kill", .ret_type = 1, .nargs = 2, .args = { { Int | IN, 0 }, { Signal | IN, 1 } } }, { .name = "kldfind", .ret_type = 1, .nargs = 1, .args = { { Name | IN, 0 } } }, { .name = "kldfirstmod", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "kldload", .ret_type = 1, .nargs = 1, .args = { { Name | IN, 0 } } }, { .name = "kldnext", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "kldstat", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Ptr, 1 } } }, { .name = "kldsym", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Kldsymcmd, 1 }, { Ptr, 2 } } }, { .name = "kldunload", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "kldunloadf", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Kldunloadflags, 1 } } }, { .name = "kse_release", .ret_type = 0, .nargs = 1, .args = { { Timespec, 0 } } }, { .name = "lchflags", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { FileFlags, 1 } } }, { .name = "lchmod", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Octal, 1 } } }, { .name = "lchown", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "link", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Name, 1 } } }, { .name = "linkat", .ret_type = 1, .nargs = 5, .args = { { Atfd, 0 }, { Name, 1 }, { Atfd, 2 }, { Name, 3 }, { Atflags, 4 } } }, { .name = "listen", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Int, 1 } } }, { .name = "lseek", .ret_type = 2, .nargs = 3, .args = { { Int, 0 }, { QuadHex, 1 }, { Whence, 2 } } }, { .name = "lstat", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Stat | OUT, 1 } } }, { .name = "lutimes", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Timeval2 | IN, 1 } } }, { .name = "madvise", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Sizet, 1 }, { Madvice, 2 } } }, { .name = "minherit", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Sizet, 1 }, { Minherit, 2 } } }, { .name = "mkdir", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Octal, 1 } } }, { .name = "mkdirat", .ret_type = 1, .nargs = 3, .args = { { Atfd, 0 }, { Name, 1 }, { Octal, 2 } } }, { .name = "mkfifo", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Octal, 1 } } }, { .name = "mkfifoat", .ret_type = 1, .nargs = 3, .args = { { Atfd, 0 }, { Name, 1 }, { Octal, 2 } } }, { .name = "mknod", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Octal, 1 }, { Int, 2 } } }, { .name = "mknodat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name, 1 }, { Octal, 2 }, { Int, 3 } } }, { .name = "mlock", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { Sizet, 1 } } }, { .name = "mlockall", .ret_type = 1, .nargs = 1, .args = { { Mlockall, 0 } } }, { .name = "mmap", .ret_type = 1, .nargs = 6, .args = { { Ptr, 0 }, { Sizet, 1 }, { Mprot, 2 }, { Mmapflags, 3 }, { Int, 4 }, { QuadHex, 5 } } }, { .name = "modfind", .ret_type = 1, .nargs = 1, .args = { { Name | IN, 0 } } }, { .name = "mount", .ret_type = 1, .nargs = 4, .args = { { Name, 0 }, { Name, 1 }, { Mountflags, 2 }, { Ptr, 3 } } }, { .name = "mprotect", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Sizet, 1 }, { Mprot, 2 } } }, { .name = "msync", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Sizet, 1 }, { Msync, 2 } } }, { .name = "munlock", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { Sizet, 1 } } }, { .name = "munmap", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { Sizet, 1 } } }, { .name = "nanosleep", .ret_type = 1, .nargs = 1, .args = { { Timespec, 0 } } }, { .name = "nmount", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { UInt, 1 }, { Mountflags, 2 } } }, { .name = "open", .ret_type = 1, .nargs = 3, .args = { { Name | IN, 0 }, { Open, 1 }, { Octal, 2 } } }, { .name = "openat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Open, 2 }, { Octal, 3 } } }, { .name = "pathconf", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Pathconf, 1 } } }, { .name = "pipe", .ret_type = 1, .nargs = 1, .args = { { PipeFds | OUT, 0 } } }, { .name = "pipe2", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { Pipe2, 1 } } }, { .name = "poll", .ret_type = 1, .nargs = 3, .args = { { Pollfd, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "posix_fadvise", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { QuadHex, 1 }, { QuadHex, 2 }, { Fadvice, 3 } } }, { .name = "posix_openpt", .ret_type = 1, .nargs = 1, .args = { { Open, 0 } } }, { .name = "pread", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | OUT, 1 }, { Sizet, 2 }, { QuadHex, 3 } } }, { .name = "procctl", .ret_type = 1, .nargs = 4, .args = { { Idtype, 0 }, { Quad, 1 }, { Procctl, 2 }, { Ptr, 3 } } }, { .name = "ptrace", .ret_type = 1, .nargs = 4, .args = { { Ptraceop, 0 }, { Int, 1 }, { Ptr, 2 }, { Int, 3 } } }, { .name = "pwrite", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | IN, 1 }, { Sizet, 2 }, { QuadHex, 3 } } }, { .name = "quotactl", .ret_type = 1, .nargs = 4, .args = { { Name, 0 }, { Quotactlcmd, 1 }, { Int, 2 }, { Ptr, 3 } } }, { .name = "read", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { BinString | OUT, 1 }, { Sizet, 2 } } }, { .name = "readlink", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Readlinkres | OUT, 1 }, { Sizet, 2 } } }, { .name = "readlinkat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name, 1 }, { Readlinkres | OUT, 2 }, { Sizet, 3 } } }, { .name = "reboot", .ret_type = 1, .nargs = 1, .args = { { Reboothowto, 0 } } }, { .name = "recvfrom", .ret_type = 1, .nargs = 6, .args = { { Int, 0 }, { BinString | OUT, 1 }, { Sizet, 2 }, { Msgflags, 3 }, { Sockaddr | OUT, 4 }, { Ptr | OUT, 5 } } }, { .name = "recvmsg", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Ptr, 1 }, { Msgflags, 2 } } }, { .name = "rename", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Name, 1 } } }, { .name = "renameat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name, 1 }, { Atfd, 2 }, { Name, 3 } } }, { .name = "rfork", .ret_type = 1, .nargs = 1, .args = { { Rforkflags, 0 } } }, { .name = "rmdir", .ret_type = 1, .nargs = 1, .args = { { Name, 0 } } }, { .name = "rtprio", .ret_type = 1, .nargs = 3, .args = { { Rtpriofunc, 0 }, { Int, 1 }, { Ptr, 2 } } }, { .name = "rtprio_thread", .ret_type = 1, .nargs = 3, .args = { { Rtpriofunc, 0 }, { Int, 1 }, { Ptr, 2 } } }, { .name = "sched_get_priority_max", .ret_type = 1, .nargs = 1, .args = { { Schedpolicy, 0 } } }, { .name = "sched_get_priority_min", .ret_type = 1, .nargs = 1, .args = { { Schedpolicy, 0 } } }, { .name = "sched_getparam", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Schedparam | OUT, 1 } } }, { .name = "sched_getscheduler", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "sched_rr_get_interval", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Timespec | OUT, 1 } } }, { .name = "sched_setparam", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Schedparam, 1 } } }, { .name = "sched_setscheduler", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Schedpolicy, 1 }, { Schedparam, 2 } } }, { .name = "sctp_generic_recvmsg", .ret_type = 1, .nargs = 7, .args = { { Int, 0 }, { Ptr | IN, 1 }, { Int, 2 }, { Sockaddr | OUT, 3 }, { Ptr | OUT, 4 }, { Ptr | OUT, 5 }, { Ptr | OUT, 6 } } }, { .name = "sctp_generic_sendmsg", .ret_type = 1, .nargs = 7, .args = { { Int, 0 }, { BinString | IN, 1 }, { Int, 2 }, { Sockaddr | IN, 3 }, { Socklent, 4 }, { Ptr | IN, 5 }, { Msgflags, 6 } } }, { .name = "select", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { Fd_set, 1 }, { Fd_set, 2 }, { Fd_set, 3 }, { Timeval, 4 } } }, { .name = "sendmsg", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Ptr, 1 }, { Msgflags, 2 } } }, { .name = "sendto", .ret_type = 1, .nargs = 6, .args = { { Int, 0 }, { BinString | IN, 1 }, { Sizet, 2 }, { Msgflags, 3 }, { Sockaddr | IN, 4 }, { Socklent | IN, 5 } } }, { .name = "setitimer", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Itimerval, 1 }, { Itimerval | OUT, 2 } } }, { .name = "setpriority", .ret_type = 1, .nargs = 3, .args = { { Priowhich, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "setrlimit", .ret_type = 1, .nargs = 2, .args = { { Resource, 0 }, { Rlimit | IN, 1 } } }, { .name = "setsockopt", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { Sockoptlevel, 1 }, { Sockoptname, 2 }, { Ptr | IN, 3 }, { Socklent, 4 } } }, { .name = "shutdown", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Shutdown, 1 } } }, { .name = "sigaction", .ret_type = 1, .nargs = 3, .args = { { Signal, 0 }, { Sigaction | IN, 1 }, { Sigaction | OUT, 2 } } }, { .name = "sigpending", .ret_type = 1, .nargs = 1, .args = { { Sigset | OUT, 0 } } }, { .name = "sigprocmask", .ret_type = 1, .nargs = 3, .args = { { Sigprocmask, 0 }, { Sigset, 1 }, { Sigset | OUT, 2 } } }, { .name = "sigqueue", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Signal, 1 }, { LongHex, 2 } } }, { .name = "sigreturn", .ret_type = 1, .nargs = 1, .args = { { Ptr, 0 } } }, { .name = "sigsuspend", .ret_type = 1, .nargs = 1, .args = { { Sigset | IN, 0 } } }, { .name = "sigtimedwait", .ret_type = 1, .nargs = 3, .args = { { Sigset | IN, 0 }, { Ptr, 1 }, { Timespec | IN, 2 } } }, { .name = "sigwait", .ret_type = 1, .nargs = 2, .args = { { Sigset | IN, 0 }, { Ptr, 1 } } }, { .name = "sigwaitinfo", .ret_type = 1, .nargs = 2, .args = { { Sigset | IN, 0 }, { Ptr, 1 } } }, { .name = "socket", .ret_type = 1, .nargs = 3, .args = { { Sockdomain, 0 }, { Socktype, 1 }, { Sockprotocol, 2 } } }, { .name = "stat", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Stat | OUT, 1 } } }, { .name = "statfs", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { StatFs | OUT, 1 } } }, { .name = "symlink", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Name, 1 } } }, { .name = "symlinkat", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Atfd, 1 }, { Name, 2 } } }, { .name = "sysarch", .ret_type = 1, .nargs = 2, .args = { { Sysarch, 0 }, { Ptr, 1 } } }, { .name = "thr_kill", .ret_type = 1, .nargs = 2, .args = { { Long, 0 }, { Signal, 1 } } }, { .name = "thr_self", .ret_type = 1, .nargs = 1, .args = { { Ptr, 0 } } }, { .name = "truncate", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { QuadHex | IN, 1 } } }, #if 0 /* Does not exist */ { .name = "umount", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Int, 2 } } }, #endif { .name = "unlink", .ret_type = 1, .nargs = 1, .args = { { Name, 0 } } }, { .name = "unlinkat", .ret_type = 1, .nargs = 3, .args = { { Atfd, 0 }, { Name, 1 }, { Atflags, 2 } } }, { .name = "unmount", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Mountflags, 1 } } }, { .name = "utimensat", .ret_type = 1, .nargs = 4, .args = { { Atfd, 0 }, { Name | IN, 1 }, { Timespec2 | IN, 2 }, { Atflags, 3 } } }, { .name = "utimes", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Timeval2 | IN, 1 } } }, { .name = "utrace", .ret_type = 1, .nargs = 1, .args = { { Utrace, 0 } } }, { .name = "wait4", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { ExitStatus | OUT, 1 }, { Waitoptions, 2 }, { Rusage | OUT, 3 } } }, { .name = "wait6", .ret_type = 1, .nargs = 6, .args = { { Idtype, 0 }, { Quad, 1 }, { ExitStatus | OUT, 2 }, { Waitoptions, 3 }, { Rusage | OUT, 4 }, { Ptr, 5 } } }, { .name = "write", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { BinString | IN, 1 }, { Sizet, 2 } } }, /* Linux ABI */ { .name = "linux_access", .ret_type = 1, .nargs = 2, .args = { { Name, 0 }, { Accessmode, 1 } } }, { .name = "linux_execve", .ret_type = 1, .nargs = 3, .args = { { Name | IN, 0 }, { ExecArgs | IN, 1 }, { ExecEnv | IN, 2 } } }, { .name = "linux_lseek", .ret_type = 2, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { Whence, 2 } } }, { .name = "linux_mkdir", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Int, 1 } } }, { .name = "linux_newfstat", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Ptr | OUT, 1 } } }, { .name = "linux_newstat", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Ptr | OUT, 1 } } }, { .name = "linux_open", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Hex, 1 }, { Octal, 2 } } }, { .name = "linux_readlink", .ret_type = 1, .nargs = 3, .args = { { Name, 0 }, { Name | OUT, 1 }, { Sizet, 2 } } }, { .name = "linux_socketcall", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { LinuxSockArgs, 1 } } }, { .name = "linux_stat64", .ret_type = 1, .nargs = 2, .args = { { Name | IN, 0 }, { Ptr | OUT, 1 } } }, /* CloudABI system calls. */ { .name = "cloudabi_sys_clock_res_get", .ret_type = 1, .nargs = 1, .args = { { CloudABIClockID, 0 } } }, { .name = "cloudabi_sys_clock_time_get", .ret_type = 1, .nargs = 2, .args = { { CloudABIClockID, 0 }, { CloudABITimestamp, 1 } } }, { .name = "cloudabi_sys_condvar_signal", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { CloudABIMFlags, 1 }, { UInt, 2 } } }, { .name = "cloudabi_sys_fd_close", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "cloudabi_sys_fd_create1", .ret_type = 1, .nargs = 1, .args = { { CloudABIFileType, 0 } } }, { .name = "cloudabi_sys_fd_create2", .ret_type = 1, .nargs = 2, .args = { { CloudABIFileType, 0 }, { PipeFds | OUT, 0 } } }, { .name = "cloudabi_sys_fd_datasync", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "cloudabi_sys_fd_dup", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "cloudabi_sys_fd_replace", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_fd_seek", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { CloudABIWhence, 2 } } }, { .name = "cloudabi_sys_fd_stat_get", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CloudABIFDStat | OUT, 1 } } }, { .name = "cloudabi_sys_fd_stat_put", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { CloudABIFDStat | IN, 1 }, { ClouduABIFDSFlags, 2 } } }, { .name = "cloudabi_sys_fd_sync", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "cloudabi_sys_file_advise", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { Int, 1 }, { Int, 2 }, { CloudABIAdvice, 3 } } }, { .name = "cloudabi_sys_file_allocate", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { Int, 2 } } }, { .name = "cloudabi_sys_file_create", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { BinString | IN, 1 }, { CloudABIFileType, 3 } } }, { .name = "cloudabi_sys_file_link", .ret_type = 1, .nargs = 4, .args = { { CloudABILookup, 0 }, { BinString | IN, 1 }, { Int, 3 }, { BinString | IN, 4 } } }, { .name = "cloudabi_sys_file_open", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | IN, 1 }, { CloudABIOFlags, 3 }, { CloudABIFDStat | IN, 4 } } }, { .name = "cloudabi_sys_file_readdir", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | OUT, 1 }, { Int, 2 }, { Int, 3 } } }, { .name = "cloudabi_sys_file_readlink", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | IN, 1 }, { BinString | OUT, 3 }, { Int, 4 } } }, { .name = "cloudabi_sys_file_rename", .ret_type = 1, .nargs = 4, .args = { { Int, 0 }, { BinString | IN, 1 }, { Int, 3 }, { BinString | IN, 4 } } }, { .name = "cloudabi_sys_file_stat_fget", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CloudABIFileStat | OUT, 1 } } }, { .name = "cloudabi_sys_file_stat_fput", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { CloudABIFileStat | IN, 1 }, { CloudABIFSFlags, 2 } } }, { .name = "cloudabi_sys_file_stat_get", .ret_type = 1, .nargs = 3, .args = { { CloudABILookup, 0 }, { BinString | IN, 1 }, { CloudABIFileStat | OUT, 3 } } }, { .name = "cloudabi_sys_file_stat_put", .ret_type = 1, .nargs = 4, .args = { { CloudABILookup, 0 }, { BinString | IN, 1 }, { CloudABIFileStat | IN, 3 }, { CloudABIFSFlags, 4 } } }, { .name = "cloudabi_sys_file_symlink", .ret_type = 1, .nargs = 3, .args = { { BinString | IN, 0 }, { Int, 2 }, { BinString | IN, 3 } } }, { .name = "cloudabi_sys_file_unlink", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { BinString | IN, 1 }, { CloudABIULFlags, 3 } } }, { .name = "cloudabi_sys_lock_unlock", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { CloudABIMFlags, 1 } } }, { .name = "cloudabi_sys_mem_advise", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Int, 1 }, { CloudABIAdvice, 2 } } }, - { .name = "cloudabi_sys_mem_lock", .ret_type = 1, .nargs = 2, - .args = { { Ptr, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_mem_map", .ret_type = 1, .nargs = 6, .args = { { Ptr, 0 }, { Int, 1 }, { CloudABIMProt, 2 }, { CloudABIMFlags, 3 }, { Int, 4 }, { Int, 5 } } }, { .name = "cloudabi_sys_mem_protect", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Int, 1 }, { CloudABIMProt, 2 } } }, { .name = "cloudabi_sys_mem_sync", .ret_type = 1, .nargs = 3, .args = { { Ptr, 0 }, { Int, 1 }, { CloudABIMSFlags, 2 } } }, - { .name = "cloudabi_sys_mem_unlock", .ret_type = 1, .nargs = 2, - .args = { { Ptr, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_mem_unmap", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_proc_exec", .ret_type = 1, .nargs = 5, .args = { { Int, 0 }, { BinString | IN, 1 }, { Int, 2 }, { IntArray, 3 }, { Int, 4 } } }, { .name = "cloudabi_sys_proc_exit", .ret_type = 1, .nargs = 1, .args = { { Int, 0 } } }, { .name = "cloudabi_sys_proc_fork", .ret_type = 1, .nargs = 0 }, { .name = "cloudabi_sys_proc_raise", .ret_type = 1, .nargs = 1, .args = { { CloudABISignal, 0 } } }, { .name = "cloudabi_sys_random_get", .ret_type = 1, .nargs = 2, .args = { { BinString | OUT, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_sock_accept", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CloudABISockStat | OUT, 1 } } }, { .name = "cloudabi_sys_sock_bind", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { BinString | IN, 2 } } }, { .name = "cloudabi_sys_sock_connect", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { Int, 1 }, { BinString | IN, 2 } } }, { .name = "cloudabi_sys_sock_listen", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { Int, 1 } } }, { .name = "cloudabi_sys_sock_shutdown", .ret_type = 1, .nargs = 2, .args = { { Int, 0 }, { CloudABISDFlags, 1 } } }, { .name = "cloudabi_sys_sock_stat_get", .ret_type = 1, .nargs = 3, .args = { { Int, 0 }, { CloudABISockStat | OUT, 1 }, { CloudABISSFlags, 2 } } }, { .name = "cloudabi_sys_thread_exit", .ret_type = 1, .nargs = 2, .args = { { Ptr, 0 }, { CloudABIMFlags, 1 } } }, { .name = "cloudabi_sys_thread_yield", .ret_type = 1, .nargs = 0 }, { .name = 0 }, }; static STAILQ_HEAD(, syscall) syscalls; /* Xlat idea taken from strace */ struct xlat { int val; const char *str; }; #define X(a) { a, #a }, #define XEND { 0, NULL } static struct xlat kevent_filters[] = { X(EVFILT_READ) X(EVFILT_WRITE) X(EVFILT_AIO) X(EVFILT_VNODE) X(EVFILT_PROC) X(EVFILT_SIGNAL) X(EVFILT_TIMER) X(EVFILT_PROCDESC) X(EVFILT_FS) X(EVFILT_LIO) X(EVFILT_USER) X(EVFILT_SENDFILE) XEND }; static struct xlat kevent_flags[] = { X(EV_ADD) X(EV_DELETE) X(EV_ENABLE) X(EV_DISABLE) X(EV_ONESHOT) X(EV_CLEAR) X(EV_RECEIPT) X(EV_DISPATCH) X(EV_FORCEONESHOT) X(EV_DROP) X(EV_FLAG1) X(EV_ERROR) X(EV_EOF) XEND }; static struct xlat kevent_user_ffctrl[] = { X(NOTE_FFNOP) X(NOTE_FFAND) X(NOTE_FFOR) X(NOTE_FFCOPY) XEND }; static struct xlat kevent_rdwr_fflags[] = { X(NOTE_LOWAT) X(NOTE_FILE_POLL) XEND }; static struct xlat kevent_vnode_fflags[] = { X(NOTE_DELETE) X(NOTE_WRITE) X(NOTE_EXTEND) X(NOTE_ATTRIB) X(NOTE_LINK) X(NOTE_RENAME) X(NOTE_REVOKE) XEND }; static struct xlat kevent_proc_fflags[] = { X(NOTE_EXIT) X(NOTE_FORK) X(NOTE_EXEC) X(NOTE_TRACK) X(NOTE_TRACKERR) X(NOTE_CHILD) XEND }; static struct xlat kevent_timer_fflags[] = { X(NOTE_SECONDS) X(NOTE_MSECONDS) X(NOTE_USECONDS) X(NOTE_NSECONDS) XEND }; static struct xlat poll_flags[] = { X(POLLSTANDARD) X(POLLIN) X(POLLPRI) X(POLLOUT) X(POLLERR) X(POLLHUP) X(POLLNVAL) X(POLLRDNORM) X(POLLRDBAND) X(POLLWRBAND) X(POLLINIGNEOF) XEND }; static struct xlat sigaction_flags[] = { X(SA_ONSTACK) X(SA_RESTART) X(SA_RESETHAND) X(SA_NOCLDSTOP) X(SA_NODEFER) X(SA_NOCLDWAIT) X(SA_SIGINFO) XEND }; static struct xlat pathconf_arg[] = { X(_PC_LINK_MAX) X(_PC_MAX_CANON) X(_PC_MAX_INPUT) X(_PC_NAME_MAX) X(_PC_PATH_MAX) X(_PC_PIPE_BUF) X(_PC_CHOWN_RESTRICTED) X(_PC_NO_TRUNC) X(_PC_VDISABLE) X(_PC_ASYNC_IO) X(_PC_PRIO_IO) X(_PC_SYNC_IO) X(_PC_ALLOC_SIZE_MIN) X(_PC_FILESIZEBITS) X(_PC_REC_INCR_XFER_SIZE) X(_PC_REC_MAX_XFER_SIZE) X(_PC_REC_MIN_XFER_SIZE) X(_PC_REC_XFER_ALIGN) X(_PC_SYMLINK_MAX) X(_PC_ACL_EXTENDED) X(_PC_ACL_PATH_MAX) X(_PC_CAP_PRESENT) X(_PC_INF_PRESENT) X(_PC_MAC_PRESENT) X(_PC_ACL_NFS4) X(_PC_MIN_HOLE_SIZE) XEND }; static struct xlat at_flags[] = { X(AT_EACCESS) X(AT_SYMLINK_NOFOLLOW) X(AT_SYMLINK_FOLLOW) X(AT_REMOVEDIR) XEND }; static struct xlat sysarch_ops[] = { #if defined(__i386__) || defined(__amd64__) X(I386_GET_LDT) X(I386_SET_LDT) X(I386_GET_IOPERM) X(I386_SET_IOPERM) X(I386_VM86) X(I386_GET_FSBASE) X(I386_SET_FSBASE) X(I386_GET_GSBASE) X(I386_SET_GSBASE) X(I386_GET_XFPUSTATE) X(AMD64_GET_FSBASE) X(AMD64_SET_FSBASE) X(AMD64_GET_GSBASE) X(AMD64_SET_GSBASE) X(AMD64_GET_XFPUSTATE) #endif XEND }; static struct xlat linux_socketcall_ops[] = { X(LINUX_SOCKET) X(LINUX_BIND) X(LINUX_CONNECT) X(LINUX_LISTEN) X(LINUX_ACCEPT) X(LINUX_GETSOCKNAME) X(LINUX_GETPEERNAME) X(LINUX_SOCKETPAIR) X(LINUX_SEND) X(LINUX_RECV) X(LINUX_SENDTO) X(LINUX_RECVFROM) X(LINUX_SHUTDOWN) X(LINUX_SETSOCKOPT) X(LINUX_GETSOCKOPT) X(LINUX_SENDMSG) X(LINUX_RECVMSG) XEND }; #undef X #define X(a) { CLOUDABI_##a, #a }, static struct xlat cloudabi_advice[] = { X(ADVICE_DONTNEED) X(ADVICE_NOREUSE) X(ADVICE_NORMAL) X(ADVICE_RANDOM) X(ADVICE_SEQUENTIAL) X(ADVICE_WILLNEED) XEND }; static struct xlat cloudabi_clockid[] = { X(CLOCK_MONOTONIC) X(CLOCK_PROCESS_CPUTIME_ID) X(CLOCK_REALTIME) X(CLOCK_THREAD_CPUTIME_ID) XEND }; static struct xlat cloudabi_errno[] = { X(E2BIG) X(EACCES) X(EADDRINUSE) X(EADDRNOTAVAIL) X(EAFNOSUPPORT) X(EAGAIN) X(EALREADY) X(EBADF) X(EBADMSG) X(EBUSY) X(ECANCELED) X(ECHILD) X(ECONNABORTED) X(ECONNREFUSED) X(ECONNRESET) X(EDEADLK) X(EDESTADDRREQ) X(EDOM) X(EDQUOT) X(EEXIST) X(EFAULT) X(EFBIG) X(EHOSTUNREACH) X(EIDRM) X(EILSEQ) X(EINPROGRESS) X(EINTR) X(EINVAL) X(EIO) X(EISCONN) X(EISDIR) X(ELOOP) X(EMFILE) X(EMLINK) X(EMSGSIZE) X(EMULTIHOP) X(ENAMETOOLONG) X(ENETDOWN) X(ENETRESET) X(ENETUNREACH) X(ENFILE) X(ENOBUFS) X(ENODEV) X(ENOENT) X(ENOEXEC) X(ENOLCK) X(ENOLINK) X(ENOMEM) X(ENOMSG) X(ENOPROTOOPT) X(ENOSPC) X(ENOSYS) X(ENOTCONN) X(ENOTDIR) X(ENOTEMPTY) X(ENOTRECOVERABLE) X(ENOTSOCK) X(ENOTSUP) X(ENOTTY) X(ENXIO) X(EOVERFLOW) X(EOWNERDEAD) X(EPERM) X(EPIPE) X(EPROTO) X(EPROTONOSUPPORT) X(EPROTOTYPE) X(ERANGE) X(EROFS) X(ESPIPE) X(ESRCH) X(ESTALE) X(ETIMEDOUT) X(ETXTBSY) X(EXDEV) X(ENOTCAPABLE) XEND }; static struct xlat cloudabi_fdflags[] = { X(FDFLAG_APPEND) X(FDFLAG_DSYNC) X(FDFLAG_NONBLOCK) X(FDFLAG_RSYNC) X(FDFLAG_SYNC) XEND }; static struct xlat cloudabi_fdsflags[] = { X(FDSTAT_FLAGS) X(FDSTAT_RIGHTS) XEND }; static struct xlat cloudabi_filetype[] = { X(FILETYPE_UNKNOWN) X(FILETYPE_BLOCK_DEVICE) X(FILETYPE_CHARACTER_DEVICE) X(FILETYPE_DIRECTORY) X(FILETYPE_FIFO) X(FILETYPE_POLL) X(FILETYPE_PROCESS) X(FILETYPE_REGULAR_FILE) X(FILETYPE_SHARED_MEMORY) - X(FILETYPE_SOCKET_DGRAM) X(FILETYPE_SOCKET_SEQPACKET) - X(FILETYPE_SOCKET_STREAM) X(FILETYPE_SYMBOLIC_LINK) + X(FILETYPE_SOCKET_DGRAM) X(FILETYPE_SOCKET_STREAM) + X(FILETYPE_SYMBOLIC_LINK) XEND }; static struct xlat cloudabi_fsflags[] = { X(FILESTAT_ATIM) X(FILESTAT_ATIM_NOW) X(FILESTAT_MTIM) X(FILESTAT_MTIM_NOW) X(FILESTAT_SIZE) XEND }; static struct xlat cloudabi_mflags[] = { X(MAP_ANON) X(MAP_FIXED) X(MAP_PRIVATE) X(MAP_SHARED) XEND }; static struct xlat cloudabi_mprot[] = { X(PROT_EXEC) X(PROT_WRITE) X(PROT_READ) XEND }; static struct xlat cloudabi_msflags[] = { X(MS_ASYNC) X(MS_INVALIDATE) X(MS_SYNC) XEND }; static struct xlat cloudabi_oflags[] = { X(O_CREAT) X(O_DIRECTORY) X(O_EXCL) X(O_TRUNC) XEND }; -static struct xlat cloudabi_sa_family[] = { - X(AF_UNSPEC) X(AF_INET) X(AF_INET6) X(AF_UNIX) - XEND -}; - static struct xlat cloudabi_sdflags[] = { X(SHUT_RD) X(SHUT_WR) XEND }; static struct xlat cloudabi_signal[] = { X(SIGABRT) X(SIGALRM) X(SIGBUS) X(SIGCHLD) X(SIGCONT) X(SIGFPE) X(SIGHUP) X(SIGILL) X(SIGINT) X(SIGKILL) X(SIGPIPE) X(SIGQUIT) X(SIGSEGV) X(SIGSTOP) X(SIGSYS) X(SIGTERM) X(SIGTRAP) X(SIGTSTP) X(SIGTTIN) X(SIGTTOU) X(SIGURG) X(SIGUSR1) X(SIGUSR2) X(SIGVTALRM) X(SIGXCPU) X(SIGXFSZ) XEND }; static struct xlat cloudabi_ssflags[] = { X(SOCKSTAT_CLEAR_ERROR) XEND }; static struct xlat cloudabi_ssstate[] = { X(SOCKSTATE_ACCEPTCONN) XEND }; static struct xlat cloudabi_ulflags[] = { X(UNLINK_REMOVEDIR) XEND }; static struct xlat cloudabi_whence[] = { X(WHENCE_CUR) X(WHENCE_END) X(WHENCE_SET) XEND }; #undef X #undef XEND /* * Searches an xlat array for a value, and returns it if found. Otherwise * return a string representation. */ static const char * lookup(struct xlat *xlat, int val, int base) { static char tmp[16]; for (; xlat->str != NULL; xlat++) if (xlat->val == val) return (xlat->str); switch (base) { case 8: sprintf(tmp, "0%o", val); break; case 16: sprintf(tmp, "0x%x", val); break; case 10: sprintf(tmp, "%u", val); break; default: errx(1,"Unknown lookup base"); break; } return (tmp); } static const char * xlookup(struct xlat *xlat, int val) { return (lookup(xlat, val, 16)); } /* * Searches an xlat array containing bitfield values. Remaining bits * set after removing the known ones are printed at the end: * IN|0x400. */ static char * xlookup_bits(struct xlat *xlat, int val) { int len, rem; static char str[512]; len = 0; rem = val; for (; xlat->str != NULL; xlat++) { if ((xlat->val & rem) == xlat->val) { /* * Don't print the "all-bits-zero" string unless all * bits are really zero. */ if (xlat->val == 0 && val != 0) continue; len += sprintf(str + len, "%s|", xlat->str); rem &= ~(xlat->val); } } /* * If we have leftover bits or didn't match anything, print * the remainder. */ if (rem || len == 0) len += sprintf(str + len, "0x%x", rem); if (len && str[len - 1] == '|') len--; str[len] = 0; return (str); } static void print_integer_arg(const char *(*decoder)(int), FILE *fp, int value) { const char *str; str = decoder(value); if (str != NULL) fputs(str, fp); else fprintf(fp, "%d", value); } static void print_mask_arg(bool (*decoder)(FILE *, int, int *), FILE *fp, int value) { int rem; if (!decoder(fp, value, &rem)) fprintf(fp, "0x%x", rem); else if (rem != 0) fprintf(fp, "|0x%x", rem); } static void print_mask_arg32(bool (*decoder)(FILE *, uint32_t, uint32_t *), FILE *fp, uint32_t value) { uint32_t rem; if (!decoder(fp, value, &rem)) fprintf(fp, "0x%x", rem); else if (rem != 0) fprintf(fp, "|0x%x", rem); } #ifndef __LP64__ /* * Add argument padding to subsequent system calls afater a Quad * syscall arguments as needed. This used to be done by hand in the * decoded_syscalls table which was ugly and error prone. It is * simpler to do the fixup of offsets at initalization time than when * decoding arguments. */ static void quad_fixup(struct syscall *sc) { int offset, prev; u_int i; offset = 0; prev = -1; for (i = 0; i < sc->nargs; i++) { /* This arg type is a dummy that doesn't use offset. */ if ((sc->args[i].type & ARG_MASK) == PipeFds) continue; assert(prev < sc->args[i].offset); prev = sc->args[i].offset; sc->args[i].offset += offset; switch (sc->args[i].type & ARG_MASK) { case Quad: case QuadHex: #ifdef __powerpc__ /* * 64-bit arguments on 32-bit powerpc must be * 64-bit aligned. If the current offset is * not aligned, the calling convention inserts * a 32-bit pad argument that should be skipped. */ if (sc->args[i].offset % 2 == 1) { sc->args[i].offset++; offset++; } #endif offset++; default: break; } } } #endif void init_syscalls(void) { struct syscall *sc; STAILQ_INIT(&syscalls); for (sc = decoded_syscalls; sc->name != NULL; sc++) { #ifndef __LP64__ quad_fixup(sc); #endif STAILQ_INSERT_HEAD(&syscalls, sc, entries); } } static struct syscall * find_syscall(struct procabi *abi, u_int number) { struct extra_syscall *es; if (number < nitems(abi->syscalls)) return (abi->syscalls[number]); STAILQ_FOREACH(es, &abi->extra_syscalls, entries) { if (es->number == number) return (es->sc); } return (NULL); } static void add_syscall(struct procabi *abi, u_int number, struct syscall *sc) { struct extra_syscall *es; if (number < nitems(abi->syscalls)) { assert(abi->syscalls[number] == NULL); abi->syscalls[number] = sc; } else { es = malloc(sizeof(*es)); es->sc = sc; es->number = number; STAILQ_INSERT_TAIL(&abi->extra_syscalls, es, entries); } } /* * If/when the list gets big, it might be desirable to do it * as a hash table or binary search. */ struct syscall * get_syscall(struct threadinfo *t, u_int number, u_int nargs) { struct syscall *sc; const char *name; char *new_name; u_int i; sc = find_syscall(t->proc->abi, number); if (sc != NULL) return (sc); name = sysdecode_syscallname(t->proc->abi->abi, number); if (name == NULL) { asprintf(&new_name, "#%d", number); name = new_name; } else new_name = NULL; STAILQ_FOREACH(sc, &syscalls, entries) { if (strcmp(name, sc->name) == 0) { add_syscall(t->proc->abi, number, sc); free(new_name); return (sc); } } /* It is unknown. Add it into the list. */ #if DEBUG fprintf(stderr, "unknown syscall %s -- setting args to %d\n", name, nargs); #endif sc = calloc(1, sizeof(struct syscall)); sc->name = name; if (new_name != NULL) sc->unknown = true; sc->ret_type = 1; sc->nargs = nargs; for (i = 0; i < nargs; i++) { sc->args[i].offset = i; /* Treat all unknown arguments as LongHex. */ sc->args[i].type = LongHex; } STAILQ_INSERT_HEAD(&syscalls, sc, entries); add_syscall(t->proc->abi, number, sc); return (sc); } /* * Copy a fixed amount of bytes from the process. */ static int get_struct(pid_t pid, void *offset, void *buf, int len) { struct ptrace_io_desc iorequest; iorequest.piod_op = PIOD_READ_D; iorequest.piod_offs = offset; iorequest.piod_addr = buf; iorequest.piod_len = len; if (ptrace(PT_IO, pid, (caddr_t)&iorequest, 0) < 0) return (-1); return (0); } #define MAXSIZE 4096 /* * Copy a string from the process. Note that it is * expected to be a C string, but if max is set, it will * only get that much. */ static char * get_string(pid_t pid, void *addr, int max) { struct ptrace_io_desc iorequest; char *buf, *nbuf; size_t offset, size, totalsize; offset = 0; if (max) size = max + 1; else { /* Read up to the end of the current page. */ size = PAGE_SIZE - ((uintptr_t)addr % PAGE_SIZE); if (size > MAXSIZE) size = MAXSIZE; } totalsize = size; buf = malloc(totalsize); if (buf == NULL) return (NULL); for (;;) { iorequest.piod_op = PIOD_READ_D; iorequest.piod_offs = (char *)addr + offset; iorequest.piod_addr = buf + offset; iorequest.piod_len = size; if (ptrace(PT_IO, pid, (caddr_t)&iorequest, 0) < 0) { free(buf); return (NULL); } if (memchr(buf + offset, '\0', size) != NULL) return (buf); offset += size; if (totalsize < MAXSIZE && max == 0) { size = MAXSIZE - totalsize; if (size > PAGE_SIZE) size = PAGE_SIZE; nbuf = realloc(buf, totalsize + size); if (nbuf == NULL) { buf[totalsize - 1] = '\0'; return (buf); } buf = nbuf; totalsize += size; } else { buf[totalsize - 1] = '\0'; return (buf); } } } static const char * strsig2(int sig) { static char tmp[32]; const char *signame; signame = sysdecode_signal(sig); if (signame == NULL) { snprintf(tmp, sizeof(tmp), "%d", sig); signame = tmp; } return (signame); } static void print_kevent(FILE *fp, struct kevent *ke, int input) { switch (ke->filter) { case EVFILT_READ: case EVFILT_WRITE: case EVFILT_VNODE: case EVFILT_PROC: case EVFILT_TIMER: case EVFILT_PROCDESC: fprintf(fp, "%ju", (uintmax_t)ke->ident); break; case EVFILT_SIGNAL: fputs(strsig2(ke->ident), fp); break; default: fprintf(fp, "%p", (void *)ke->ident); } fprintf(fp, ",%s,%s,", xlookup(kevent_filters, ke->filter), xlookup_bits(kevent_flags, ke->flags)); switch (ke->filter) { case EVFILT_READ: case EVFILT_WRITE: fputs(xlookup_bits(kevent_rdwr_fflags, ke->fflags), fp); break; case EVFILT_VNODE: fputs(xlookup_bits(kevent_vnode_fflags, ke->fflags), fp); break; case EVFILT_PROC: case EVFILT_PROCDESC: fputs(xlookup_bits(kevent_proc_fflags, ke->fflags), fp); break; case EVFILT_TIMER: fputs(xlookup_bits(kevent_timer_fflags, ke->fflags), fp); break; case EVFILT_USER: { int ctrl, data; ctrl = ke->fflags & NOTE_FFCTRLMASK; data = ke->fflags & NOTE_FFLAGSMASK; if (input) { fputs(xlookup(kevent_user_ffctrl, ctrl), fp); if (ke->fflags & NOTE_TRIGGER) fputs("|NOTE_TRIGGER", fp); if (data != 0) fprintf(fp, "|%#x", data); } else { fprintf(fp, "%#x", data); } break; } default: fprintf(fp, "%#x", ke->fflags); } fprintf(fp, ",%#jx,%p", (uintmax_t)ke->data, ke->udata); } static void print_utrace(FILE *fp, void *utrace_addr, size_t len) { unsigned char *utrace_buffer; fprintf(fp, "{ "); if (sysdecode_utrace(fp, utrace_addr, len)) { fprintf(fp, " }"); return; } utrace_buffer = utrace_addr; fprintf(fp, "%zu:", len); while (len--) fprintf(fp, " %02x", *utrace_buffer++); fprintf(fp, " }"); } /* * Converts a syscall argument into a string. Said string is * allocated via malloc(), so needs to be free()'d. sc is * a pointer to the syscall description (see above); args is * an array of all of the system call arguments. */ char * print_arg(struct syscall_args *sc, unsigned long *args, long *retval, struct trussinfo *trussinfo) { FILE *fp; char *tmp; size_t tmplen; pid_t pid; fp = open_memstream(&tmp, &tmplen); pid = trussinfo->curthread->proc->pid; switch (sc->type & ARG_MASK) { case Hex: fprintf(fp, "0x%x", (int)args[sc->offset]); break; case Octal: fprintf(fp, "0%o", (int)args[sc->offset]); break; case Int: fprintf(fp, "%d", (int)args[sc->offset]); break; case UInt: fprintf(fp, "%u", (unsigned int)args[sc->offset]); break; case PUInt: { unsigned int val; if (get_struct(pid, (void *)args[sc->offset], &val, sizeof(val)) == 0) fprintf(fp, "{ %u }", val); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case LongHex: fprintf(fp, "0x%lx", args[sc->offset]); break; case Long: fprintf(fp, "%ld", args[sc->offset]); break; case Sizet: fprintf(fp, "%zu", (size_t)args[sc->offset]); break; case Name: { /* NULL-terminated string. */ char *tmp2; tmp2 = get_string(pid, (void*)args[sc->offset], 0); fprintf(fp, "\"%s\"", tmp2); free(tmp2); break; } case BinString: { /* * Binary block of data that might have printable characters. * XXX If type|OUT, assume that the length is the syscall's * return value. Otherwise, assume that the length of the block * is in the next syscall argument. */ int max_string = trussinfo->strsize; char tmp2[max_string + 1], *tmp3; int len; int truncated = 0; if (sc->type & OUT) len = retval[0]; else len = args[sc->offset + 1]; /* * Don't print more than max_string characters, to avoid word * wrap. If we have to truncate put some ... after the string. */ if (len > max_string) { len = max_string; truncated = 1; } if (len && get_struct(pid, (void*)args[sc->offset], &tmp2, len) != -1) { tmp3 = malloc(len * 4 + 1); while (len) { if (strvisx(tmp3, tmp2, len, VIS_CSTYLE|VIS_TAB|VIS_NL) <= max_string) break; len--; truncated = 1; } fprintf(fp, "\"%s\"%s", tmp3, truncated ? "..." : ""); free(tmp3); } else { fprintf(fp, "0x%lx", args[sc->offset]); } break; } case ExecArgs: case ExecEnv: case StringArray: { uintptr_t addr; union { char *strarray[0]; char buf[PAGE_SIZE]; } u; char *string; size_t len; u_int first, i; /* * Only parse argv[] and environment arrays from exec calls * if requested. */ if (((sc->type & ARG_MASK) == ExecArgs && (trussinfo->flags & EXECVEARGS) == 0) || ((sc->type & ARG_MASK) == ExecEnv && (trussinfo->flags & EXECVEENVS) == 0)) { fprintf(fp, "0x%lx", args[sc->offset]); break; } /* * Read a page of pointers at a time. Punt if the top-level * pointer is not aligned. Note that the first read is of * a partial page. */ addr = args[sc->offset]; if (addr % sizeof(char *) != 0) { fprintf(fp, "0x%lx", args[sc->offset]); break; } len = PAGE_SIZE - (addr & PAGE_MASK); if (get_struct(pid, (void *)addr, u.buf, len) == -1) { fprintf(fp, "0x%lx", args[sc->offset]); break; } fputc('[', fp); first = 1; i = 0; while (u.strarray[i] != NULL) { string = get_string(pid, u.strarray[i], 0); fprintf(fp, "%s \"%s\"", first ? "" : ",", string); free(string); first = 0; i++; if (i == len / sizeof(char *)) { addr += len; len = PAGE_SIZE; if (get_struct(pid, (void *)addr, u.buf, len) == -1) { fprintf(fp, ", "); break; } i = 0; } } fputs(" ]", fp); break; } #ifdef __LP64__ case Quad: fprintf(fp, "%ld", args[sc->offset]); break; case QuadHex: fprintf(fp, "0x%lx", args[sc->offset]); break; #else case Quad: case QuadHex: { unsigned long long ll; #if _BYTE_ORDER == _LITTLE_ENDIAN ll = (unsigned long long)args[sc->offset + 1] << 32 | args[sc->offset]; #else ll = (unsigned long long)args[sc->offset] << 32 | args[sc->offset + 1]; #endif if ((sc->type & ARG_MASK) == Quad) fprintf(fp, "%lld", ll); else fprintf(fp, "0x%llx", ll); break; } #endif case PQuadHex: { uint64_t val; if (get_struct(pid, (void *)args[sc->offset], &val, sizeof(val)) == 0) fprintf(fp, "{ 0x%jx }", (uintmax_t)val); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Ptr: fprintf(fp, "0x%lx", args[sc->offset]); break; case Readlinkres: { char *tmp2; if (retval[0] == -1) break; tmp2 = get_string(pid, (void*)args[sc->offset], retval[0]); fprintf(fp, "\"%s\"", tmp2); free(tmp2); break; } case Ioctl: { const char *temp; unsigned long cmd; cmd = args[sc->offset]; temp = sysdecode_ioctlname(cmd); if (temp) fputs(temp, fp); else { fprintf(fp, "0x%lx { IO%s%s 0x%lx('%c'), %lu, %lu }", cmd, cmd & IOC_OUT ? "R" : "", cmd & IOC_IN ? "W" : "", IOCGROUP(cmd), isprint(IOCGROUP(cmd)) ? (char)IOCGROUP(cmd) : '?', cmd & 0xFF, IOCPARM_LEN(cmd)); } break; } case Timespec: { struct timespec ts; if (get_struct(pid, (void *)args[sc->offset], &ts, sizeof(ts)) != -1) fprintf(fp, "{ %jd.%09ld }", (intmax_t)ts.tv_sec, ts.tv_nsec); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Timespec2: { struct timespec ts[2]; const char *sep; unsigned int i; if (get_struct(pid, (void *)args[sc->offset], &ts, sizeof(ts)) != -1) { fputs("{ ", fp); sep = ""; for (i = 0; i < nitems(ts); i++) { fputs(sep, fp); sep = ", "; switch (ts[i].tv_nsec) { case UTIME_NOW: fprintf(fp, "UTIME_NOW"); break; case UTIME_OMIT: fprintf(fp, "UTIME_OMIT"); break; default: fprintf(fp, "%jd.%09ld", (intmax_t)ts[i].tv_sec, ts[i].tv_nsec); break; } } fputs(" }", fp); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Timeval: { struct timeval tv; if (get_struct(pid, (void *)args[sc->offset], &tv, sizeof(tv)) != -1) fprintf(fp, "{ %jd.%06ld }", (intmax_t)tv.tv_sec, tv.tv_usec); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Timeval2: { struct timeval tv[2]; if (get_struct(pid, (void *)args[sc->offset], &tv, sizeof(tv)) != -1) fprintf(fp, "{ %jd.%06ld, %jd.%06ld }", (intmax_t)tv[0].tv_sec, tv[0].tv_usec, (intmax_t)tv[1].tv_sec, tv[1].tv_usec); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Itimerval: { struct itimerval itv; if (get_struct(pid, (void *)args[sc->offset], &itv, sizeof(itv)) != -1) fprintf(fp, "{ %jd.%06ld, %jd.%06ld }", (intmax_t)itv.it_interval.tv_sec, itv.it_interval.tv_usec, (intmax_t)itv.it_value.tv_sec, itv.it_value.tv_usec); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case LinuxSockArgs: { struct linux_socketcall_args largs; if (get_struct(pid, (void *)args[sc->offset], (void *)&largs, sizeof(largs)) != -1) fprintf(fp, "{ %s, 0x%lx }", lookup(linux_socketcall_ops, largs.what, 10), (long unsigned int)largs.args); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Pollfd: { /* * XXX: A Pollfd argument expects the /next/ syscall argument * to be the number of fds in the array. This matches the poll * syscall. */ struct pollfd *pfd; int numfds = args[sc->offset + 1]; size_t bytes = sizeof(struct pollfd) * numfds; int i; if ((pfd = malloc(bytes)) == NULL) err(1, "Cannot malloc %zu bytes for pollfd array", bytes); if (get_struct(pid, (void *)args[sc->offset], pfd, bytes) != -1) { fputs("{", fp); for (i = 0; i < numfds; i++) { fprintf(fp, " %d/%s", pfd[i].fd, xlookup_bits(poll_flags, pfd[i].events)); } fputs(" }", fp); } else { fprintf(fp, "0x%lx", args[sc->offset]); } free(pfd); break; } case Fd_set: { /* * XXX: A Fd_set argument expects the /first/ syscall argument * to be the number of fds in the array. This matches the * select syscall. */ fd_set *fds; int numfds = args[0]; size_t bytes = _howmany(numfds, _NFDBITS) * _NFDBITS; int i; if ((fds = malloc(bytes)) == NULL) err(1, "Cannot malloc %zu bytes for fd_set array", bytes); if (get_struct(pid, (void *)args[sc->offset], fds, bytes) != -1) { fputs("{", fp); for (i = 0; i < numfds; i++) { if (FD_ISSET(i, fds)) fprintf(fp, " %d", i); } fputs(" }", fp); } else fprintf(fp, "0x%lx", args[sc->offset]); free(fds); break; } case Signal: fputs(strsig2(args[sc->offset]), fp); break; case Sigset: { long sig; sigset_t ss; int i, first; sig = args[sc->offset]; if (get_struct(pid, (void *)args[sc->offset], (void *)&ss, sizeof(ss)) == -1) { fprintf(fp, "0x%lx", args[sc->offset]); break; } fputs("{ ", fp); first = 1; for (i = 1; i < sys_nsig; i++) { if (sigismember(&ss, i)) { fprintf(fp, "%s%s", !first ? "|" : "", strsig2(i)); first = 0; } } if (!first) fputc(' ', fp); fputc('}', fp); break; } case Sigprocmask: print_integer_arg(sysdecode_sigprocmask_how, fp, args[sc->offset]); break; case Fcntlflag: /* XXX: Output depends on the value of the previous argument. */ if (sysdecode_fcntl_arg_p(args[sc->offset - 1])) sysdecode_fcntl_arg(fp, args[sc->offset - 1], args[sc->offset], 16); break; case Open: print_mask_arg(sysdecode_open_flags, fp, args[sc->offset]); break; case Fcntl: print_integer_arg(sysdecode_fcntl_cmd, fp, args[sc->offset]); break; case Mprot: print_mask_arg(sysdecode_mmap_prot, fp, args[sc->offset]); break; case Mmapflags: print_mask_arg(sysdecode_mmap_flags, fp, args[sc->offset]); break; case Whence: print_integer_arg(sysdecode_whence, fp, args[sc->offset]); break; case Sockdomain: print_integer_arg(sysdecode_socketdomain, fp, args[sc->offset]); break; case Socktype: print_mask_arg(sysdecode_socket_type, fp, args[sc->offset]); break; case Shutdown: print_integer_arg(sysdecode_shutdown_how, fp, args[sc->offset]); break; case Resource: print_integer_arg(sysdecode_rlimit, fp, args[sc->offset]); break; case RusageWho: print_integer_arg(sysdecode_getrusage_who, fp, args[sc->offset]); break; case Pathconf: fputs(xlookup(pathconf_arg, args[sc->offset]), fp); break; case Rforkflags: print_mask_arg(sysdecode_rfork_flags, fp, args[sc->offset]); break; case Sockaddr: { char addr[64]; struct sockaddr_in *lsin; struct sockaddr_in6 *lsin6; struct sockaddr_un *sun; struct sockaddr *sa; socklen_t len; u_char *q; if (args[sc->offset] == 0) { fputs("NULL", fp); break; } /* * Extract the address length from the next argument. If * this is an output sockaddr (OUT is set), then the * next argument is a pointer to a socklen_t. Otherwise * the next argument contains a socklen_t by value. */ if (sc->type & OUT) { if (get_struct(pid, (void *)args[sc->offset + 1], &len, sizeof(len)) == -1) { fprintf(fp, "0x%lx", args[sc->offset]); break; } } else len = args[sc->offset + 1]; /* If the length is too small, just bail. */ if (len < sizeof(*sa)) { fprintf(fp, "0x%lx", args[sc->offset]); break; } sa = calloc(1, len); if (get_struct(pid, (void *)args[sc->offset], sa, len) == -1) { free(sa); fprintf(fp, "0x%lx", args[sc->offset]); break; } switch (sa->sa_family) { case AF_INET: if (len < sizeof(*lsin)) goto sockaddr_short; lsin = (struct sockaddr_in *)(void *)sa; inet_ntop(AF_INET, &lsin->sin_addr, addr, sizeof(addr)); fprintf(fp, "{ AF_INET %s:%d }", addr, htons(lsin->sin_port)); break; case AF_INET6: if (len < sizeof(*lsin6)) goto sockaddr_short; lsin6 = (struct sockaddr_in6 *)(void *)sa; inet_ntop(AF_INET6, &lsin6->sin6_addr, addr, sizeof(addr)); fprintf(fp, "{ AF_INET6 [%s]:%d }", addr, htons(lsin6->sin6_port)); break; case AF_UNIX: sun = (struct sockaddr_un *)sa; fprintf(fp, "{ AF_UNIX \"%.*s\" }", (int)(len - offsetof(struct sockaddr_un, sun_path)), sun->sun_path); break; default: sockaddr_short: fprintf(fp, "{ sa_len = %d, sa_family = %d, sa_data = {", (int)sa->sa_len, (int)sa->sa_family); for (q = (u_char *)sa->sa_data; q < (u_char *)sa + len; q++) fprintf(fp, "%s 0x%02x", q == (u_char *)sa->sa_data ? "" : ",", *q); fputs(" } }", fp); } free(sa); break; } case Sigaction: { struct sigaction sa; if (get_struct(pid, (void *)args[sc->offset], &sa, sizeof(sa)) != -1) { fputs("{ ", fp); if (sa.sa_handler == SIG_DFL) fputs("SIG_DFL", fp); else if (sa.sa_handler == SIG_IGN) fputs("SIG_IGN", fp); else fprintf(fp, "%p", sa.sa_handler); fprintf(fp, " %s ss_t }", xlookup_bits(sigaction_flags, sa.sa_flags)); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Kevent: { /* * XXX XXX: The size of the array is determined by either the * next syscall argument, or by the syscall return value, * depending on which argument number we are. This matches the * kevent syscall, but luckily that's the only syscall that uses * them. */ struct kevent *ke; int numevents = -1; size_t bytes; int i; if (sc->offset == 1) numevents = args[sc->offset+1]; else if (sc->offset == 3 && retval[0] != -1) numevents = retval[0]; if (numevents >= 0) { bytes = sizeof(struct kevent) * numevents; if ((ke = malloc(bytes)) == NULL) err(1, "Cannot malloc %zu bytes for kevent array", bytes); } else ke = NULL; if (numevents >= 0 && get_struct(pid, (void *)args[sc->offset], ke, bytes) != -1) { fputc('{', fp); for (i = 0; i < numevents; i++) { fputc(' ', fp); print_kevent(fp, &ke[i], sc->offset == 1); } fputs(" }", fp); } else { fprintf(fp, "0x%lx", args[sc->offset]); } free(ke); break; } case Stat: { struct stat st; if (get_struct(pid, (void *)args[sc->offset], &st, sizeof(st)) != -1) { char mode[12]; strmode(st.st_mode, mode); fprintf(fp, "{ mode=%s,inode=%ju,size=%jd,blksize=%ld }", mode, (uintmax_t)st.st_ino, (intmax_t)st.st_size, (long)st.st_blksize); } else { fprintf(fp, "0x%lx", args[sc->offset]); } break; } case Stat11: { struct freebsd11_stat st; if (get_struct(pid, (void *)args[sc->offset], &st, sizeof(st)) != -1) { char mode[12]; strmode(st.st_mode, mode); fprintf(fp, "{ mode=%s,inode=%ju,size=%jd,blksize=%ld }", mode, (uintmax_t)st.st_ino, (intmax_t)st.st_size, (long)st.st_blksize); } else { fprintf(fp, "0x%lx", args[sc->offset]); } break; } case StatFs: { unsigned int i; struct statfs buf; if (get_struct(pid, (void *)args[sc->offset], &buf, sizeof(buf)) != -1) { char fsid[17]; bzero(fsid, sizeof(fsid)); if (buf.f_fsid.val[0] != 0 || buf.f_fsid.val[1] != 0) { for (i = 0; i < sizeof(buf.f_fsid); i++) snprintf(&fsid[i*2], sizeof(fsid) - (i*2), "%02x", ((u_char *)&buf.f_fsid)[i]); } fprintf(fp, "{ fstypename=%s,mntonname=%s,mntfromname=%s," "fsid=%s }", buf.f_fstypename, buf.f_mntonname, buf.f_mntfromname, fsid); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Rusage: { struct rusage ru; if (get_struct(pid, (void *)args[sc->offset], &ru, sizeof(ru)) != -1) { fprintf(fp, "{ u=%jd.%06ld,s=%jd.%06ld,in=%ld,out=%ld }", (intmax_t)ru.ru_utime.tv_sec, ru.ru_utime.tv_usec, (intmax_t)ru.ru_stime.tv_sec, ru.ru_stime.tv_usec, ru.ru_inblock, ru.ru_oublock); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Rlimit: { struct rlimit rl; if (get_struct(pid, (void *)args[sc->offset], &rl, sizeof(rl)) != -1) { fprintf(fp, "{ cur=%ju,max=%ju }", rl.rlim_cur, rl.rlim_max); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case ExitStatus: { int status; if (get_struct(pid, (void *)args[sc->offset], &status, sizeof(status)) != -1) { fputs("{ ", fp); if (WIFCONTINUED(status)) fputs("CONTINUED", fp); else if (WIFEXITED(status)) fprintf(fp, "EXITED,val=%d", WEXITSTATUS(status)); else if (WIFSIGNALED(status)) fprintf(fp, "SIGNALED,sig=%s%s", strsig2(WTERMSIG(status)), WCOREDUMP(status) ? ",cored" : ""); else fprintf(fp, "STOPPED,sig=%s", strsig2(WTERMSIG(status))); fputs(" }", fp); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Waitoptions: print_mask_arg(sysdecode_wait6_options, fp, args[sc->offset]); break; case Idtype: print_integer_arg(sysdecode_idtype, fp, args[sc->offset]); break; case Procctl: print_integer_arg(sysdecode_procctl_cmd, fp, args[sc->offset]); break; case Umtxop: print_integer_arg(sysdecode_umtx_op, fp, args[sc->offset]); break; case Atfd: print_integer_arg(sysdecode_atfd, fp, args[sc->offset]); break; case Atflags: fputs(xlookup_bits(at_flags, args[sc->offset]), fp); break; case Accessmode: print_mask_arg(sysdecode_access_mode, fp, args[sc->offset]); break; case Sysarch: fputs(xlookup(sysarch_ops, args[sc->offset]), fp); break; case PipeFds: /* * The pipe() system call in the kernel returns its * two file descriptors via return values. However, * the interface exposed by libc is that pipe() * accepts a pointer to an array of descriptors. * Format the output to match the libc API by printing * the returned file descriptors as a fake argument. * * Overwrite the first retval to signal a successful * return as well. */ fprintf(fp, "{ %ld, %ld }", retval[0], retval[1]); retval[0] = 0; break; case Utrace: { size_t len; void *utrace_addr; len = args[sc->offset + 1]; utrace_addr = calloc(1, len); if (get_struct(pid, (void *)args[sc->offset], (void *)utrace_addr, len) != -1) print_utrace(fp, utrace_addr, len); else fprintf(fp, "0x%lx", args[sc->offset]); free(utrace_addr); break; } case IntArray: { int descriptors[16]; unsigned long i, ndescriptors; bool truncated; ndescriptors = args[sc->offset + 1]; truncated = false; if (ndescriptors > nitems(descriptors)) { ndescriptors = nitems(descriptors); truncated = true; } if (get_struct(pid, (void *)args[sc->offset], descriptors, ndescriptors * sizeof(descriptors[0])) != -1) { fprintf(fp, "{"); for (i = 0; i < ndescriptors; i++) fprintf(fp, i == 0 ? " %d" : ", %d", descriptors[i]); fprintf(fp, truncated ? ", ... }" : " }"); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Pipe2: print_mask_arg(sysdecode_pipe2_flags, fp, args[sc->offset]); break; case CapFcntlRights: { uint32_t rights; if (sc->type & OUT) { if (get_struct(pid, (void *)args[sc->offset], &rights, sizeof(rights)) == -1) { fprintf(fp, "0x%lx", args[sc->offset]); break; } } else rights = args[sc->offset]; print_mask_arg32(sysdecode_cap_fcntlrights, fp, rights); break; } case Fadvice: print_integer_arg(sysdecode_fadvice, fp, args[sc->offset]); break; case FileFlags: { fflags_t rem; if (!sysdecode_fileflags(fp, args[sc->offset], &rem)) fprintf(fp, "0x%x", rem); else if (rem != 0) fprintf(fp, "|0x%x", rem); break; } case Flockop: print_mask_arg(sysdecode_flock_operation, fp, args[sc->offset]); break; case Getfsstatmode: print_integer_arg(sysdecode_getfsstat_mode, fp, args[sc->offset]); break; case Kldsymcmd: print_integer_arg(sysdecode_kldsym_cmd, fp, args[sc->offset]); break; case Kldunloadflags: print_integer_arg(sysdecode_kldunload_flags, fp, args[sc->offset]); break; case Madvice: print_integer_arg(sysdecode_madvice, fp, args[sc->offset]); break; case Socklent: fprintf(fp, "%u", (socklen_t)args[sc->offset]); break; case Sockprotocol: { const char *temp; int domain, protocol; domain = args[sc->offset - 2]; protocol = args[sc->offset]; if (protocol == 0) { fputs("0", fp); } else { temp = sysdecode_socket_protocol(domain, protocol); if (temp) { fputs(temp, fp); } else { fprintf(fp, "%d", protocol); } } break; } case Sockoptlevel: print_integer_arg(sysdecode_sockopt_level, fp, args[sc->offset]); break; case Sockoptname: { const char *temp; int level, name; level = args[sc->offset - 1]; name = args[sc->offset]; temp = sysdecode_sockopt_name(level, name); if (temp) { fputs(temp, fp); } else { fprintf(fp, "%d", name); } break; } case Msgflags: print_mask_arg(sysdecode_msg_flags, fp, args[sc->offset]); break; case CapRights: { cap_rights_t rights; if (get_struct(pid, (void *)args[sc->offset], &rights, sizeof(rights)) != -1) { fputs("{ ", fp); sysdecode_cap_rights(fp, &rights); fputs(" }", fp); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case Acltype: print_integer_arg(sysdecode_acltype, fp, args[sc->offset]); break; case Extattrnamespace: print_integer_arg(sysdecode_extattrnamespace, fp, args[sc->offset]); break; case Minherit: print_integer_arg(sysdecode_minherit_inherit, fp, args[sc->offset]); break; case Mlockall: print_mask_arg(sysdecode_mlockall_flags, fp, args[sc->offset]); break; case Mountflags: print_mask_arg(sysdecode_mount_flags, fp, args[sc->offset]); break; case Msync: print_mask_arg(sysdecode_msync_flags, fp, args[sc->offset]); break; case Priowhich: print_integer_arg(sysdecode_prio_which, fp, args[sc->offset]); break; case Ptraceop: print_integer_arg(sysdecode_ptrace_request, fp, args[sc->offset]); break; case Quotactlcmd: if (!sysdecode_quotactl_cmd(fp, args[sc->offset])) fprintf(fp, "%#x", (int)args[sc->offset]); break; case Reboothowto: print_mask_arg(sysdecode_reboot_howto, fp, args[sc->offset]); break; case Rtpriofunc: print_integer_arg(sysdecode_rtprio_function, fp, args[sc->offset]); break; case Schedpolicy: print_integer_arg(sysdecode_scheduler_policy, fp, args[sc->offset]); break; case Schedparam: { struct sched_param sp; if (get_struct(pid, (void *)args[sc->offset], &sp, sizeof(sp)) != -1) fprintf(fp, "{ %d }", sp.sched_priority); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case CloudABIAdvice: fputs(xlookup(cloudabi_advice, args[sc->offset]), fp); break; case CloudABIClockID: fputs(xlookup(cloudabi_clockid, args[sc->offset]), fp); break; case ClouduABIFDSFlags: fputs(xlookup_bits(cloudabi_fdsflags, args[sc->offset]), fp); break; case CloudABIFDStat: { cloudabi_fdstat_t fds; if (get_struct(pid, (void *)args[sc->offset], &fds, sizeof(fds)) != -1) { fprintf(fp, "{ %s, ", xlookup(cloudabi_filetype, fds.fs_filetype)); fprintf(fp, "%s, ... }", xlookup_bits(cloudabi_fdflags, fds.fs_flags)); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case CloudABIFileStat: { cloudabi_filestat_t fsb; if (get_struct(pid, (void *)args[sc->offset], &fsb, sizeof(fsb)) != -1) fprintf(fp, "{ %s, %ju }", xlookup(cloudabi_filetype, fsb.st_filetype), (uintmax_t)fsb.st_size); else fprintf(fp, "0x%lx", args[sc->offset]); break; } case CloudABIFileType: fputs(xlookup(cloudabi_filetype, args[sc->offset]), fp); break; case CloudABIFSFlags: fputs(xlookup_bits(cloudabi_fsflags, args[sc->offset]), fp); break; case CloudABILookup: if ((args[sc->offset] & CLOUDABI_LOOKUP_SYMLINK_FOLLOW) != 0) fprintf(fp, "%d|LOOKUP_SYMLINK_FOLLOW", (int)args[sc->offset]); else fprintf(fp, "%d", (int)args[sc->offset]); break; case CloudABIMFlags: fputs(xlookup_bits(cloudabi_mflags, args[sc->offset]), fp); break; case CloudABIMProt: fputs(xlookup_bits(cloudabi_mprot, args[sc->offset]), fp); break; case CloudABIMSFlags: fputs(xlookup_bits(cloudabi_msflags, args[sc->offset]), fp); break; case CloudABIOFlags: fputs(xlookup_bits(cloudabi_oflags, args[sc->offset]), fp); break; case CloudABISDFlags: fputs(xlookup_bits(cloudabi_sdflags, args[sc->offset]), fp); break; case CloudABISignal: fputs(xlookup(cloudabi_signal, args[sc->offset]), fp); break; case CloudABISockStat: { cloudabi_sockstat_t ss; if (get_struct(pid, (void *)args[sc->offset], &ss, sizeof(ss)) != -1) { - fprintf(fp, "{ %s, ", xlookup( - cloudabi_sa_family, ss.ss_sockname.sa_family)); - fprintf(fp, "%s, ", xlookup( - cloudabi_sa_family, ss.ss_peername.sa_family)); fprintf(fp, "%s, ", xlookup( cloudabi_errno, ss.ss_error)); fprintf(fp, "%s }", xlookup_bits( cloudabi_ssstate, ss.ss_state)); } else fprintf(fp, "0x%lx", args[sc->offset]); break; } case CloudABISSFlags: fputs(xlookup_bits(cloudabi_ssflags, args[sc->offset]), fp); break; case CloudABITimestamp: fprintf(fp, "%lu.%09lus", args[sc->offset] / 1000000000, args[sc->offset] % 1000000000); break; case CloudABIULFlags: fputs(xlookup_bits(cloudabi_ulflags, args[sc->offset]), fp); break; case CloudABIWhence: fputs(xlookup(cloudabi_whence, args[sc->offset]), fp); break; default: errx(1, "Invalid argument type %d\n", sc->type & ARG_MASK); } fclose(fp); return (tmp); } /* * Print (to outfile) the system call and its arguments. */ void print_syscall(struct trussinfo *trussinfo) { struct threadinfo *t; const char *name; char **s_args; int i, len, nargs; t = trussinfo->curthread; name = t->cs.sc->name; nargs = t->cs.nargs; s_args = t->cs.s_args; len = print_line_prefix(trussinfo); len += fprintf(trussinfo->outfile, "%s(", name); for (i = 0; i < nargs; i++) { if (s_args[i] != NULL) len += fprintf(trussinfo->outfile, "%s", s_args[i]); else len += fprintf(trussinfo->outfile, ""); len += fprintf(trussinfo->outfile, "%s", i < (nargs - 1) ? "," : ""); } len += fprintf(trussinfo->outfile, ")"); for (i = 0; i < 6 - (len / 8); i++) fprintf(trussinfo->outfile, "\t"); } void print_syscall_ret(struct trussinfo *trussinfo, int errorp, long *retval) { struct timespec timediff; struct threadinfo *t; struct syscall *sc; int error; t = trussinfo->curthread; sc = t->cs.sc; if (trussinfo->flags & COUNTONLY) { timespecsubt(&t->after, &t->before, &timediff); timespecadd(&sc->time, &timediff, &sc->time); sc->ncalls++; if (errorp) sc->nerror++; return; } print_syscall(trussinfo); fflush(trussinfo->outfile); if (retval == NULL) { /* * This system call resulted in the current thread's exit, * so there is no return value or error to display. */ fprintf(trussinfo->outfile, "\n"); return; } if (errorp) { error = sysdecode_abi_to_freebsd_errno(t->proc->abi->abi, retval[0]); fprintf(trussinfo->outfile, " ERR#%ld '%s'\n", retval[0], error == INT_MAX ? "Unknown error" : strerror(error)); } #ifndef __LP64__ else if (sc->ret_type == 2) { off_t off; #if _BYTE_ORDER == _LITTLE_ENDIAN off = (off_t)retval[1] << 32 | retval[0]; #else off = (off_t)retval[0] << 32 | retval[1]; #endif fprintf(trussinfo->outfile, " = %jd (0x%jx)\n", (intmax_t)off, (intmax_t)off); } #endif else fprintf(trussinfo->outfile, " = %ld (0x%lx)\n", retval[0], retval[0]); } void print_summary(struct trussinfo *trussinfo) { struct timespec total = {0, 0}; struct syscall *sc; int ncall, nerror; fprintf(trussinfo->outfile, "%-20s%15s%8s%8s\n", "syscall", "seconds", "calls", "errors"); ncall = nerror = 0; STAILQ_FOREACH(sc, &syscalls, entries) if (sc->ncalls) { fprintf(trussinfo->outfile, "%-20s%5jd.%09ld%8d%8d\n", sc->name, (intmax_t)sc->time.tv_sec, sc->time.tv_nsec, sc->ncalls, sc->nerror); timespecadd(&total, &sc->time, &total); ncall += sc->ncalls; nerror += sc->nerror; } fprintf(trussinfo->outfile, "%20s%15s%8s%8s\n", "", "-------------", "-------", "-------"); fprintf(trussinfo->outfile, "%-20s%5jd.%09ld%8d%8d\n", "", (intmax_t)total.tv_sec, total.tv_nsec, ncall, nerror); } Index: projects/runtime-coverage =================================================================== --- projects/runtime-coverage (revision 321584) +++ projects/runtime-coverage (revision 321585) Property changes on: projects/runtime-coverage ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r321458-321584