Index: projects/clang380-import/contrib/llvm/patches/patch-02-add-CC-aliases.diff =================================================================== --- projects/clang380-import/contrib/llvm/patches/patch-02-add-CC-aliases.diff (revision 292950) +++ projects/clang380-import/contrib/llvm/patches/patch-02-add-CC-aliases.diff (nonexistent) @@ -1,23 +0,0 @@ -This patch adds "CC" and "clang-CC" to the list of program name aliases which -invoke the C++ compiler. - -Introduced here: http://svnweb.freebsd.org/changeset/base/257109 - -Index: tools/clang/tools/driver/driver.cpp -=================================================================== ---- tools/clang/tools/driver/driver.cpp -+++ tools/clang/tools/driver/driver.cpp -@@ -213,11 +213,13 @@ static const DriverSuffix *FindDriverSuffix(String - {"clang", nullptr}, - {"clang++", "--driver-mode=g++"}, - {"clang-c++", "--driver-mode=g++"}, -+ {"clang-CC", "--driver-mode=g++"}, - {"clang-cc", nullptr}, - {"clang-cpp", "--driver-mode=cpp"}, - {"clang-g++", "--driver-mode=g++"}, - {"clang-gcc", nullptr}, - {"clang-cl", "--driver-mode=cl"}, -+ {"CC", "--driver-mode=g++"}, - {"cc", nullptr}, - {"cpp", "--driver-mode=cpp"}, - {"cl", "--driver-mode=cl"}, Property changes on: projects/clang380-import/contrib/llvm/patches/patch-02-add-CC-aliases.diff ___________________________________________________________________ Deleted: svn:mime-type ## -1 +0,0 ## -text/x-diff \ No newline at end of property Index: projects/clang380-import/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp =================================================================== --- projects/clang380-import/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp (revision 292950) +++ projects/clang380-import/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp (revision 292951) @@ -1,674 +1,668 @@ //===--- ToolChain.cpp - Collections of tools for one platform ------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "Tools.h" #include "clang/Basic/ObjCRuntime.h" #include "clang/Driver/Action.h" #include "clang/Driver/Driver.h" #include "clang/Driver/DriverDiagnostic.h" #include "clang/Driver/Options.h" #include "clang/Driver/SanitizerArgs.h" #include "clang/Driver/ToolChain.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Option/Arg.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetParser.h" using namespace clang::driver; using namespace clang::driver::tools; using namespace clang; using namespace llvm; using namespace llvm::opt; static llvm::opt::Arg *GetRTTIArgument(const ArgList &Args) { return Args.getLastArg(options::OPT_mkernel, options::OPT_fapple_kext, options::OPT_fno_rtti, options::OPT_frtti); } static ToolChain::RTTIMode CalculateRTTIMode(const ArgList &Args, const llvm::Triple &Triple, const Arg *CachedRTTIArg) { // Explicit rtti/no-rtti args if (CachedRTTIArg) { if (CachedRTTIArg->getOption().matches(options::OPT_frtti)) return ToolChain::RM_EnabledExplicitly; else return ToolChain::RM_DisabledExplicitly; } // -frtti is default, except for the PS4 CPU. if (!Triple.isPS4CPU()) return ToolChain::RM_EnabledImplicitly; // On the PS4, turning on c++ exceptions turns on rtti. // We're assuming that, if we see -fexceptions, rtti gets turned on. Arg *Exceptions = Args.getLastArgNoClaim( options::OPT_fcxx_exceptions, options::OPT_fno_cxx_exceptions, options::OPT_fexceptions, options::OPT_fno_exceptions); if (Exceptions && (Exceptions->getOption().matches(options::OPT_fexceptions) || Exceptions->getOption().matches(options::OPT_fcxx_exceptions))) return ToolChain::RM_EnabledImplicitly; return ToolChain::RM_DisabledImplicitly; } ToolChain::ToolChain(const Driver &D, const llvm::Triple &T, const ArgList &Args) : D(D), Triple(T), Args(Args), CachedRTTIArg(GetRTTIArgument(Args)), CachedRTTIMode(CalculateRTTIMode(Args, Triple, CachedRTTIArg)) { if (Arg *A = Args.getLastArg(options::OPT_mthread_model)) if (!isThreadModelSupported(A->getValue())) D.Diag(diag::err_drv_invalid_thread_model_for_target) << A->getValue() << A->getAsString(Args); } ToolChain::~ToolChain() { } vfs::FileSystem &ToolChain::getVFS() const { return getDriver().getVFS(); } bool ToolChain::useIntegratedAs() const { return Args.hasFlag(options::OPT_fintegrated_as, options::OPT_fno_integrated_as, IsIntegratedAssemblerDefault()); } const SanitizerArgs& ToolChain::getSanitizerArgs() const { if (!SanitizerArguments.get()) SanitizerArguments.reset(new SanitizerArgs(*this, Args)); return *SanitizerArguments.get(); } namespace { struct DriverSuffix { const char *Suffix; const char *ModeFlag; }; const DriverSuffix *FindDriverSuffix(StringRef ProgName) { // A list of known driver suffixes. Suffixes are compared against the // program name in order. If there is a match, the frontend type is updated as // necessary by applying the ModeFlag. static const DriverSuffix DriverSuffixes[] = { {"clang", nullptr}, {"clang++", "--driver-mode=g++"}, {"clang-c++", "--driver-mode=g++"}, -#ifdef __FreeBSD__ - {"clang-CC", "--driver-mode=g++"}, -#endif {"clang-cc", nullptr}, {"clang-cpp", "--driver-mode=cpp"}, {"clang-g++", "--driver-mode=g++"}, {"clang-gcc", nullptr}, {"clang-cl", "--driver-mode=cl"}, -#ifdef __FreeBSD__ - {"CC", "--driver-mode=g++"}, -#endif {"cc", nullptr}, {"cpp", "--driver-mode=cpp"}, {"cl", "--driver-mode=cl"}, {"++", "--driver-mode=g++"}, }; for (size_t i = 0; i < llvm::array_lengthof(DriverSuffixes); ++i) if (ProgName.endswith(DriverSuffixes[i].Suffix)) return &DriverSuffixes[i]; return nullptr; } /// Normalize the program name from argv[0] by stripping the file extension if /// present and lower-casing the string on Windows. std::string normalizeProgramName(llvm::StringRef Argv0) { std::string ProgName = llvm::sys::path::stem(Argv0); #ifdef LLVM_ON_WIN32 // Transform to lowercase for case insensitive file systems. std::transform(ProgName.begin(), ProgName.end(), ProgName.begin(), ::tolower); #endif return ProgName; } const DriverSuffix *parseDriverSuffix(StringRef ProgName) { // Try to infer frontend type and default target from the program name by // comparing it against DriverSuffixes in order. // If there is a match, the function tries to identify a target as prefix. // E.g. "x86_64-linux-clang" as interpreted as suffix "clang" with target // prefix "x86_64-linux". If such a target prefix is found, it may be // added via -target as implicit first argument. const DriverSuffix *DS = FindDriverSuffix(ProgName); if (!DS) { // Try again after stripping any trailing version number: // clang++3.5 -> clang++ ProgName = ProgName.rtrim("0123456789."); DS = FindDriverSuffix(ProgName); } if (!DS) { // Try again after stripping trailing -component. // clang++-tot -> clang++ ProgName = ProgName.slice(0, ProgName.rfind('-')); DS = FindDriverSuffix(ProgName); } return DS; } } // anonymous namespace std::pair ToolChain::getTargetAndModeFromProgramName(StringRef PN) { std::string ProgName = normalizeProgramName(PN); const DriverSuffix *DS = parseDriverSuffix(ProgName); if (!DS) return std::make_pair("", ""); std::string ModeFlag = DS->ModeFlag == nullptr ? "" : DS->ModeFlag; std::string::size_type LastComponent = ProgName.rfind('-', ProgName.size() - strlen(DS->Suffix)); if (LastComponent == std::string::npos) return std::make_pair("", ModeFlag); // Infer target from the prefix. StringRef Prefix(ProgName); Prefix = Prefix.slice(0, LastComponent); std::string IgnoredError; std::string Target; if (llvm::TargetRegistry::lookupTarget(Prefix, IgnoredError)) { Target = Prefix; } return std::make_pair(Target, ModeFlag); } StringRef ToolChain::getDefaultUniversalArchName() const { // In universal driver terms, the arch name accepted by -arch isn't exactly // the same as the ones that appear in the triple. Roughly speaking, this is // an inverse of the darwin::getArchTypeForDarwinArchName() function, but the // only interesting special case is powerpc. switch (Triple.getArch()) { case llvm::Triple::ppc: return "ppc"; case llvm::Triple::ppc64: return "ppc64"; case llvm::Triple::ppc64le: return "ppc64le"; default: return Triple.getArchName(); } } bool ToolChain::IsUnwindTablesDefault() const { return false; } Tool *ToolChain::getClang() const { if (!Clang) Clang.reset(new tools::Clang(*this)); return Clang.get(); } Tool *ToolChain::buildAssembler() const { return new tools::ClangAs(*this); } Tool *ToolChain::buildLinker() const { llvm_unreachable("Linking is not supported by this toolchain"); } Tool *ToolChain::getAssemble() const { if (!Assemble) Assemble.reset(buildAssembler()); return Assemble.get(); } Tool *ToolChain::getClangAs() const { if (!Assemble) Assemble.reset(new tools::ClangAs(*this)); return Assemble.get(); } Tool *ToolChain::getLink() const { if (!Link) Link.reset(buildLinker()); return Link.get(); } Tool *ToolChain::getTool(Action::ActionClass AC) const { switch (AC) { case Action::AssembleJobClass: return getAssemble(); case Action::LinkJobClass: return getLink(); case Action::InputClass: case Action::BindArchClass: case Action::CudaDeviceClass: case Action::CudaHostClass: case Action::LipoJobClass: case Action::DsymutilJobClass: case Action::VerifyDebugInfoJobClass: llvm_unreachable("Invalid tool kind."); case Action::CompileJobClass: case Action::PrecompileJobClass: case Action::PreprocessJobClass: case Action::AnalyzeJobClass: case Action::MigrateJobClass: case Action::VerifyPCHJobClass: case Action::BackendJobClass: return getClang(); } llvm_unreachable("Invalid tool kind."); } static StringRef getArchNameForCompilerRTLib(const ToolChain &TC, const ArgList &Args) { const llvm::Triple &Triple = TC.getTriple(); bool IsWindows = Triple.isOSWindows(); if (Triple.isWindowsMSVCEnvironment() && TC.getArch() == llvm::Triple::x86) return "i386"; if (TC.getArch() == llvm::Triple::arm || TC.getArch() == llvm::Triple::armeb) return (arm::getARMFloatABI(TC, Args) == arm::FloatABI::Hard && !IsWindows) ? "armhf" : "arm"; return TC.getArchName(); } std::string ToolChain::getCompilerRT(const ArgList &Args, StringRef Component, bool Shared) const { const llvm::Triple &TT = getTriple(); const char *Env = TT.isAndroid() ? "-android" : ""; bool IsITANMSVCWindows = TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment(); StringRef Arch = getArchNameForCompilerRTLib(*this, Args); const char *Prefix = IsITANMSVCWindows ? "" : "lib"; const char *Suffix = Shared ? (Triple.isOSWindows() ? ".dll" : ".so") : (IsITANMSVCWindows ? ".lib" : ".a"); SmallString<128> Path(getDriver().ResourceDir); StringRef OSLibName = Triple.isOSFreeBSD() ? "freebsd" : getOS(); llvm::sys::path::append(Path, "lib", OSLibName); llvm::sys::path::append(Path, Prefix + Twine("clang_rt.") + Component + "-" + Arch + Env + Suffix); return Path.str(); } const char *ToolChain::getCompilerRTArgString(const llvm::opt::ArgList &Args, StringRef Component, bool Shared) const { return Args.MakeArgString(getCompilerRT(Args, Component, Shared)); } bool ToolChain::needsProfileRT(const ArgList &Args) { if (Args.hasFlag(options::OPT_fprofile_arcs, options::OPT_fno_profile_arcs, false) || Args.hasArg(options::OPT_fprofile_generate) || Args.hasArg(options::OPT_fprofile_generate_EQ) || Args.hasArg(options::OPT_fprofile_instr_generate) || Args.hasArg(options::OPT_fprofile_instr_generate_EQ) || Args.hasArg(options::OPT_fcreate_profile) || Args.hasArg(options::OPT_coverage)) return true; return false; } Tool *ToolChain::SelectTool(const JobAction &JA) const { if (getDriver().ShouldUseClangCompiler(JA)) return getClang(); Action::ActionClass AC = JA.getKind(); if (AC == Action::AssembleJobClass && useIntegratedAs()) return getClangAs(); return getTool(AC); } std::string ToolChain::GetFilePath(const char *Name) const { return D.GetFilePath(Name, *this); } std::string ToolChain::GetProgramPath(const char *Name) const { return D.GetProgramPath(Name, *this); } std::string ToolChain::GetLinkerPath() const { if (Arg *A = Args.getLastArg(options::OPT_fuse_ld_EQ)) { StringRef Suffix = A->getValue(); // If we're passed -fuse-ld= with no argument, or with the argument ld, // then use whatever the default system linker is. if (Suffix.empty() || Suffix == "ld") return GetProgramPath("ld"); llvm::SmallString<8> LinkerName("ld."); LinkerName.append(Suffix); std::string LinkerPath(GetProgramPath(LinkerName.c_str())); if (llvm::sys::fs::exists(LinkerPath)) return LinkerPath; getDriver().Diag(diag::err_drv_invalid_linker_name) << A->getAsString(Args); return ""; } return GetProgramPath(DefaultLinker); } types::ID ToolChain::LookupTypeForExtension(const char *Ext) const { return types::lookupTypeForExtension(Ext); } bool ToolChain::HasNativeLLVMSupport() const { return false; } bool ToolChain::isCrossCompiling() const { llvm::Triple HostTriple(LLVM_HOST_TRIPLE); switch (HostTriple.getArch()) { // The A32/T32/T16 instruction sets are not separate architectures in this // context. case llvm::Triple::arm: case llvm::Triple::armeb: case llvm::Triple::thumb: case llvm::Triple::thumbeb: return getArch() != llvm::Triple::arm && getArch() != llvm::Triple::thumb && getArch() != llvm::Triple::armeb && getArch() != llvm::Triple::thumbeb; default: return HostTriple.getArch() != getArch(); } } ObjCRuntime ToolChain::getDefaultObjCRuntime(bool isNonFragile) const { return ObjCRuntime(isNonFragile ? ObjCRuntime::GNUstep : ObjCRuntime::GCC, VersionTuple()); } bool ToolChain::isThreadModelSupported(const StringRef Model) const { if (Model == "single") { // FIXME: 'single' is only supported on ARM and WebAssembly so far. return Triple.getArch() == llvm::Triple::arm || Triple.getArch() == llvm::Triple::armeb || Triple.getArch() == llvm::Triple::thumb || Triple.getArch() == llvm::Triple::thumbeb || Triple.getArch() == llvm::Triple::wasm32 || Triple.getArch() == llvm::Triple::wasm64; } else if (Model == "posix") return true; return false; } std::string ToolChain::ComputeLLVMTriple(const ArgList &Args, types::ID InputType) const { switch (getTriple().getArch()) { default: return getTripleString(); case llvm::Triple::x86_64: { llvm::Triple Triple = getTriple(); if (!Triple.isOSBinFormatMachO()) return getTripleString(); if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) { // x86_64h goes in the triple. Other -march options just use the // vanilla triple we already have. StringRef MArch = A->getValue(); if (MArch == "x86_64h") Triple.setArchName(MArch); } return Triple.getTriple(); } case llvm::Triple::aarch64: { llvm::Triple Triple = getTriple(); if (!Triple.isOSBinFormatMachO()) return getTripleString(); // FIXME: older versions of ld64 expect the "arm64" component in the actual // triple string and query it to determine whether an LTO file can be // handled. Remove this when we don't care any more. Triple.setArchName("arm64"); return Triple.getTriple(); } case llvm::Triple::arm: case llvm::Triple::armeb: case llvm::Triple::thumb: case llvm::Triple::thumbeb: { // FIXME: Factor into subclasses. llvm::Triple Triple = getTriple(); bool IsBigEndian = getTriple().getArch() == llvm::Triple::armeb || getTriple().getArch() == llvm::Triple::thumbeb; // Handle pseudo-target flags '-mlittle-endian'/'-EL' and // '-mbig-endian'/'-EB'. if (Arg *A = Args.getLastArg(options::OPT_mlittle_endian, options::OPT_mbig_endian)) { IsBigEndian = !A->getOption().matches(options::OPT_mlittle_endian); } // Thumb2 is the default for V7 on Darwin. // // FIXME: Thumb should just be another -target-feaure, not in the triple. StringRef MCPU, MArch; if (const Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) MCPU = A->getValue(); if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) MArch = A->getValue(); std::string CPU = Triple.isOSBinFormatMachO() ? tools::arm::getARMCPUForMArch(MArch, Triple).str() : tools::arm::getARMTargetCPU(MCPU, MArch, Triple); StringRef Suffix = tools::arm::getLLVMArchSuffixForARM(CPU, MArch, Triple); bool IsMProfile = ARM::parseArchProfile(Suffix) == ARM::PK_M; bool ThumbDefault = IsMProfile || (ARM::parseArchVersion(Suffix) == 7 && getTriple().isOSBinFormatMachO()); // FIXME: this is invalid for WindowsCE if (getTriple().isOSWindows()) ThumbDefault = true; std::string ArchName; if (IsBigEndian) ArchName = "armeb"; else ArchName = "arm"; // Assembly files should start in ARM mode, unless arch is M-profile. if ((InputType != types::TY_PP_Asm && Args.hasFlag(options::OPT_mthumb, options::OPT_mno_thumb, ThumbDefault)) || IsMProfile) { if (IsBigEndian) ArchName = "thumbeb"; else ArchName = "thumb"; } Triple.setArchName(ArchName + Suffix.str()); return Triple.getTriple(); } } } std::string ToolChain::ComputeEffectiveClangTriple(const ArgList &Args, types::ID InputType) const { return ComputeLLVMTriple(Args, InputType); } void ToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, ArgStringList &CC1Args) const { // Each toolchain should provide the appropriate include flags. } void ToolChain::addClangTargetOptions(const ArgList &DriverArgs, ArgStringList &CC1Args) const { } void ToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {} void ToolChain::addProfileRTLibs(const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs) const { if (!needsProfileRT(Args)) return; CmdArgs.push_back(getCompilerRTArgString(Args, "profile")); return; } ToolChain::RuntimeLibType ToolChain::GetRuntimeLibType( const ArgList &Args) const { if (Arg *A = Args.getLastArg(options::OPT_rtlib_EQ)) { StringRef Value = A->getValue(); if (Value == "compiler-rt") return ToolChain::RLT_CompilerRT; if (Value == "libgcc") return ToolChain::RLT_Libgcc; getDriver().Diag(diag::err_drv_invalid_rtlib_name) << A->getAsString(Args); } return GetDefaultRuntimeLibType(); } ToolChain::CXXStdlibType ToolChain::GetCXXStdlibType(const ArgList &Args) const{ if (Arg *A = Args.getLastArg(options::OPT_stdlib_EQ)) { StringRef Value = A->getValue(); if (Value == "libc++") return ToolChain::CST_Libcxx; if (Value == "libstdc++") return ToolChain::CST_Libstdcxx; getDriver().Diag(diag::err_drv_invalid_stdlib_name) << A->getAsString(Args); } return ToolChain::CST_Libstdcxx; } /// \brief Utility function to add a system include directory to CC1 arguments. /*static*/ void ToolChain::addSystemInclude(const ArgList &DriverArgs, ArgStringList &CC1Args, const Twine &Path) { CC1Args.push_back("-internal-isystem"); CC1Args.push_back(DriverArgs.MakeArgString(Path)); } /// \brief Utility function to add a system include directory with extern "C" /// semantics to CC1 arguments. /// /// Note that this should be used rarely, and only for directories that /// historically and for legacy reasons are treated as having implicit extern /// "C" semantics. These semantics are *ignored* by and large today, but its /// important to preserve the preprocessor changes resulting from the /// classification. /*static*/ void ToolChain::addExternCSystemInclude(const ArgList &DriverArgs, ArgStringList &CC1Args, const Twine &Path) { CC1Args.push_back("-internal-externc-isystem"); CC1Args.push_back(DriverArgs.MakeArgString(Path)); } void ToolChain::addExternCSystemIncludeIfExists(const ArgList &DriverArgs, ArgStringList &CC1Args, const Twine &Path) { if (llvm::sys::fs::exists(Path)) addExternCSystemInclude(DriverArgs, CC1Args, Path); } /// \brief Utility function to add a list of system include directories to CC1. /*static*/ void ToolChain::addSystemIncludes(const ArgList &DriverArgs, ArgStringList &CC1Args, ArrayRef Paths) { for (StringRef Path : Paths) { CC1Args.push_back("-internal-isystem"); CC1Args.push_back(DriverArgs.MakeArgString(Path)); } } void ToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs, ArgStringList &CC1Args) const { // Header search paths should be handled by each of the subclasses. // Historically, they have not been, and instead have been handled inside of // the CC1-layer frontend. As the logic is hoisted out, this generic function // will slowly stop being called. // // While it is being called, replicate a bit of a hack to propagate the // '-stdlib=' flag down to CC1 so that it can in turn customize the C++ // header search paths with it. Once all systems are overriding this // function, the CC1 flag and this line can be removed. DriverArgs.AddAllArgs(CC1Args, options::OPT_stdlib_EQ); } void ToolChain::AddCXXStdlibLibArgs(const ArgList &Args, ArgStringList &CmdArgs) const { CXXStdlibType Type = GetCXXStdlibType(Args); switch (Type) { case ToolChain::CST_Libcxx: CmdArgs.push_back("-lc++"); break; case ToolChain::CST_Libstdcxx: CmdArgs.push_back("-lstdc++"); break; } } void ToolChain::AddFilePathLibArgs(const ArgList &Args, ArgStringList &CmdArgs) const { for (const auto &LibPath : getFilePaths()) if(LibPath.length() > 0) CmdArgs.push_back(Args.MakeArgString(StringRef("-L") + LibPath)); } void ToolChain::AddCCKextLibArgs(const ArgList &Args, ArgStringList &CmdArgs) const { CmdArgs.push_back("-lcc_kext"); } bool ToolChain::AddFastMathRuntimeIfAvailable(const ArgList &Args, ArgStringList &CmdArgs) const { // Do not check for -fno-fast-math or -fno-unsafe-math when -Ofast passed // (to keep the linker options consistent with gcc and clang itself). if (!isOptimizationLevelFast(Args)) { // Check if -ffast-math or -funsafe-math. Arg *A = Args.getLastArg(options::OPT_ffast_math, options::OPT_fno_fast_math, options::OPT_funsafe_math_optimizations, options::OPT_fno_unsafe_math_optimizations); if (!A || A->getOption().getID() == options::OPT_fno_fast_math || A->getOption().getID() == options::OPT_fno_unsafe_math_optimizations) return false; } // If crtfastmath.o exists add it to the arguments. std::string Path = GetFilePath("crtfastmath.o"); if (Path == "crtfastmath.o") // Not found. return false; CmdArgs.push_back(Args.MakeArgString(Path)); return true; } SanitizerMask ToolChain::getSupportedSanitizers() const { // Return sanitizers which don't require runtime support and are not // platform dependent. using namespace SanitizerKind; SanitizerMask Res = (Undefined & ~Vptr & ~Function) | (CFI & ~CFIICall) | CFICastStrict | UnsignedIntegerOverflow | LocalBounds; if (getTriple().getArch() == llvm::Triple::x86 || getTriple().getArch() == llvm::Triple::x86_64) Res |= CFIICall; return Res; } void ToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs, ArgStringList &CC1Args) const {} Index: projects/clang380-import/contrib/llvm/tools/clang =================================================================== --- projects/clang380-import/contrib/llvm/tools/clang (revision 292950) +++ projects/clang380-import/contrib/llvm/tools/clang (revision 292951) Property changes on: projects/clang380-import/contrib/llvm/tools/clang ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm/tools/clang:r292913-292950 Index: projects/clang380-import/contrib/llvm =================================================================== --- projects/clang380-import/contrib/llvm (revision 292950) +++ projects/clang380-import/contrib/llvm (revision 292951) Property changes on: projects/clang380-import/contrib/llvm ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm:r292913-292950 Index: projects/clang380-import/sbin/reboot/reboot.c =================================================================== --- projects/clang380-import/sbin/reboot/reboot.c (revision 292950) +++ projects/clang380-import/sbin/reboot/reboot.c (revision 292951) @@ -1,269 +1,269 @@ /* * Copyright (c) 1980, 1986, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if 0 #ifndef lint static const char copyright[] = "@(#) Copyright (c) 1980, 1986, 1993\n\ The Regents of the University of California. All rights reserved.\n"; #endif /* not lint */ #ifndef lint static char sccsid[] = "@(#)reboot.c 8.1 (Berkeley) 6/5/93"; #endif /* not lint */ #endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void usage(void); static u_int get_pageins(void); static int dohalt; int main(int argc, char *argv[]) { struct utmpx utx; const struct passwd *pw; int ch, howto, i, fd, lflag, nflag, qflag, sverrno, Nflag; u_int pageins; const char *user, *kernel = NULL; if (strcmp(getprogname(), "halt") == 0) { dohalt = 1; howto = RB_HALT; } else howto = 0; - lflag = nflag = qflag = 0; + lflag = nflag = qflag = Nflag = 0; while ((ch = getopt(argc, argv, "dk:lNnpqr")) != -1) switch(ch) { case 'd': howto |= RB_DUMP; break; case 'k': kernel = optarg; break; case 'l': lflag = 1; break; case 'n': nflag = 1; howto |= RB_NOSYNC; break; case 'N': nflag = 1; Nflag = 1; break; case 'p': howto |= RB_POWEROFF; break; case 'q': qflag = 1; break; case 'r': howto |= RB_REROOT; break; case '?': default: usage(); } argc -= optind; argv += optind; if ((howto & (RB_DUMP | RB_HALT)) == (RB_DUMP | RB_HALT)) errx(1, "cannot dump (-d) when halting; must reboot instead"); if (Nflag && (howto & RB_NOSYNC) != 0) errx(1, "-N cannot be used with -n"); if ((howto & RB_REROOT) != 0 && howto != RB_REROOT) errx(1, "-r cannot be used with -d, -n, or -p"); if (geteuid()) { errno = EPERM; err(1, NULL); } if (qflag) { reboot(howto); err(1, NULL); } if (kernel != NULL) { fd = open("/boot/nextboot.conf", O_WRONLY | O_CREAT | O_TRUNC, 0444); if (fd > -1) { (void)write(fd, "nextboot_enable=\"YES\"\n", 22); (void)write(fd, "kernel=\"", 8L); (void)write(fd, kernel, strlen(kernel)); (void)write(fd, "\"\n", 2); close(fd); } } /* Log the reboot. */ if (!lflag) { if ((user = getlogin()) == NULL) user = (pw = getpwuid(getuid())) ? pw->pw_name : "???"; if (dohalt) { openlog("halt", 0, LOG_AUTH | LOG_CONS); syslog(LOG_CRIT, "halted by %s", user); } else if (howto & RB_REROOT) { openlog("reroot", 0, LOG_AUTH | LOG_CONS); syslog(LOG_CRIT, "rerooted by %s", user); } else { openlog("reboot", 0, LOG_AUTH | LOG_CONS); syslog(LOG_CRIT, "rebooted by %s", user); } } utx.ut_type = SHUTDOWN_TIME; gettimeofday(&utx.ut_tv, NULL); pututxline(&utx); /* * Do a sync early on, so disks start transfers while we're off * killing processes. Don't worry about writes done before the * processes die, the reboot system call syncs the disks. */ if (!nflag) sync(); /* * Ignore signals that we can get as a result of killing * parents, group leaders, etc. */ (void)signal(SIGHUP, SIG_IGN); (void)signal(SIGINT, SIG_IGN); (void)signal(SIGQUIT, SIG_IGN); (void)signal(SIGTERM, SIG_IGN); (void)signal(SIGTSTP, SIG_IGN); /* * If we're running in a pipeline, we don't want to die * after killing whatever we're writing to. */ (void)signal(SIGPIPE, SIG_IGN); /* * Only init(8) can perform rerooting. */ if (howto & RB_REROOT) { if (kill(1, SIGEMT) == -1) err(1, "SIGEMT init"); return (0); } /* Just stop init -- if we fail, we'll restart it. */ if (kill(1, SIGTSTP) == -1) err(1, "SIGTSTP init"); /* Send a SIGTERM first, a chance to save the buffers. */ if (kill(-1, SIGTERM) == -1 && errno != ESRCH) err(1, "SIGTERM processes"); /* * After the processes receive the signal, start the rest of the * buffers on their way. Wait 5 seconds between the SIGTERM and * the SIGKILL to give everybody a chance. If there is a lot of * paging activity then wait longer, up to a maximum of approx * 60 seconds. */ sleep(2); for (i = 0; i < 20; i++) { pageins = get_pageins(); if (!nflag) sync(); sleep(3); if (get_pageins() == pageins) break; } for (i = 1;; ++i) { if (kill(-1, SIGKILL) == -1) { if (errno == ESRCH) break; goto restart; } if (i > 5) { (void)fprintf(stderr, "WARNING: some process(es) wouldn't die\n"); break; } (void)sleep(2 * i); } reboot(howto); /* FALLTHROUGH */ restart: sverrno = errno; errx(1, "%s%s", kill(1, SIGHUP) == -1 ? "(can't restart init): " : "", strerror(sverrno)); /* NOTREACHED */ } static void usage(void) { (void)fprintf(stderr, dohalt ? "usage: halt [-lnpq] [-k kernel]\n" : "usage: reboot [-dlnpq] [-k kernel]\n"); exit(1); } static u_int get_pageins(void) { u_int pageins; size_t len; len = sizeof(pageins); if (sysctlbyname("vm.stats.vm.v_swappgsin", &pageins, &len, NULL, 0) != 0) { warnx("v_swappgsin"); return (0); } return pageins; } Index: projects/clang380-import/sbin =================================================================== --- projects/clang380-import/sbin (revision 292950) +++ projects/clang380-import/sbin (revision 292951) Property changes on: projects/clang380-import/sbin ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sbin:r292913-292950 Index: projects/clang380-import/sys/dev/mlx5/mlx5_en/en.h =================================================================== --- projects/clang380-import/sys/dev/mlx5/mlx5_en/en.h (revision 292950) +++ projects/clang380-import/sys/dev/mlx5/mlx5_en/en.h (revision 292951) @@ -1,791 +1,792 @@ /*- * Copyright (c) 2015 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MLX5_EN_H_ #define _MLX5_EN_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_rss.h" #ifdef RSS #include #include #endif #include #ifdef HAVE_TURBO_LRO #include "tcp_tlro.h" #endif #include #include #include #include #include #include #include #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x7 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE 0x7 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd /* freeBSD HW LRO is limited by 16KB - the size of max mbuf */ #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ MJUM16BYTES #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80 #define MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ 0x7 #define MLX5E_CACHELINE_SIZE CACHE_LINE_SIZE #define MLX5E_HW2SW_MTU(hwmtu) \ ((hwmtu) - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN)) #define MLX5E_SW2HW_MTU(swmtu) \ ((swmtu) + (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN)) #define MLX5E_SW2MB_MTU(swmtu) \ (MLX5E_SW2HW_MTU(swmtu) + MLX5E_NET_IP_ALIGN) #define MLX5E_MTU_MIN 72 /* Min MTU allowed by the kernel */ #define MLX5E_MTU_MAX MIN(ETHERMTU_JUMBO, MJUM16BYTES) /* Max MTU of Ethernet * jumbo frames */ #define MLX5E_BUDGET_MAX 8192 /* RX and TX */ #define MLX5E_RX_BUDGET_MAX 256 #define MLX5E_SQ_BF_BUDGET 16 #define MLX5E_SQ_TX_QUEUE_SIZE 4096 /* SQ drbr queue size */ #define MLX5E_MAX_TX_NUM_TC 8 /* units */ #define MLX5E_MAX_TX_HEADER 128 /* bytes */ #define MLX5E_MAX_TX_PAYLOAD_SIZE 65536 /* bytes */ #define MLX5E_MAX_TX_MBUF_SIZE 65536 /* bytes */ #define MLX5E_MAX_TX_MBUF_FRAGS \ ((MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS) - \ (MLX5E_MAX_TX_HEADER / MLX5_SEND_WQE_DS)) /* units */ #define MLX5E_MAX_TX_INLINE \ (MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \ sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start)) /* bytes */ MALLOC_DECLARE(M_MLX5EN); struct mlx5_core_dev; struct mlx5e_cq; typedef void (mlx5e_cq_comp_t)(struct mlx5_core_cq *); #define MLX5E_STATS_COUNT(a,b,c,d) a #define MLX5E_STATS_VAR(a,b,c,d) b; #define MLX5E_STATS_DESC(a,b,c,d) c, d, #define MLX5E_VPORT_STATS(m) \ /* HW counters */ \ m(+1, u64 rx_packets, "rx_packets", "Received packets") \ m(+1, u64 rx_bytes, "rx_bytes", "Received bytes") \ m(+1, u64 tx_packets, "tx_packets", "Transmitted packets") \ m(+1, u64 tx_bytes, "tx_bytes", "Transmitted bytes") \ m(+1, u64 rx_error_packets, "rx_error_packets", "Received error packets") \ m(+1, u64 rx_error_bytes, "rx_error_bytes", "Received error bytes") \ m(+1, u64 tx_error_packets, "tx_error_packets", "Transmitted error packets") \ m(+1, u64 tx_error_bytes, "tx_error_bytes", "Transmitted error bytes") \ m(+1, u64 rx_unicast_packets, "rx_unicast_packets", "Received unicast packets") \ m(+1, u64 rx_unicast_bytes, "rx_unicast_bytes", "Received unicast bytes") \ m(+1, u64 tx_unicast_packets, "tx_unicast_packets", "Transmitted unicast packets") \ m(+1, u64 tx_unicast_bytes, "tx_unicast_bytes", "Transmitted unicast bytes") \ m(+1, u64 rx_multicast_packets, "rx_multicast_packets", "Received multicast packets") \ m(+1, u64 rx_multicast_bytes, "rx_multicast_bytes", "Received multicast bytes") \ m(+1, u64 tx_multicast_packets, "tx_multicast_packets", "Transmitted multicast packets") \ m(+1, u64 tx_multicast_bytes, "tx_multicast_bytes", "Transmitted multicast bytes") \ m(+1, u64 rx_broadcast_packets, "rx_broadcast_packets", "Received broadcast packets") \ m(+1, u64 rx_broadcast_bytes, "rx_broadcast_bytes", "Received broadcast bytes") \ m(+1, u64 tx_broadcast_packets, "tx_broadcast_packets", "Transmitted broadcast packets") \ m(+1, u64 tx_broadcast_bytes, "tx_broadcast_bytes", "Transmitted broadcast bytes") \ m(+1, u64 rx_out_of_buffer, "rx_out_of_buffer", "Receive out of buffer, no recv wqes events") \ /* SW counters */ \ m(+1, u64 tso_packets, "tso_packets", "Transmitted TSO packets") \ m(+1, u64 tso_bytes, "tso_bytes", "Transmitted TSO bytes") \ m(+1, u64 lro_packets, "lro_packets", "Received LRO packets") \ m(+1, u64 lro_bytes, "lro_bytes", "Received LRO bytes") \ m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO") \ m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO") \ m(+1, u64 rx_csum_good, "rx_csum_good", "Received checksum valid packets") \ m(+1, u64 rx_csum_none, "rx_csum_none", "Received no checksum packets") \ m(+1, u64 tx_csum_offload, "tx_csum_offload", "Transmit checksum offload packets") \ m(+1, u64 tx_queue_dropped, "tx_queue_dropped", "Transmit queue dropped") \ m(+1, u64 tx_defragged, "tx_defragged", "Transmit queue defragged") \ m(+1, u64 rx_wqe_err, "rx_wqe_err", "Receive WQE errors") #define MLX5E_VPORT_STATS_NUM (0 MLX5E_VPORT_STATS(MLX5E_STATS_COUNT)) struct mlx5e_vport_stats { struct sysctl_ctx_list ctx; u64 arg [0]; MLX5E_VPORT_STATS(MLX5E_STATS_VAR) u32 rx_out_of_buffer_prev; }; #define MLX5E_PPORT_IEEE802_3_STATS(m) \ m(+1, u64 frames_tx, "frames_tx", "Frames transmitted") \ m(+1, u64 frames_rx, "frames_rx", "Frames received") \ m(+1, u64 check_seq_err, "check_seq_err", "Sequence errors") \ m(+1, u64 alignment_err, "alignment_err", "Alignment errors") \ m(+1, u64 octets_tx, "octets_tx", "Bytes transmitted") \ m(+1, u64 octets_received, "octets_received", "Bytes received") \ m(+1, u64 multicast_xmitted, "multicast_xmitted", "Multicast transmitted") \ m(+1, u64 broadcast_xmitted, "broadcast_xmitted", "Broadcast transmitted") \ m(+1, u64 multicast_rx, "multicast_rx", "Multicast received") \ m(+1, u64 broadcast_rx, "broadcast_rx", "Broadcast received") \ m(+1, u64 in_range_len_errors, "in_range_len_errors", "In range length errors") \ m(+1, u64 out_of_range_len, "out_of_range_len", "Out of range length errors") \ m(+1, u64 too_long_errors, "too_long_errors", "Too long errors") \ m(+1, u64 symbol_err, "symbol_err", "Symbol errors") \ m(+1, u64 mac_control_tx, "mac_control_tx", "MAC control transmitted") \ m(+1, u64 mac_control_rx, "mac_control_rx", "MAC control received") \ m(+1, u64 unsupported_op_rx, "unsupported_op_rx", "Unsupported operation received") \ m(+1, u64 pause_ctrl_rx, "pause_ctrl_rx", "Pause control received") \ m(+1, u64 pause_ctrl_tx, "pause_ctrl_tx", "Pause control transmitted") #define MLX5E_PPORT_RFC2819_STATS(m) \ m(+1, u64 drop_events, "drop_events", "Dropped events") \ m(+1, u64 octets, "octets", "Octets") \ m(+1, u64 pkts, "pkts", "Packets") \ m(+1, u64 broadcast_pkts, "broadcast_pkts", "Broadcast packets") \ m(+1, u64 multicast_pkts, "multicast_pkts", "Multicast packets") \ m(+1, u64 crc_align_errors, "crc_align_errors", "CRC alignment errors") \ m(+1, u64 undersize_pkts, "undersize_pkts", "Undersized packets") \ m(+1, u64 oversize_pkts, "oversize_pkts", "Oversized packets") \ m(+1, u64 fragments, "fragments", "Fragments") \ m(+1, u64 jabbers, "jabbers", "Jabbers") \ m(+1, u64 collisions, "collisions", "Collisions") #define MLX5E_PPORT_RFC2819_STATS_DEBUG(m) \ m(+1, u64 p64octets, "p64octets", "Bytes") \ m(+1, u64 p65to127octets, "p65to127octets", "Bytes") \ m(+1, u64 p128to255octets, "p128to255octets", "Bytes") \ m(+1, u64 p256to511octets, "p256to511octets", "Bytes") \ m(+1, u64 p512to1023octets, "p512to1023octets", "Bytes") \ m(+1, u64 p1024to1518octets, "p1024to1518octets", "Bytes") \ m(+1, u64 p1519to2047octets, "p1519to2047octets", "Bytes") \ m(+1, u64 p2048to4095octets, "p2048to4095octets", "Bytes") \ m(+1, u64 p4096to8191octets, "p4096to8191octets", "Bytes") \ m(+1, u64 p8192to10239octets, "p8192to10239octets", "Bytes") #define MLX5E_PPORT_RFC2863_STATS_DEBUG(m) \ m(+1, u64 in_octets, "in_octets", "In octets") \ m(+1, u64 in_ucast_pkts, "in_ucast_pkts", "In unicast packets") \ m(+1, u64 in_discards, "in_discards", "In discards") \ m(+1, u64 in_errors, "in_errors", "In errors") \ m(+1, u64 in_unknown_protos, "in_unknown_protos", "In unknown protocols") \ m(+1, u64 out_octets, "out_octets", "Out octets") \ m(+1, u64 out_ucast_pkts, "out_ucast_pkts", "Out unicast packets") \ m(+1, u64 out_discards, "out_discards", "Out discards") \ m(+1, u64 out_errors, "out_errors", "Out errors") \ m(+1, u64 in_multicast_pkts, "in_multicast_pkts", "In multicast packets") \ m(+1, u64 in_broadcast_pkts, "in_broadcast_pkts", "In broadcast packets") \ m(+1, u64 out_multicast_pkts, "out_multicast_pkts", "Out multicast packets") \ m(+1, u64 out_broadcast_pkts, "out_broadcast_pkts", "Out broadcast packets") #define MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m) \ m(+1, u64 time_since_last_clear, "time_since_last_clear", \ "Time since the last counters clear event (msec)") \ m(+1, u64 symbol_errors, "symbol_errors", "Symbol errors") \ m(+1, u64 sync_headers_errors, "sync_headers_errors", "Sync header error counter") \ m(+1, u64 bip_errors_lane0, "edpl_bip_errors_lane0", \ "Indicates the number of PRBS errors on lane 0") \ m(+1, u64 bip_errors_lane1, "edpl_bip_errors_lane1", \ "Indicates the number of PRBS errors on lane 1") \ m(+1, u64 bip_errors_lane2, "edpl_bip_errors_lane2", \ "Indicates the number of PRBS errors on lane 2") \ m(+1, u64 bip_errors_lane3, "edpl_bip_errors_lane3", \ "Indicates the number of PRBS errors on lane 3") \ m(+1, u64 fc_corrected_blocks_lane0, "fc_corrected_blocks_lane0", \ "FEC correctable block counter lane 0") \ m(+1, u64 fc_corrected_blocks_lane1, "fc_corrected_blocks_lane1", \ "FEC correctable block counter lane 1") \ m(+1, u64 fc_corrected_blocks_lane2, "fc_corrected_blocks_lane2", \ "FEC correctable block counter lane 2") \ m(+1, u64 fc_corrected_blocks_lane3, "fc_corrected_blocks_lane3", \ "FEC correctable block counter lane 3") \ m(+1, u64 rs_corrected_blocks, "rs_corrected_blocks", \ "FEC correcable block counter") \ m(+1, u64 rs_uncorrectable_blocks, "rs_uncorrectable_blocks", \ "FEC uncorrecable block counter") \ m(+1, u64 rs_no_errors_blocks, "rs_no_errors_blocks", \ "The number of RS-FEC blocks received that had no errors") \ m(+1, u64 rs_single_error_blocks, "rs_single_error_blocks", \ "The number of corrected RS-FEC blocks received that had" \ "exactly 1 error symbol") \ m(+1, u64 rs_corrected_symbols_total, "rs_corrected_symbols_total", \ "Port FEC corrected symbol counter") \ m(+1, u64 rs_corrected_symbols_lane0, "rs_corrected_symbols_lane0", \ "FEC corrected symbol counter lane 0") \ m(+1, u64 rs_corrected_symbols_lane1, "rs_corrected_symbols_lane1", \ "FEC corrected symbol counter lane 1") \ m(+1, u64 rs_corrected_symbols_lane2, "rs_corrected_symbols_lane2", \ "FEC corrected symbol counter lane 2") \ m(+1, u64 rs_corrected_symbols_lane3, "rs_corrected_symbols_lane3", \ "FEC corrected symbol counter lane 3") \ /* * Make sure to update mlx5e_update_pport_counters() * when adding a new MLX5E_PPORT_STATS block */ #define MLX5E_PPORT_STATS(m) \ MLX5E_PPORT_IEEE802_3_STATS(m) \ MLX5E_PPORT_RFC2819_STATS(m) #define MLX5E_PORT_STATS_DEBUG(m) \ MLX5E_PPORT_RFC2819_STATS_DEBUG(m) \ MLX5E_PPORT_RFC2863_STATS_DEBUG(m) \ MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m) #define MLX5E_PPORT_IEEE802_3_STATS_NUM \ (0 MLX5E_PPORT_IEEE802_3_STATS(MLX5E_STATS_COUNT)) #define MLX5E_PPORT_RFC2819_STATS_NUM \ (0 MLX5E_PPORT_RFC2819_STATS(MLX5E_STATS_COUNT)) #define MLX5E_PPORT_STATS_NUM \ (0 MLX5E_PPORT_STATS(MLX5E_STATS_COUNT)) #define MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM \ (0 MLX5E_PPORT_RFC2819_STATS_DEBUG(MLX5E_STATS_COUNT)) #define MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM \ (0 MLX5E_PPORT_RFC2863_STATS_DEBUG(MLX5E_STATS_COUNT)) #define MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM \ (0 MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(MLX5E_STATS_COUNT)) #define MLX5E_PORT_STATS_DEBUG_NUM \ (0 MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_COUNT)) struct mlx5e_pport_stats { struct sysctl_ctx_list ctx; u64 arg [0]; MLX5E_PPORT_STATS(MLX5E_STATS_VAR) }; struct mlx5e_port_stats_debug { struct sysctl_ctx_list ctx; u64 arg [0]; MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_VAR) }; #define MLX5E_RQ_STATS(m) \ m(+1, u64 packets, "packets", "Received packets") \ m(+1, u64 csum_none, "csum_none", "Received packets") \ m(+1, u64 lro_packets, "lro_packets", "Received packets") \ m(+1, u64 lro_bytes, "lro_bytes", "Received packets") \ m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO") \ m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO") \ m(+1, u64 wqe_err, "wqe_err", "Received packets") #define MLX5E_RQ_STATS_NUM (0 MLX5E_RQ_STATS(MLX5E_STATS_COUNT)) struct mlx5e_rq_stats { struct sysctl_ctx_list ctx; u64 arg [0]; MLX5E_RQ_STATS(MLX5E_STATS_VAR) }; #define MLX5E_SQ_STATS(m) \ m(+1, u64 packets, "packets", "Transmitted packets") \ m(+1, u64 tso_packets, "tso_packets", "Transmitted packets") \ m(+1, u64 tso_bytes, "tso_bytes", "Transmitted bytes") \ m(+1, u64 csum_offload_none, "csum_offload_none", "Transmitted packets") \ m(+1, u64 defragged, "defragged", "Transmitted packets") \ m(+1, u64 dropped, "dropped", "Transmitted packets") \ m(+1, u64 nop, "nop", "Transmitted packets") #define MLX5E_SQ_STATS_NUM (0 MLX5E_SQ_STATS(MLX5E_STATS_COUNT)) struct mlx5e_sq_stats { struct sysctl_ctx_list ctx; u64 arg [0]; MLX5E_SQ_STATS(MLX5E_STATS_VAR) }; struct mlx5e_stats { struct mlx5e_vport_stats vport; struct mlx5e_pport_stats pport; struct mlx5e_port_stats_debug port_stats_debug; }; struct mlx5e_params { u8 log_sq_size; u8 log_rq_size; u16 num_channels; u8 default_vlan_prio; u8 num_tc; u8 rx_cq_moderation_mode; u8 tx_cq_moderation_mode; u16 rx_cq_moderation_usec; u16 rx_cq_moderation_pkts; u16 tx_cq_moderation_usec; u16 tx_cq_moderation_pkts; u16 min_rx_wqes; bool hw_lro_en; bool cqe_zipping_en; u32 lro_wqe_sz; u16 rx_hash_log_tbl_sz; }; #define MLX5E_PARAMS(m) \ m(+1, u64 tx_pauseframe_control, "tx_pauseframe_control", "Set to enable TX pause frames. Clear to disable.") \ m(+1, u64 rx_pauseframe_control, "rx_pauseframe_control", "Set to enable RX pause frames. Clear to disable.") \ m(+1, u64 tx_queue_size_max, "tx_queue_size_max", "Max send queue size") \ m(+1, u64 rx_queue_size_max, "rx_queue_size_max", "Max receive queue size") \ m(+1, u64 tx_queue_size, "tx_queue_size", "Default send queue size") \ m(+1, u64 rx_queue_size, "rx_queue_size", "Default receive queue size") \ m(+1, u64 channels, "channels", "Default number of channels") \ m(+1, u64 coalesce_usecs_max, "coalesce_usecs_max", "Maximum usecs for joining packets") \ m(+1, u64 coalesce_pkts_max, "coalesce_pkts_max", "Maximum packets to join") \ m(+1, u64 rx_coalesce_usecs, "rx_coalesce_usecs", "Limit in usec for joining rx packets") \ m(+1, u64 rx_coalesce_pkts, "rx_coalesce_pkts", "Maximum number of rx packets to join") \ m(+1, u64 rx_coalesce_mode, "rx_coalesce_mode", "0: EQE mode 1: CQE mode") \ m(+1, u64 tx_coalesce_usecs, "tx_coalesce_usecs", "Limit in usec for joining tx packets") \ m(+1, u64 tx_coalesce_pkts, "tx_coalesce_pkts", "Maximum number of tx packets to join") \ m(+1, u64 tx_coalesce_mode, "tx_coalesce_mode", "0: EQE mode 1: CQE mode") \ m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro") \ m(+1, u64 cqe_zipping, "cqe_zipping", "0 : CQE zipping disabled") #define MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT)) struct mlx5e_params_ethtool { u64 arg [0]; MLX5E_PARAMS(MLX5E_STATS_VAR) }; /* EEPROM Standards for plug in modules */ #ifndef MLX5E_ETH_MODULE_SFF_8472 #define MLX5E_ETH_MODULE_SFF_8472 0x1 #define MLX5E_ETH_MODULE_SFF_8472_LEN 128 #endif #ifndef MLX5E_ETH_MODULE_SFF_8636 #define MLX5E_ETH_MODULE_SFF_8636 0x2 #define MLX5E_ETH_MODULE_SFF_8636_LEN 256 #endif #ifndef MLX5E_ETH_MODULE_SFF_8436 #define MLX5E_ETH_MODULE_SFF_8436 0x3 #define MLX5E_ETH_MODULE_SFF_8436_LEN 256 #endif /* EEPROM I2C Addresses */ #define MLX5E_I2C_ADDR_LOW 0x50 #define MLX5E_I2C_ADDR_HIGH 0x51 #define MLX5E_EEPROM_LOW_PAGE 0x0 #define MLX5E_EEPROM_HIGH_PAGE 0x3 #define MLX5E_EEPROM_HIGH_PAGE_OFFSET 128 #define MLX5E_EEPROM_PAGE_LENGTH 256 #define MLX5E_EEPROM_INFO_BYTES 0x3 struct mlx5e_cq { /* data path - accessed per cqe */ struct mlx5_cqwq wq; /* data path - accessed per HW polling */ struct mlx5_core_cq mcq; struct mlx5e_channel *channel; /* control */ struct mlx5_wq_ctrl wq_ctrl; } __aligned(MLX5E_CACHELINE_SIZE); struct mlx5e_rq_mbuf { bus_dmamap_t dma_map; caddr_t data; struct mbuf *mbuf; }; struct mlx5e_rq { /* data path */ struct mlx5_wq_ll wq; struct mtx mtx; bus_dma_tag_t dma_tag; u32 wqe_sz; struct mlx5e_rq_mbuf *mbuf; struct device *pdev; struct ifnet *ifp; struct mlx5e_rq_stats stats; struct mlx5e_cq cq; #ifdef HAVE_TURBO_LRO struct tlro_ctrl lro; #else struct lro_ctrl lro; #endif volatile int enabled; int ix; /* control */ struct mlx5_wq_ctrl wq_ctrl; u32 rqn; struct mlx5e_channel *channel; } __aligned(MLX5E_CACHELINE_SIZE); struct mlx5e_sq_mbuf { bus_dmamap_t dma_map; struct mbuf *mbuf; u32 num_bytes; u32 num_wqebbs; }; enum { MLX5E_SQ_READY, MLX5E_SQ_FULL }; struct mlx5e_sq { /* data path */ struct mtx lock; bus_dma_tag_t dma_tag; struct mtx comp_lock; /* dirtied @completion */ u16 cc; /* dirtied @xmit */ u16 pc __aligned(MLX5E_CACHELINE_SIZE); u16 bf_offset; struct mlx5e_sq_stats stats; struct mlx5e_cq cq; struct task sq_task; struct taskqueue *sq_tq; /* pointers to per packet info: write@xmit, read@completion */ struct mlx5e_sq_mbuf *mbuf; struct buf_ring *br; /* read only */ struct mlx5_wq_cyc wq; void __iomem *uar_map; void __iomem *uar_bf_map; u32 sqn; u32 bf_buf_size; struct device *pdev; u32 mkey_be; /* control path */ struct mlx5_wq_ctrl wq_ctrl; struct mlx5_uar uar; struct mlx5e_channel *channel; int tc; unsigned int queue_state; } __aligned(MLX5E_CACHELINE_SIZE); static inline bool mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n) { return ((sq->wq.sz_m1 & (sq->cc - sq->pc)) >= n || sq->cc == sq->pc); } struct mlx5e_channel { /* data path */ struct mlx5e_rq rq; struct mlx5e_sq sq[MLX5E_MAX_TX_NUM_TC]; struct device *pdev; struct ifnet *ifp; u32 mkey_be; u8 num_tc; /* control */ struct mlx5e_priv *priv; int ix; int cpu; } __aligned(MLX5E_CACHELINE_SIZE); enum mlx5e_traffic_types { MLX5E_TT_IPV4_TCP, MLX5E_TT_IPV6_TCP, MLX5E_TT_IPV4_UDP, MLX5E_TT_IPV6_UDP, MLX5E_TT_IPV4_IPSEC_AH, MLX5E_TT_IPV6_IPSEC_AH, MLX5E_TT_IPV4_IPSEC_ESP, MLX5E_TT_IPV6_IPSEC_ESP, MLX5E_TT_IPV4, MLX5E_TT_IPV6, MLX5E_TT_ANY, MLX5E_NUM_TT, }; enum { MLX5E_RQT_SPREADING = 0, MLX5E_RQT_DEFAULT_RQ = 1, MLX5E_NUM_RQT = 2, }; struct mlx5e_eth_addr_info { u8 addr [ETH_ALEN + 2]; u32 tt_vec; u32 ft_ix[MLX5E_NUM_TT]; /* flow table index per traffic type */ }; #define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE) struct mlx5e_eth_addr_hash_node; struct mlx5e_eth_addr_hash_head { struct mlx5e_eth_addr_hash_node *lh_first; }; struct mlx5e_eth_addr_db { struct mlx5e_eth_addr_hash_head if_uc[MLX5E_ETH_ADDR_HASH_SIZE]; struct mlx5e_eth_addr_hash_head if_mc[MLX5E_ETH_ADDR_HASH_SIZE]; struct mlx5e_eth_addr_info broadcast; struct mlx5e_eth_addr_info allmulti; struct mlx5e_eth_addr_info promisc; bool broadcast_enabled; bool allmulti_enabled; bool promisc_enabled; }; enum { MLX5E_STATE_ASYNC_EVENTS_ENABLE, MLX5E_STATE_OPENED, }; struct mlx5e_vlan_db { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; u32 active_vlans_ft_ix[VLAN_N_VID]; u32 untagged_rule_ft_ix; u32 any_vlan_rule_ft_ix; bool filter_disabled; }; struct mlx5e_flow_table { void *vlan; void *main; }; struct mlx5e_priv { /* priv data path fields - start */ int order_base_2_num_channels; int queue_mapping_channel_mask; int num_tc; int default_vlan_prio; /* priv data path fields - end */ unsigned long state; int gone; #define PRIV_LOCK(priv) sx_xlock(&(priv)->state_lock) #define PRIV_UNLOCK(priv) sx_xunlock(&(priv)->state_lock) #define PRIV_LOCKED(priv) sx_xlocked(&(priv)->state_lock) struct sx state_lock; /* Protects Interface state */ struct mlx5_uar cq_uar; u32 pdn; u32 tdn; struct mlx5_core_mr mr; struct mlx5e_channel *volatile *channel; u32 tisn[MLX5E_MAX_TX_NUM_TC]; u32 rqtn; u32 tirn[MLX5E_NUM_TT]; struct mlx5e_flow_table ft; struct mlx5e_eth_addr_db eth_addr; struct mlx5e_vlan_db vlan; struct mlx5e_params params; struct mlx5e_params_ethtool params_ethtool; struct mtx async_events_mtx; /* sync hw events */ struct work_struct update_stats_work; struct work_struct update_carrier_work; struct work_struct set_rx_mode_work; struct mlx5_core_dev *mdev; struct ifnet *ifp; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_ifnet; struct sysctl_oid *sysctl_hw; int sysctl_debug; struct mlx5e_stats stats; int counter_set_id; eventhandler_tag vlan_detach; eventhandler_tag vlan_attach; struct ifmedia media; int media_status_last; int media_active_last; struct callout watchdog; }; #define MLX5E_NET_IP_ALIGN 2 struct mlx5e_tx_wqe { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_wqe_eth_seg eth; }; struct mlx5e_rx_wqe { struct mlx5_wqe_srq_next_seg next; struct mlx5_wqe_data_seg data; }; struct mlx5e_eeprom { int lock_bit; int i2c_addr; int page_num; int device_addr; int module_num; int len; int type; int page_valid; u32 *data; }; enum mlx5e_link_mode { MLX5E_1000BASE_CX_SGMII = 0, MLX5E_1000BASE_KX = 1, MLX5E_10GBASE_CX4 = 2, MLX5E_10GBASE_KX4 = 3, MLX5E_10GBASE_KR = 4, MLX5E_20GBASE_KR2 = 5, MLX5E_40GBASE_CR4 = 6, MLX5E_40GBASE_KR4 = 7, MLX5E_56GBASE_R4 = 8, MLX5E_10GBASE_CR = 12, MLX5E_10GBASE_SR = 13, - MLX5E_10GBASE_ER = 14, + MLX5E_10GBASE_LR = 14, MLX5E_40GBASE_SR4 = 15, MLX5E_40GBASE_LR4 = 16, MLX5E_100GBASE_CR4 = 20, MLX5E_100GBASE_SR4 = 21, MLX5E_100GBASE_KR4 = 22, MLX5E_100GBASE_LR4 = 23, MLX5E_100BASE_TX = 24, MLX5E_100BASE_T = 25, MLX5E_10GBASE_T = 26, MLX5E_25GBASE_CR = 27, MLX5E_25GBASE_KR = 28, MLX5E_25GBASE_SR = 29, MLX5E_50GBASE_CR2 = 30, MLX5E_50GBASE_KR2 = 31, MLX5E_LINK_MODES_NUMBER, }; #define MLX5E_PROT_MASK(link_mode) (1 << (link_mode)) #define MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 1ULL) int mlx5e_xmit(struct ifnet *, struct mbuf *); int mlx5e_open_locked(struct ifnet *); int mlx5e_close_locked(struct ifnet *); void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, int event); void mlx5e_rx_cq_comp(struct mlx5_core_cq *); void mlx5e_tx_cq_comp(struct mlx5_core_cq *); struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq); void mlx5e_tx_que(void *context, int pending); int mlx5e_open_flow_table(struct mlx5e_priv *priv); void mlx5e_close_flow_table(struct mlx5e_priv *priv); void mlx5e_set_rx_mode_core(struct mlx5e_priv *priv); void mlx5e_set_rx_mode_work(struct work_struct *work); void mlx5e_vlan_rx_add_vid(void *, struct ifnet *, u16); void mlx5e_vlan_rx_kill_vid(void *, struct ifnet *, u16); void mlx5e_enable_vlan_filter(struct mlx5e_priv *priv); void mlx5e_disable_vlan_filter(struct mlx5e_priv *priv); int mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv); void mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv); static inline void mlx5e_tx_notify_hw(struct mlx5e_sq *sq, struct mlx5e_tx_wqe *wqe, int bf_sz) { u16 ofst = MLX5_BF_OFFSET + sq->bf_offset; /* ensure wqe is visible to device before updating doorbell record */ wmb(); *sq->wq.db = cpu_to_be32(sq->pc); /* * Ensure the doorbell record is visible to device before ringing * the doorbell: */ wmb(); if (bf_sz) { __iowrite64_copy(sq->uar_bf_map + ofst, &wqe->ctrl, bf_sz); /* flush the write-combining mapped buffer */ wmb(); } else { mlx5_write64((__be32 *)&wqe->ctrl, sq->uar_map + ofst, NULL); } sq->bf_offset ^= sq->bf_buf_size; } static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) { struct mlx5_core_cq *mcq; mcq = &cq->mcq; mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc); } extern const struct ethtool_ops mlx5e_ethtool_ops; void mlx5e_create_ethtool(struct mlx5e_priv *); void mlx5e_create_stats(struct sysctl_ctx_list *, struct sysctl_oid_list *, const char *, const char **, unsigned, u64 *); void mlx5e_send_nop(struct mlx5e_sq *, u32, bool); +int mlx5e_refresh_channel_params(struct mlx5e_priv *); #endif /* _MLX5_EN_H_ */ Index: projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c =================================================================== --- projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c (revision 292950) +++ projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c (revision 292951) @@ -1,545 +1,553 @@ /*- * Copyright (c) 2015 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "en.h" #include void mlx5e_create_stats(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *parent, const char *buffer, const char **desc, unsigned num, u64 * arg) { struct sysctl_oid *node; unsigned x; sysctl_ctx_init(ctx); node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, buffer, CTLFLAG_RD, NULL, "Statistics"); if (node == NULL) return; for (x = 0; x != num; x++) { SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(node), OID_AUTO, desc[2 * x], CTLFLAG_RD, arg + x, desc[2 * x + 1]); } } static int mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS) { struct mlx5e_priv *priv = arg1; uint64_t value; int was_opened; int error; PRIV_LOCK(priv); value = priv->params_ethtool.arg[arg2]; if (req != NULL) { error = sysctl_handle_64(oidp, &value, 0, req); if (error || req->newptr == NULL || value == priv->params_ethtool.arg[arg2]) goto done; /* assign new value */ priv->params_ethtool.arg[arg2] = value; } else { error = 0; } - /* check if device is gone */ if (priv->gone) { error = ENXIO; goto done; } + /* import RX coal time */ + if (priv->params_ethtool.rx_coalesce_usecs < 1) + priv->params_ethtool.rx_coalesce_usecs = 0; + else if (priv->params_ethtool.rx_coalesce_usecs > + MLX5E_FLD_MAX(cqc, cq_period)) { + priv->params_ethtool.rx_coalesce_usecs = + MLX5E_FLD_MAX(cqc, cq_period); + } + priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs; + /* import RX coal pkts */ + if (priv->params_ethtool.rx_coalesce_pkts < 1) + priv->params_ethtool.rx_coalesce_pkts = 0; + else if (priv->params_ethtool.rx_coalesce_pkts > + MLX5E_FLD_MAX(cqc, cq_max_count)) { + priv->params_ethtool.rx_coalesce_pkts = + MLX5E_FLD_MAX(cqc, cq_max_count); + } + priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts; + + /* import TX coal time */ + if (priv->params_ethtool.tx_coalesce_usecs < 1) + priv->params_ethtool.tx_coalesce_usecs = 0; + else if (priv->params_ethtool.tx_coalesce_usecs > + MLX5E_FLD_MAX(cqc, cq_period)) { + priv->params_ethtool.tx_coalesce_usecs = + MLX5E_FLD_MAX(cqc, cq_period); + } + priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs; + + /* import TX coal pkts */ + if (priv->params_ethtool.tx_coalesce_pkts < 1) + priv->params_ethtool.tx_coalesce_pkts = 0; + else if (priv->params_ethtool.tx_coalesce_pkts > + MLX5E_FLD_MAX(cqc, cq_max_count)) { + priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count); + } + priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts; + if (&priv->params_ethtool.arg[arg2] == &priv->params_ethtool.rx_pauseframe_control || &priv->params_ethtool.arg[arg2] == &priv->params_ethtool.tx_pauseframe_control) { /* range check parameters */ priv->params_ethtool.rx_pauseframe_control = priv->params_ethtool.rx_pauseframe_control ? 1 : 0; priv->params_ethtool.tx_pauseframe_control = priv->params_ethtool.tx_pauseframe_control ? 1 : 0; /* update firmware */ error = -mlx5_set_port_pause(priv->mdev, 1, priv->params_ethtool.rx_pauseframe_control, priv->params_ethtool.tx_pauseframe_control); goto done; } was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (was_opened) - mlx5e_close_locked(priv->ifp); + if (was_opened) { + u64 *xarg = priv->params_ethtool.arg + arg2; + if (xarg == &priv->params_ethtool.tx_coalesce_pkts || + xarg == &priv->params_ethtool.rx_coalesce_pkts || + xarg == &priv->params_ethtool.tx_coalesce_usecs || + xarg == &priv->params_ethtool.rx_coalesce_usecs) { + /* avoid downing and upping the network interface */ + error = mlx5e_refresh_channel_params(priv); + goto done; + } + mlx5e_close_locked(priv->ifp); + } /* import TX queue size */ if (priv->params_ethtool.tx_queue_size < (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)) { priv->params_ethtool.tx_queue_size = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE); } else if (priv->params_ethtool.tx_queue_size > priv->params_ethtool.tx_queue_size_max) { priv->params_ethtool.tx_queue_size = priv->params_ethtool.tx_queue_size_max; } priv->params.log_sq_size = order_base_2(priv->params_ethtool.tx_queue_size); /* import RX queue size */ if (priv->params_ethtool.rx_queue_size < (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE)) { priv->params_ethtool.rx_queue_size = (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE); } else if (priv->params_ethtool.rx_queue_size > priv->params_ethtool.rx_queue_size_max) { priv->params_ethtool.rx_queue_size = priv->params_ethtool.rx_queue_size_max; } priv->params.log_rq_size = order_base_2(priv->params_ethtool.rx_queue_size); priv->params.min_rx_wqes = min_t (u16, priv->params_ethtool.rx_queue_size - 1, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES); /* import number of channels */ if (priv->params_ethtool.channels < 1) priv->params_ethtool.channels = 1; else if (priv->params_ethtool.channels > (u64) priv->mdev->priv.eq_table.num_comp_vectors) { priv->params_ethtool.channels = (u64) priv->mdev->priv.eq_table.num_comp_vectors; } priv->params.num_channels = priv->params_ethtool.channels; /* import RX mode */ if (priv->params_ethtool.rx_coalesce_mode != 0) priv->params_ethtool.rx_coalesce_mode = 1; priv->params.rx_cq_moderation_mode = priv->params_ethtool.rx_coalesce_mode; /* import TX mode */ if (priv->params_ethtool.tx_coalesce_mode != 0) priv->params_ethtool.tx_coalesce_mode = 1; priv->params.tx_cq_moderation_mode = priv->params_ethtool.tx_coalesce_mode; - - /* import RX coal time */ - if (priv->params_ethtool.rx_coalesce_usecs < 1) - priv->params_ethtool.rx_coalesce_usecs = 0; - else if (priv->params_ethtool.rx_coalesce_usecs > - MLX5E_FLD_MAX(cqc, cq_period)) { - priv->params_ethtool.rx_coalesce_usecs = - MLX5E_FLD_MAX(cqc, cq_period); - } - priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs; - - /* import RX coal pkts */ - if (priv->params_ethtool.rx_coalesce_pkts < 1) - priv->params_ethtool.rx_coalesce_pkts = 0; - else if (priv->params_ethtool.rx_coalesce_pkts > - MLX5E_FLD_MAX(cqc, cq_max_count)) { - priv->params_ethtool.rx_coalesce_pkts = - MLX5E_FLD_MAX(cqc, cq_max_count); - } - priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts; - - /* import TX coal time */ - if (priv->params_ethtool.tx_coalesce_usecs < 1) - priv->params_ethtool.tx_coalesce_usecs = 0; - else if (priv->params_ethtool.tx_coalesce_usecs > - MLX5E_FLD_MAX(cqc, cq_period)) { - priv->params_ethtool.tx_coalesce_usecs = - MLX5E_FLD_MAX(cqc, cq_period); - } - priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs; - - /* import TX coal pkts */ - if (priv->params_ethtool.tx_coalesce_pkts < 1) - priv->params_ethtool.tx_coalesce_pkts = 0; - else if (priv->params_ethtool.tx_coalesce_pkts > - MLX5E_FLD_MAX(cqc, cq_max_count)) { - priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count); - } - priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts; /* we always agree to turn off HW LRO - but not always to turn on */ if (priv->params_ethtool.hw_lro) { if (priv->params_ethtool.hw_lro != 1) { priv->params_ethtool.hw_lro = priv->params.hw_lro_en; error = EINVAL; goto done; } if (priv->ifp->if_capenable & IFCAP_LRO) priv->params.hw_lro_en = !!MLX5_CAP_ETH(priv->mdev, lro_cap); else { /* set the correct (0) value to params_ethtool.hw_lro, issue a warning and return error */ priv->params_ethtool.hw_lro = 0; error = EINVAL; if_printf(priv->ifp, "Can't set HW_LRO to a device with LRO turned off"); goto done; } } else { priv->params.hw_lro_en = false; } if (&priv->params_ethtool.arg[arg2] == &priv->params_ethtool.cqe_zipping) { if (priv->params_ethtool.cqe_zipping && MLX5_CAP_GEN(priv->mdev, cqe_compression)) { priv->params.cqe_zipping_en = true; priv->params_ethtool.cqe_zipping = 1; } else { priv->params.cqe_zipping_en = false; priv->params_ethtool.cqe_zipping = 0; } } if (was_opened) mlx5e_open_locked(priv->ifp); done: PRIV_UNLOCK(priv); return (error); } /* * Read the first three bytes of the eeprom in order to get the needed info * for the whole reading. * Byte 0 - Identifier byte * Byte 1 - Revision byte * Byte 2 - Status byte */ static int mlx5e_get_eeprom_info(struct mlx5e_priv *priv, struct mlx5e_eeprom *eeprom) { struct mlx5_core_dev *dev = priv->mdev; u32 data = 0; int size_read = 0; int ret; ret = mlx5_query_module_num(dev, &eeprom->module_num); if (ret) { if_printf(priv->ifp, "%s:%d: Failed query module error=%d\n", __func__, __LINE__, ret); return (ret); } /* Read the first three bytes to get Identifier, Revision and Status */ ret = mlx5_query_eeprom(dev, eeprom->i2c_addr, eeprom->page_num, eeprom->device_addr, MLX5E_EEPROM_INFO_BYTES, eeprom->module_num, &data, &size_read); if (ret) { if_printf(priv->ifp, "%s:%d: Failed query eeprom module error=0x%x\n", __func__, __LINE__, ret); return (ret); } switch (data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) { case SFF_8024_ID_QSFP: eeprom->type = MLX5E_ETH_MODULE_SFF_8436; eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN; break; case SFF_8024_ID_QSFPPLUS: case SFF_8024_ID_QSFP28: if ((data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) == SFF_8024_ID_QSFP28 || ((data & MLX5_EEPROM_REVISION_ID_BYTE_MASK) >> 8) >= 0x3) { eeprom->type = MLX5E_ETH_MODULE_SFF_8636; eeprom->len = MLX5E_ETH_MODULE_SFF_8636_LEN; } else { eeprom->type = MLX5E_ETH_MODULE_SFF_8436; eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN; } if ((data & MLX5_EEPROM_PAGE_3_VALID_BIT_MASK) == 0) eeprom->page_valid = 1; break; case SFF_8024_ID_SFP: eeprom->type = MLX5E_ETH_MODULE_SFF_8472; eeprom->len = MLX5E_ETH_MODULE_SFF_8472_LEN; break; default: if_printf(priv->ifp, "%s:%d: Not recognized cable type = 0x%x(%s)\n", __func__, __LINE__, data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK, sff_8024_id[data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK]); return (EINVAL); } return (0); } /* Read both low and high pages of the eeprom */ static int mlx5e_get_eeprom(struct mlx5e_priv *priv, struct mlx5e_eeprom *ee) { struct mlx5_core_dev *dev = priv->mdev; int size_read = 0; int ret; if (ee->len == 0) return (EINVAL); /* Read low page of the eeprom */ while (ee->device_addr < ee->len) { ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num, ee->device_addr, ee->len - ee->device_addr, ee->module_num, ee->data + (ee->device_addr / 4), &size_read); if (ret) { if_printf(priv->ifp, "%s:%d: Failed reading eeprom, " "error = 0x%02x\n", __func__, __LINE__, ret); return (ret); } ee->device_addr += size_read; } /* Read high page of the eeprom */ if (ee->page_valid) { ee->device_addr = MLX5E_EEPROM_HIGH_PAGE_OFFSET; ee->page_num = MLX5E_EEPROM_HIGH_PAGE; size_read = 0; while (ee->device_addr < MLX5E_EEPROM_PAGE_LENGTH) { ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num, ee->device_addr, MLX5E_EEPROM_PAGE_LENGTH - ee->device_addr, ee->module_num, ee->data + (ee->len / 4) + ((ee->device_addr - MLX5E_EEPROM_HIGH_PAGE_OFFSET) / 4), &size_read); if (ret) { if_printf(priv->ifp, "%s:%d: Failed reading eeprom, " "error = 0x%02x\n", __func__, __LINE__, ret); return (ret); } ee->device_addr += size_read; } } return (0); } static void mlx5e_print_eeprom(struct mlx5e_eeprom *eeprom) { int row; int index_in_row; int byte_to_write = 0; int line_length = 16; printf("\nOffset\t\tValues\n"); printf("------\t\t------"); while (byte_to_write < eeprom->len) { printf("\n0x%04X\t\t", byte_to_write); for (index_in_row = 0; index_in_row < line_length; index_in_row++) { printf("%02X ", ((u8 *)eeprom->data)[byte_to_write]); byte_to_write++; } } if (eeprom->page_valid) { row = MLX5E_EEPROM_HIGH_PAGE_OFFSET; printf("\n\nUpper Page 0x03\n"); printf("\nOffset\t\tValues\n"); printf("------\t\t------"); while (row < MLX5E_EEPROM_PAGE_LENGTH) { printf("\n0x%04X\t\t", row); for (index_in_row = 0; index_in_row < line_length; index_in_row++) { printf("%02X ", ((u8 *)eeprom->data)[byte_to_write]); byte_to_write++; row++; } } } } /* * Read cable EEPROM module information by first inspecting the first * three bytes to get the initial information for a whole reading. * Information will be printed to dmesg. */ static int mlx5e_read_eeprom(SYSCTL_HANDLER_ARGS) { struct mlx5e_priv *priv = arg1; struct mlx5e_eeprom eeprom; int error; int result = 0; PRIV_LOCK(priv); error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) goto done; /* Check if device is gone */ if (priv->gone) { error = ENXIO; goto done; } if (result == 1) { eeprom.i2c_addr = MLX5E_I2C_ADDR_LOW; eeprom.device_addr = 0; eeprom.page_num = MLX5E_EEPROM_LOW_PAGE; eeprom.page_valid = 0; /* Read three first bytes to get important info */ error = mlx5e_get_eeprom_info(priv, &eeprom); if (error) { if_printf(priv->ifp, "%s:%d: Failed reading eeprom's " "initial information\n", __func__, __LINE__); error = 0; goto done; } /* * Allocate needed length buffer and additional space for * page 0x03 */ eeprom.data = malloc(eeprom.len + MLX5E_EEPROM_PAGE_LENGTH, M_MLX5EN, M_WAITOK | M_ZERO); /* Read the whole eeprom information */ error = mlx5e_get_eeprom(priv, &eeprom); if (error) { if_printf(priv->ifp, "%s:%d: Failed reading eeprom\n", __func__, __LINE__); error = 0; /* * Continue printing partial information in case of * an error */ } mlx5e_print_eeprom(&eeprom); free(eeprom.data, M_MLX5EN); } done: PRIV_UNLOCK(priv); return (error); } static const char *mlx5e_params_desc[] = { MLX5E_PARAMS(MLX5E_STATS_DESC) }; static const char *mlx5e_port_stats_debug_desc[] = { MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_DESC) }; static int mlx5e_ethtool_debug_stats(SYSCTL_HANDLER_ARGS) { struct mlx5e_priv *priv = arg1; int error; int sys_debug; sys_debug = priv->sysctl_debug; error = sysctl_handle_int(oidp, &priv->sysctl_debug, 0, req); if (error || !req->newptr) return (error); priv->sysctl_debug = !!priv->sysctl_debug; if (sys_debug == priv->sysctl_debug) return (error); if (priv->sysctl_debug) mlx5e_create_stats(&priv->stats.port_stats_debug.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), "debug_stats", mlx5e_port_stats_debug_desc, MLX5E_PORT_STATS_DEBUG_NUM, priv->stats.port_stats_debug.arg); else sysctl_ctx_free(&priv->stats.port_stats_debug.ctx); return (error); } void mlx5e_create_ethtool(struct mlx5e_priv *priv) { struct sysctl_oid *node; const char *pnameunit; unsigned x; /* set some defaults */ priv->params_ethtool.tx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; priv->params_ethtool.rx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; priv->params_ethtool.tx_queue_size = 1 << priv->params.log_sq_size; priv->params_ethtool.rx_queue_size = 1 << priv->params.log_rq_size; priv->params_ethtool.channels = priv->params.num_channels; priv->params_ethtool.coalesce_pkts_max = MLX5E_FLD_MAX(cqc, cq_max_count); priv->params_ethtool.coalesce_usecs_max = MLX5E_FLD_MAX(cqc, cq_period); priv->params_ethtool.rx_coalesce_mode = priv->params.rx_cq_moderation_mode; priv->params_ethtool.rx_coalesce_usecs = priv->params.rx_cq_moderation_usec; priv->params_ethtool.rx_coalesce_pkts = priv->params.rx_cq_moderation_pkts; priv->params_ethtool.tx_coalesce_mode = priv->params.tx_cq_moderation_mode; priv->params_ethtool.tx_coalesce_usecs = priv->params.tx_cq_moderation_usec; priv->params_ethtool.tx_coalesce_pkts = priv->params.tx_cq_moderation_pkts; priv->params_ethtool.hw_lro = priv->params.hw_lro_en; priv->params_ethtool.cqe_zipping = priv->params.cqe_zipping_en; /* create root node */ node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, "conf", CTLFLAG_RW, NULL, "Configuration"); if (node == NULL) return; for (x = 0; x != MLX5E_PARAMS_NUM; x++) { /* check for read-only parameter */ if (strstr(mlx5e_params_desc[2 * x], "_max") != NULL) { SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU", mlx5e_params_desc[2 * x + 1]); } else { #if (__FreeBSD_version < 1100000) char path[64]; #endif /* * NOTE: In FreeBSD-11 and newer the * CTLFLAG_RWTUN flag will take care of * loading default sysctl value from the * kernel environment, if any: */ SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU", mlx5e_params_desc[2 * x + 1]); #if (__FreeBSD_version < 1100000) /* compute path for sysctl */ snprintf(path, sizeof(path), "dev.mce.%d.conf.%s", device_get_unit(priv->mdev->pdev->dev.bsddev), mlx5e_params_desc[2 * x]); /* try to fetch tunable, if any */ if (TUNABLE_QUAD_FETCH(path, &priv->params_ethtool.arg[x])) mlx5e_ethtool_handler(NULL, priv, x, NULL); #endif } } SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "debug_stats", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, &mlx5e_ethtool_debug_stats, "I", "Extended debug statistics"); pnameunit = device_get_nameunit(priv->mdev->pdev->dev.bsddev); SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "device_name", CTLFLAG_RD, __DECONST(void *, pnameunit), 0, "PCI device name"); /* EEPROM support */ SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "eeprom_info", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, mlx5e_read_eeprom, "I", "EEPROM information"); } Index: projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_main.c =================================================================== --- projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision 292950) +++ projects/clang380-import/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision 292951) @@ -1,2971 +1,3038 @@ /*- * Copyright (c) 2015 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "en.h" #include #include #define ETH_DRIVER_VERSION "3.1.0-dev" char mlx5e_version[] = "Mellanox Ethernet driver" " (" ETH_DRIVER_VERSION ")"; struct mlx5e_rq_param { u32 rqc [MLX5_ST_SZ_DW(rqc)]; struct mlx5_wq_param wq; }; struct mlx5e_sq_param { u32 sqc [MLX5_ST_SZ_DW(sqc)]; struct mlx5_wq_param wq; }; struct mlx5e_cq_param { u32 cqc [MLX5_ST_SZ_DW(cqc)]; struct mlx5_wq_param wq; u16 eq_ix; }; struct mlx5e_channel_param { struct mlx5e_rq_param rq; struct mlx5e_sq_param sq; struct mlx5e_cq_param rx_cq; struct mlx5e_cq_param tx_cq; }; static const struct { u32 subtype; u64 baudrate; } mlx5e_mode_table[MLX5E_LINK_MODES_NUMBER] = { [MLX5E_1000BASE_CX_SGMII] = { .subtype = IFM_1000_CX_SGMII, .baudrate = IF_Mbps(1000ULL), }, [MLX5E_1000BASE_KX] = { .subtype = IFM_1000_KX, .baudrate = IF_Mbps(1000ULL), }, [MLX5E_10GBASE_CX4] = { .subtype = IFM_10G_CX4, .baudrate = IF_Gbps(10ULL), }, [MLX5E_10GBASE_KX4] = { .subtype = IFM_10G_KX4, .baudrate = IF_Gbps(10ULL), }, [MLX5E_10GBASE_KR] = { .subtype = IFM_10G_KR, .baudrate = IF_Gbps(10ULL), }, [MLX5E_20GBASE_KR2] = { .subtype = IFM_20G_KR2, .baudrate = IF_Gbps(20ULL), }, [MLX5E_40GBASE_CR4] = { .subtype = IFM_40G_CR4, .baudrate = IF_Gbps(40ULL), }, [MLX5E_40GBASE_KR4] = { .subtype = IFM_40G_KR4, .baudrate = IF_Gbps(40ULL), }, [MLX5E_56GBASE_R4] = { .subtype = IFM_56G_R4, .baudrate = IF_Gbps(56ULL), }, [MLX5E_10GBASE_CR] = { .subtype = IFM_10G_CR1, .baudrate = IF_Gbps(10ULL), }, [MLX5E_10GBASE_SR] = { .subtype = IFM_10G_SR, .baudrate = IF_Gbps(10ULL), }, - [MLX5E_10GBASE_ER] = { - .subtype = IFM_10G_ER, + [MLX5E_10GBASE_LR] = { + .subtype = IFM_10G_LR, .baudrate = IF_Gbps(10ULL), }, [MLX5E_40GBASE_SR4] = { .subtype = IFM_40G_SR4, .baudrate = IF_Gbps(40ULL), }, [MLX5E_40GBASE_LR4] = { .subtype = IFM_40G_LR4, .baudrate = IF_Gbps(40ULL), }, [MLX5E_100GBASE_CR4] = { .subtype = IFM_100G_CR4, .baudrate = IF_Gbps(100ULL), }, [MLX5E_100GBASE_SR4] = { .subtype = IFM_100G_SR4, .baudrate = IF_Gbps(100ULL), }, [MLX5E_100GBASE_KR4] = { .subtype = IFM_100G_KR4, .baudrate = IF_Gbps(100ULL), }, [MLX5E_100GBASE_LR4] = { .subtype = IFM_100G_LR4, .baudrate = IF_Gbps(100ULL), }, [MLX5E_100BASE_TX] = { .subtype = IFM_100_TX, .baudrate = IF_Mbps(100ULL), }, [MLX5E_100BASE_T] = { .subtype = IFM_100_T, .baudrate = IF_Mbps(100ULL), }, [MLX5E_10GBASE_T] = { .subtype = IFM_10G_T, .baudrate = IF_Gbps(10ULL), }, [MLX5E_25GBASE_CR] = { .subtype = IFM_25G_CR, .baudrate = IF_Gbps(25ULL), }, [MLX5E_25GBASE_KR] = { .subtype = IFM_25G_KR, .baudrate = IF_Gbps(25ULL), }, [MLX5E_25GBASE_SR] = { .subtype = IFM_25G_SR, .baudrate = IF_Gbps(25ULL), }, [MLX5E_50GBASE_CR2] = { .subtype = IFM_50G_CR2, .baudrate = IF_Gbps(50ULL), }, [MLX5E_50GBASE_KR2] = { .subtype = IFM_50G_KR2, .baudrate = IF_Gbps(50ULL), }, }; MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet"); static void mlx5e_update_carrier(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; u32 out[MLX5_ST_SZ_DW(ptys_reg)]; u32 eth_proto_oper; int error; u8 port_state; u8 i; port_state = mlx5_query_vport_state(mdev, MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT); if (port_state == VPORT_STATE_UP) { priv->media_status_last |= IFM_ACTIVE; } else { priv->media_status_last &= ~IFM_ACTIVE; priv->media_active_last = IFM_ETHER; if_link_state_change(priv->ifp, LINK_STATE_DOWN); return; } error = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN); if (error) { priv->media_active_last = IFM_ETHER; priv->ifp->if_baudrate = 1; if_printf(priv->ifp, "%s: query port ptys failed: 0x%x\n", __func__, error); return; } eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper); for (i = 0; i != MLX5E_LINK_MODES_NUMBER; i++) { if (mlx5e_mode_table[i].baudrate == 0) continue; if (MLX5E_PROT_MASK(i) & eth_proto_oper) { priv->ifp->if_baudrate = mlx5e_mode_table[i].baudrate; priv->media_active_last = mlx5e_mode_table[i].subtype | IFM_ETHER | IFM_FDX; } } if_link_state_change(priv->ifp, LINK_STATE_UP); } static void mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr) { struct mlx5e_priv *priv = dev->if_softc; ifmr->ifm_status = priv->media_status_last; ifmr->ifm_active = priv->media_active_last | (priv->params_ethtool.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) | (priv->params_ethtool.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0); } static u32 mlx5e_find_link_mode(u32 subtype) { u32 i; u32 link_mode = 0; for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) { if (mlx5e_mode_table[i].baudrate == 0) continue; if (mlx5e_mode_table[i].subtype == subtype) link_mode |= MLX5E_PROT_MASK(i); } return (link_mode); } static int mlx5e_media_change(struct ifnet *dev) { struct mlx5e_priv *priv = dev->if_softc; struct mlx5_core_dev *mdev = priv->mdev; u32 eth_proto_cap; u32 link_mode; int locked; int error; locked = PRIV_LOCKED(priv); if (!locked) PRIV_LOCK(priv); if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) { error = EINVAL; goto done; } link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media)); error = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN); if (error) { if_printf(dev, "Query port media capability failed\n"); goto done; } if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) link_mode = eth_proto_cap; else link_mode = link_mode & eth_proto_cap; if (!link_mode) { if_printf(dev, "Not supported link mode requested\n"); error = EINVAL; goto done; } mlx5_set_port_status(mdev, MLX5_PORT_DOWN); mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN); mlx5_set_port_status(mdev, MLX5_PORT_UP); done: if (!locked) PRIV_UNLOCK(priv); return (error); } static void mlx5e_update_carrier_work(struct work_struct *work) { struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, update_carrier_work); PRIV_LOCK(priv); if (test_bit(MLX5E_STATE_OPENED, &priv->state)) mlx5e_update_carrier(priv); PRIV_UNLOCK(priv); } static void mlx5e_update_pport_counters(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_pport_stats *s = &priv->stats.pport; struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug; u32 *in; u32 *out; u64 *ptr; unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg); unsigned x; unsigned y; in = mlx5_vzalloc(sz); out = mlx5_vzalloc(sz); if (in == NULL || out == NULL) goto free_out; ptr = (uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set); MLX5_SET(ppcnt_reg, in, local_port, 1); MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP); mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++) s->arg[y] = be64toh(ptr[x]); MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP); mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++) s->arg[y] = be64toh(ptr[x]); for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM + MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++) s_debug->arg[y] = be64toh(ptr[x]); MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP); mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++) s_debug->arg[y] = be64toh(ptr[x]); MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++) s_debug->arg[y] = be64toh(ptr[x]); free_out: kvfree(in); kvfree(out); } static void mlx5e_update_stats_work(struct work_struct *work) { struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, update_stats_work); struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_vport_stats *s = &priv->stats.vport; struct mlx5e_rq_stats *rq_stats; struct mlx5e_sq_stats *sq_stats; struct buf_ring *sq_br; #if (__FreeBSD_version < 1100000) struct ifnet *ifp = priv->ifp; #endif u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)]; u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); u64 tso_packets = 0; u64 tso_bytes = 0; u64 tx_queue_dropped = 0; u64 tx_defragged = 0; u64 tx_offload_none = 0; u64 lro_packets = 0; u64 lro_bytes = 0; u64 sw_lro_queued = 0; u64 sw_lro_flushed = 0; u64 rx_csum_none = 0; u64 rx_wqe_err = 0; u32 rx_out_of_buffer = 0; int i; int j; PRIV_LOCK(priv); out = mlx5_vzalloc(outlen); if (out == NULL) goto free_out; if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) goto free_out; /* Collect firts the SW counters and then HW for consistency */ for (i = 0; i < priv->params.num_channels; i++) { struct mlx5e_rq *rq = &priv->channel[i]->rq; rq_stats = &priv->channel[i]->rq.stats; /* collect stats from LRO */ rq_stats->sw_lro_queued = rq->lro.lro_queued; rq_stats->sw_lro_flushed = rq->lro.lro_flushed; sw_lro_queued += rq_stats->sw_lro_queued; sw_lro_flushed += rq_stats->sw_lro_flushed; lro_packets += rq_stats->lro_packets; lro_bytes += rq_stats->lro_bytes; rx_csum_none += rq_stats->csum_none; rx_wqe_err += rq_stats->wqe_err; for (j = 0; j < priv->num_tc; j++) { sq_stats = &priv->channel[i]->sq[j].stats; sq_br = priv->channel[i]->sq[j].br; tso_packets += sq_stats->tso_packets; tso_bytes += sq_stats->tso_bytes; tx_queue_dropped += sq_stats->dropped; tx_queue_dropped += sq_br->br_drops; tx_defragged += sq_stats->defragged; tx_offload_none += sq_stats->csum_offload_none; } } /* update counters */ s->tso_packets = tso_packets; s->tso_bytes = tso_bytes; s->tx_queue_dropped = tx_queue_dropped; s->tx_defragged = tx_defragged; s->lro_packets = lro_packets; s->lro_bytes = lro_bytes; s->sw_lro_queued = sw_lro_queued; s->sw_lro_flushed = sw_lro_flushed; s->rx_csum_none = rx_csum_none; s->rx_wqe_err = rx_wqe_err; /* HW counters */ memset(in, 0, sizeof(in)); MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER); MLX5_SET(query_vport_counter_in, in, op_mod, 0); MLX5_SET(query_vport_counter_in, in, other_vport, 0); memset(out, 0, outlen); /* get number of out-of-buffer drops first */ if (mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id, &rx_out_of_buffer)) goto free_out; /* accumulate difference into a 64-bit counter */ s->rx_out_of_buffer += (u64)(u32)(rx_out_of_buffer - s->rx_out_of_buffer_prev); s->rx_out_of_buffer_prev = rx_out_of_buffer; /* get port statistics */ if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen)) goto free_out; #define MLX5_GET_CTR(out, x) \ MLX5_GET64(query_vport_counter_out, out, x) s->rx_error_packets = MLX5_GET_CTR(out, received_errors.packets); s->rx_error_bytes = MLX5_GET_CTR(out, received_errors.octets); s->tx_error_packets = MLX5_GET_CTR(out, transmit_errors.packets); s->tx_error_bytes = MLX5_GET_CTR(out, transmit_errors.octets); s->rx_unicast_packets = MLX5_GET_CTR(out, received_eth_unicast.packets); s->rx_unicast_bytes = MLX5_GET_CTR(out, received_eth_unicast.octets); s->tx_unicast_packets = MLX5_GET_CTR(out, transmitted_eth_unicast.packets); s->tx_unicast_bytes = MLX5_GET_CTR(out, transmitted_eth_unicast.octets); s->rx_multicast_packets = MLX5_GET_CTR(out, received_eth_multicast.packets); s->rx_multicast_bytes = MLX5_GET_CTR(out, received_eth_multicast.octets); s->tx_multicast_packets = MLX5_GET_CTR(out, transmitted_eth_multicast.packets); s->tx_multicast_bytes = MLX5_GET_CTR(out, transmitted_eth_multicast.octets); s->rx_broadcast_packets = MLX5_GET_CTR(out, received_eth_broadcast.packets); s->rx_broadcast_bytes = MLX5_GET_CTR(out, received_eth_broadcast.octets); s->tx_broadcast_packets = MLX5_GET_CTR(out, transmitted_eth_broadcast.packets); s->tx_broadcast_bytes = MLX5_GET_CTR(out, transmitted_eth_broadcast.octets); s->rx_packets = s->rx_unicast_packets + s->rx_multicast_packets + s->rx_broadcast_packets - s->rx_out_of_buffer; s->rx_bytes = s->rx_unicast_bytes + s->rx_multicast_bytes + s->rx_broadcast_bytes; s->tx_packets = s->tx_unicast_packets + s->tx_multicast_packets + s->tx_broadcast_packets; s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes + s->tx_broadcast_bytes; /* Update calculated offload counters */ s->tx_csum_offload = s->tx_packets - tx_offload_none; s->rx_csum_good = s->rx_packets - s->rx_csum_none; /* Update per port counters */ mlx5e_update_pport_counters(priv); #if (__FreeBSD_version < 1100000) /* no get_counters interface in fbsd 10 */ ifp->if_ipackets = s->rx_packets; ifp->if_ierrors = s->rx_error_packets; ifp->if_iqdrops = s->rx_out_of_buffer; ifp->if_opackets = s->tx_packets; ifp->if_oerrors = s->tx_error_packets; ifp->if_snd.ifq_drops = s->tx_queue_dropped; ifp->if_ibytes = s->rx_bytes; ifp->if_obytes = s->tx_bytes; #endif free_out: kvfree(out); PRIV_UNLOCK(priv); } static void mlx5e_update_stats(void *arg) { struct mlx5e_priv *priv = arg; schedule_work(&priv->update_stats_work); callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv); } static void mlx5e_async_event_sub(struct mlx5e_priv *priv, enum mlx5_dev_event event) { switch (event) { case MLX5_DEV_EVENT_PORT_UP: case MLX5_DEV_EVENT_PORT_DOWN: schedule_work(&priv->update_carrier_work); break; default: break; } } static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv, enum mlx5_dev_event event, unsigned long param) { struct mlx5e_priv *priv = vpriv; mtx_lock(&priv->async_events_mtx); if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state)) mlx5e_async_event_sub(priv, event); mtx_unlock(&priv->async_events_mtx); } static void mlx5e_enable_async_events(struct mlx5e_priv *priv) { set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state); } static void mlx5e_disable_async_events(struct mlx5e_priv *priv) { mtx_lock(&priv->async_events_mtx); clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state); mtx_unlock(&priv->async_events_mtx); } static const char *mlx5e_rq_stats_desc[] = { MLX5E_RQ_STATS(MLX5E_STATS_DESC) }; static int mlx5e_create_rq(struct mlx5e_channel *c, struct mlx5e_rq_param *param, struct mlx5e_rq *rq) { struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; char buffer[16]; void *rqc = param->rqc; void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); int wq_sz; int err; int i; /* Create DMA descriptor TAG */ if ((err = -bus_dma_tag_create( bus_get_dma_tag(mdev->pdev->dev.bsddev), 1, /* any alignment */ 0, /* no boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ 1, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &rq->dma_tag))) goto done; err = mlx5_wq_ll_create(mdev, ¶m->wq, rqc_wq, &rq->wq, &rq->wq_ctrl); if (err) goto err_free_dma_tag; rq->wq.db = &rq->wq.db[MLX5_RCV_DBR]; if (priv->params.hw_lro_en) { rq->wqe_sz = priv->params.lro_wqe_sz; } else { rq->wqe_sz = MLX5E_SW2MB_MTU(priv->ifp->if_mtu); } if (rq->wqe_sz > MJUM16BYTES) { err = -ENOMEM; goto err_rq_wq_destroy; } else if (rq->wqe_sz > MJUM9BYTES) { rq->wqe_sz = MJUM16BYTES; } else if (rq->wqe_sz > MJUMPAGESIZE) { rq->wqe_sz = MJUM9BYTES; } else if (rq->wqe_sz > MCLBYTES) { rq->wqe_sz = MJUMPAGESIZE; } else { rq->wqe_sz = MCLBYTES; } wq_sz = mlx5_wq_ll_get_size(&rq->wq); rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO); if (rq->mbuf == NULL) { err = -ENOMEM; goto err_rq_wq_destroy; } for (i = 0; i != wq_sz; i++) { struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i); uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN; err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map); if (err != 0) { while (i--) bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map); goto err_rq_mbuf_free; } wqe->data.lkey = c->mkey_be; wqe->data.byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING); } rq->pdev = c->pdev; rq->ifp = c->ifp; rq->channel = c; rq->ix = c->ix; snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix); mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM, rq->stats.arg); #ifdef HAVE_TURBO_LRO if (tcp_tlro_init(&rq->lro, c->ifp, MLX5E_BUDGET_MAX) != 0) rq->lro.mbuf = NULL; #else if (tcp_lro_init(&rq->lro)) rq->lro.lro_cnt = 0; else rq->lro.ifp = c->ifp; #endif return (0); err_rq_mbuf_free: free(rq->mbuf, M_MLX5EN); err_rq_wq_destroy: mlx5_wq_destroy(&rq->wq_ctrl); err_free_dma_tag: bus_dma_tag_destroy(rq->dma_tag); done: return (err); } static void mlx5e_destroy_rq(struct mlx5e_rq *rq) { int wq_sz; int i; /* destroy all sysctl nodes */ sysctl_ctx_free(&rq->stats.ctx); /* free leftover LRO packets, if any */ #ifdef HAVE_TURBO_LRO tcp_tlro_free(&rq->lro); #else tcp_lro_free(&rq->lro); #endif wq_sz = mlx5_wq_ll_get_size(&rq->wq); for (i = 0; i != wq_sz; i++) { if (rq->mbuf[i].mbuf != NULL) { bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map); m_freem(rq->mbuf[i].mbuf); } bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map); } free(rq->mbuf, M_MLX5EN); mlx5_wq_destroy(&rq->wq_ctrl); } static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) { struct mlx5e_channel *c = rq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; void *in; void *rqc; void *wq; int inlen; int err; inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rq->wq_ctrl.buf.npages; in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); wq = MLX5_ADDR_OF(rqc, rqc, wq); memcpy(rqc, param->rqc, sizeof(param->rqc)); MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn); MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); MLX5_SET(rqc, rqc, flush_in_error_en, 1); if (priv->counter_set_id >= 0) MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id); MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); mlx5_fill_page_array(&rq->wq_ctrl.buf, (__be64 *) MLX5_ADDR_OF(wq, wq, pas)); err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); kvfree(in); return (err); } static int mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state) { struct mlx5e_channel *c = rq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; void *in; void *rqc; int inlen; int err; inlen = MLX5_ST_SZ_BYTES(modify_rq_in); in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); MLX5_SET(modify_rq_in, in, rqn, rq->rqn); MLX5_SET(modify_rq_in, in, rq_state, curr_state); MLX5_SET(rqc, rqc, state, next_state); err = mlx5_core_modify_rq(mdev, in, inlen); kvfree(in); return (err); } static void mlx5e_disable_rq(struct mlx5e_rq *rq) { struct mlx5e_channel *c = rq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; mlx5_core_destroy_rq(mdev, rq->rqn); } static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq) { struct mlx5e_channel *c = rq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_wq_ll *wq = &rq->wq; int i; for (i = 0; i < 1000; i++) { if (wq->cur_sz >= priv->params.min_rx_wqes) return (0); msleep(4); } return (-ETIMEDOUT); } static int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_rq_param *param, struct mlx5e_rq *rq) { int err; int i; err = mlx5e_create_rq(c, param, rq); if (err) return (err); err = mlx5e_enable_rq(rq, param); if (err) goto err_destroy_rq; err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); if (err) goto err_disable_rq; c->rq.enabled = 1; /* * Test send queues, which will trigger * "mlx5e_post_rx_wqes()": */ for (i = 0; i != c->num_tc; i++) mlx5e_send_nop(&c->sq[i], 1, true); return (0); err_disable_rq: mlx5e_disable_rq(rq); err_destroy_rq: mlx5e_destroy_rq(rq); return (err); } static void mlx5e_close_rq(struct mlx5e_rq *rq) { rq->enabled = 0; mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR); } static void mlx5e_close_rq_wait(struct mlx5e_rq *rq) { /* wait till RQ is empty */ while (!mlx5_wq_ll_is_empty(&rq->wq)) { msleep(4); rq->cq.mcq.comp(&rq->cq.mcq); } mlx5e_disable_rq(rq); mlx5e_destroy_rq(rq); } static void mlx5e_free_sq_db(struct mlx5e_sq *sq) { int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); int x; for (x = 0; x != wq_sz; x++) bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map); free(sq->mbuf, M_MLX5EN); } static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq) { int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); int err; int x; sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO); if (sq->mbuf == NULL) return (-ENOMEM); /* Create DMA descriptor MAPs */ for (x = 0; x != wq_sz; x++) { err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map); if (err != 0) { while (x--) bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map); free(sq->mbuf, M_MLX5EN); return (err); } } return (0); } static const char *mlx5e_sq_stats_desc[] = { MLX5E_SQ_STATS(MLX5E_STATS_DESC) }; static int mlx5e_create_sq(struct mlx5e_channel *c, int tc, struct mlx5e_sq_param *param, struct mlx5e_sq *sq) { struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; char buffer[16]; void *sqc = param->sqc; void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq); #ifdef RSS cpuset_t cpu_mask; int cpu_id; #endif int err; /* Create DMA descriptor TAG */ if ((err = -bus_dma_tag_create( bus_get_dma_tag(mdev->pdev->dev.bsddev), 1, /* any alignment */ 0, /* no boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MLX5E_MAX_TX_PAYLOAD_SIZE, /* maxsize */ MLX5E_MAX_TX_MBUF_FRAGS, /* nsegments */ MLX5E_MAX_TX_MBUF_SIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sq->dma_tag))) goto done; err = mlx5_alloc_map_uar(mdev, &sq->uar); if (err) goto err_free_dma_tag; err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq, &sq->wq_ctrl); if (err) goto err_unmap_free_uar; sq->wq.db = &sq->wq.db[MLX5_SND_DBR]; sq->uar_map = sq->uar.map; sq->uar_bf_map = sq->uar.bf_map; sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2; err = mlx5e_alloc_sq_db(sq); if (err) goto err_sq_wq_destroy; sq->pdev = c->pdev; sq->mkey_be = c->mkey_be; sq->channel = c; sq->tc = tc; sq->br = buf_ring_alloc(MLX5E_SQ_TX_QUEUE_SIZE, M_MLX5EN, M_WAITOK, &sq->lock); if (sq->br == NULL) { if_printf(c->ifp, "%s: Failed allocating sq drbr buffer\n", __func__); err = -ENOMEM; goto err_free_sq_db; } sq->sq_tq = taskqueue_create_fast("mlx5e_que", M_WAITOK, taskqueue_thread_enqueue, &sq->sq_tq); if (sq->sq_tq == NULL) { if_printf(c->ifp, "%s: Failed allocating taskqueue\n", __func__); err = -ENOMEM; goto err_free_drbr; } TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq); #ifdef RSS cpu_id = rss_getcpu(c->ix % rss_getnumbuckets()); CPU_SETOF(cpu_id, &cpu_mask); taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask, "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id); #else taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc); #endif snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc); mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM, sq->stats.arg); return (0); err_free_drbr: buf_ring_free(sq->br, M_MLX5EN); err_free_sq_db: mlx5e_free_sq_db(sq); err_sq_wq_destroy: mlx5_wq_destroy(&sq->wq_ctrl); err_unmap_free_uar: mlx5_unmap_free_uar(mdev, &sq->uar); err_free_dma_tag: bus_dma_tag_destroy(sq->dma_tag); done: return (err); } static void mlx5e_destroy_sq(struct mlx5e_sq *sq) { struct mlx5e_channel *c = sq->channel; struct mlx5e_priv *priv = c->priv; /* destroy all sysctl nodes */ sysctl_ctx_free(&sq->stats.ctx); mlx5e_free_sq_db(sq); mlx5_wq_destroy(&sq->wq_ctrl); mlx5_unmap_free_uar(priv->mdev, &sq->uar); taskqueue_drain(sq->sq_tq, &sq->sq_task); taskqueue_free(sq->sq_tq); buf_ring_free(sq->br, M_MLX5EN); } static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param) { struct mlx5e_channel *c = sq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; void *in; void *sqc; void *wq; int inlen; int err; inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * sq->wq_ctrl.buf.npages; in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); wq = MLX5_ADDR_OF(sqc, sqc, wq); memcpy(sqc, param->sqc, sizeof(param->sqc)); MLX5_SET(sqc, sqc, tis_num_0, priv->tisn[sq->tc]); MLX5_SET(sqc, sqc, cqn, c->sq[sq->tc].cq.mcq.cqn); MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); MLX5_SET(sqc, sqc, tis_lst_sz, 1); MLX5_SET(sqc, sqc, flush_in_error_en, 1); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, uar_page, sq->uar.index); MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); mlx5_fill_page_array(&sq->wq_ctrl.buf, (__be64 *) MLX5_ADDR_OF(wq, wq, pas)); err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn); kvfree(in); return (err); } static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state) { struct mlx5e_channel *c = sq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; void *in; void *sqc; int inlen; int err; inlen = MLX5_ST_SZ_BYTES(modify_sq_in); in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); MLX5_SET(modify_sq_in, in, sqn, sq->sqn); MLX5_SET(modify_sq_in, in, sq_state, curr_state); MLX5_SET(sqc, sqc, state, next_state); err = mlx5_core_modify_sq(mdev, in, inlen); kvfree(in); return (err); } static void mlx5e_disable_sq(struct mlx5e_sq *sq) { struct mlx5e_channel *c = sq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; mlx5_core_destroy_sq(mdev, sq->sqn); } static int mlx5e_open_sq(struct mlx5e_channel *c, int tc, struct mlx5e_sq_param *param, struct mlx5e_sq *sq) { int err; err = mlx5e_create_sq(c, tc, param, sq); if (err) return (err); err = mlx5e_enable_sq(sq, param); if (err) goto err_destroy_sq; err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY); if (err) goto err_disable_sq; atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_READY); return (0); err_disable_sq: mlx5e_disable_sq(sq); err_destroy_sq: mlx5e_destroy_sq(sq); return (err); } static void mlx5e_close_sq(struct mlx5e_sq *sq) { /* ensure hw is notified of all pending wqes */ if (mlx5e_sq_has_room_for(sq, 1)) mlx5e_send_nop(sq, 1, true); mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR); } static void mlx5e_close_sq_wait(struct mlx5e_sq *sq) { /* wait till SQ is empty */ while (sq->cc != sq->pc) { msleep(4); sq->cq.mcq.comp(&sq->cq.mcq); } mlx5e_disable_sq(sq); mlx5e_destroy_sq(sq); } static int mlx5e_create_cq(struct mlx5e_channel *c, struct mlx5e_cq_param *param, struct mlx5e_cq *cq, mlx5e_cq_comp_t *comp) { struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_core_cq *mcq = &cq->mcq; int eqn_not_used; int irqn; int err; u32 i; param->wq.buf_numa_node = 0; param->wq.db_numa_node = 0; param->eq_ix = c->ix; err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, &cq->wq_ctrl); if (err) return (err); mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn); mcq->cqe_sz = 64; mcq->set_ci_db = cq->wq_ctrl.db.db; mcq->arm_db = cq->wq_ctrl.db.db + 1; *mcq->set_ci_db = 0; *mcq->arm_db = 0; mcq->vector = param->eq_ix; mcq->comp = comp; mcq->event = mlx5e_cq_error_event; mcq->irqn = irqn; mcq->uar = &priv->cq_uar; for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); cqe->op_own = 0xf1; } cq->channel = c; return (0); } static void mlx5e_destroy_cq(struct mlx5e_cq *cq) { mlx5_wq_destroy(&cq->wq_ctrl); } static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, u8 moderation_mode) { struct mlx5e_channel *c = cq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_core_cq *mcq = &cq->mcq; void *in; void *cqc; int inlen; int irqn_not_used; int eqn; int err; inlen = MLX5_ST_SZ_BYTES(create_cq_in) + sizeof(u64) * cq->wq_ctrl.buf.npages; in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); memcpy(cqc, param->cqc, sizeof(param->cqc)); mlx5_fill_page_array(&cq->wq_ctrl.buf, (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas)); mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used); MLX5_SET(cqc, cqc, cq_period_mode, moderation_mode); MLX5_SET(cqc, cqc, c_eqn, eqn); MLX5_SET(cqc, cqc, uar_page, mcq->uar->index); MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - PAGE_SHIFT); MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); err = mlx5_core_create_cq(mdev, mcq, in, inlen); kvfree(in); if (err) return (err); mlx5e_cq_arm(cq); return (0); } static void mlx5e_disable_cq(struct mlx5e_cq *cq) { struct mlx5e_channel *c = cq->channel; struct mlx5e_priv *priv = c->priv; struct mlx5_core_dev *mdev = priv->mdev; mlx5_core_destroy_cq(mdev, &cq->mcq); } static int mlx5e_open_cq(struct mlx5e_channel *c, struct mlx5e_cq_param *param, struct mlx5e_cq *cq, mlx5e_cq_comp_t *comp, u8 moderation_mode) { int err; err = mlx5e_create_cq(c, param, cq, comp); if (err) return (err); err = mlx5e_enable_cq(cq, param, moderation_mode); if (err) goto err_destroy_cq; return (0); err_destroy_cq: mlx5e_destroy_cq(cq); return (err); } static void mlx5e_close_cq(struct mlx5e_cq *cq) { mlx5e_disable_cq(cq); mlx5e_destroy_cq(cq); } static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, struct mlx5e_channel_param *cparam) { u8 tx_moderation_mode; int err; int tc; switch (c->priv->params.tx_cq_moderation_mode) { case 0: tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; break; default: if (MLX5_CAP_GEN(c->priv->mdev, cq_period_start_from_cqe)) tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE; else tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; break; } for (tc = 0; tc < c->num_tc; tc++) { /* open completion queue */ err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq, &mlx5e_tx_cq_comp, tx_moderation_mode); if (err) goto err_close_tx_cqs; } return (0); err_close_tx_cqs: for (tc--; tc >= 0; tc--) mlx5e_close_cq(&c->sq[tc].cq); return (err); } static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) { int tc; for (tc = 0; tc < c->num_tc; tc++) mlx5e_close_cq(&c->sq[tc].cq); } static int mlx5e_open_sqs(struct mlx5e_channel *c, struct mlx5e_channel_param *cparam) { int err; int tc; for (tc = 0; tc < c->num_tc; tc++) { err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]); if (err) goto err_close_sqs; } return (0); err_close_sqs: for (tc--; tc >= 0; tc--) { mlx5e_close_sq(&c->sq[tc]); mlx5e_close_sq_wait(&c->sq[tc]); } return (err); } static void mlx5e_close_sqs(struct mlx5e_channel *c) { int tc; for (tc = 0; tc < c->num_tc; tc++) mlx5e_close_sq(&c->sq[tc]); } static void mlx5e_close_sqs_wait(struct mlx5e_channel *c) { int tc; for (tc = 0; tc < c->num_tc; tc++) mlx5e_close_sq_wait(&c->sq[tc]); } static void mlx5e_chan_mtx_init(struct mlx5e_channel *c) { int tc; mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF); for (tc = 0; tc < c->num_tc; tc++) { mtx_init(&c->sq[tc].lock, "mlx5tx", MTX_NETWORK_LOCK, MTX_DEF); mtx_init(&c->sq[tc].comp_lock, "mlx5comp", MTX_NETWORK_LOCK, MTX_DEF); } } static void mlx5e_chan_mtx_destroy(struct mlx5e_channel *c) { int tc; mtx_destroy(&c->rq.mtx); for (tc = 0; tc < c->num_tc; tc++) { mtx_destroy(&c->sq[tc].lock); mtx_destroy(&c->sq[tc].comp_lock); } } static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_channel_param *cparam, struct mlx5e_channel *volatile *cp) { struct mlx5e_channel *c; u8 rx_moderation_mode; int err; c = malloc(sizeof(*c), M_MLX5EN, M_WAITOK | M_ZERO); if (c == NULL) return (-ENOMEM); c->priv = priv; c->ix = ix; c->cpu = 0; c->pdev = &priv->mdev->pdev->dev; c->ifp = priv->ifp; c->mkey_be = cpu_to_be32(priv->mr.key); c->num_tc = priv->num_tc; /* init mutexes */ mlx5e_chan_mtx_init(c); /* open transmit completion queue */ err = mlx5e_open_tx_cqs(c, cparam); if (err) goto err_free; switch (priv->params.rx_cq_moderation_mode) { case 0: rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; break; default: if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe)) rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE; else rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; break; } /* open receive completion queue */ err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq, &mlx5e_rx_cq_comp, rx_moderation_mode); if (err) goto err_close_tx_cqs; err = mlx5e_open_sqs(c, cparam); if (err) goto err_close_rx_cq; err = mlx5e_open_rq(c, &cparam->rq, &c->rq); if (err) goto err_close_sqs; /* store channel pointer */ *cp = c; /* poll receive queue initially */ c->rq.cq.mcq.comp(&c->rq.cq.mcq); return (0); err_close_sqs: mlx5e_close_sqs(c); mlx5e_close_sqs_wait(c); err_close_rx_cq: mlx5e_close_cq(&c->rq.cq); err_close_tx_cqs: mlx5e_close_tx_cqs(c); err_free: /* destroy mutexes */ mlx5e_chan_mtx_destroy(c); free(c, M_MLX5EN); return (err); } static void mlx5e_close_channel(struct mlx5e_channel *volatile *pp) { struct mlx5e_channel *c = *pp; /* check if channel is already closed */ if (c == NULL) return; mlx5e_close_rq(&c->rq); mlx5e_close_sqs(c); } static void mlx5e_close_channel_wait(struct mlx5e_channel *volatile *pp) { struct mlx5e_channel *c = *pp; /* check if channel is already closed */ if (c == NULL) return; /* ensure channel pointer is no longer used */ *pp = NULL; mlx5e_close_rq_wait(&c->rq); mlx5e_close_sqs_wait(c); mlx5e_close_cq(&c->rq.cq); mlx5e_close_tx_cqs(c); /* destroy mutexes */ mlx5e_chan_mtx_destroy(c); free(c, M_MLX5EN); } static void mlx5e_build_rq_param(struct mlx5e_priv *priv, struct mlx5e_rq_param *param) { void *rqc = param->rqc; void *wq = MLX5_ADDR_OF(rqc, rqc, wq); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST); MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe))); MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size); MLX5_SET(wq, wq, pd, priv->pdn); param->wq.buf_numa_node = 0; param->wq.db_numa_node = 0; param->wq.linear = 1; } static void mlx5e_build_sq_param(struct mlx5e_priv *priv, struct mlx5e_sq_param *param) { void *sqc = param->sqc; void *wq = MLX5_ADDR_OF(sqc, sqc, wq); MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size); MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); MLX5_SET(wq, wq, pd, priv->pdn); param->wq.buf_numa_node = 0; param->wq.db_numa_node = 0; param->wq.linear = 1; } static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, struct mlx5e_cq_param *param) { void *cqc = param->cqc; MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index); } static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, struct mlx5e_cq_param *param) { void *cqc = param->cqc; /* * TODO The sysctl to control on/off is a bool value for now, which means * we only support CSUM, once HASH is implemnted we'll need to address that. */ if (priv->params.cqe_zipping_en) { MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM); MLX5_SET(cqc, cqc, cqe_compression_en, 1); } MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size); MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec); MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts); mlx5e_build_common_cq_param(priv, param); } static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, struct mlx5e_cq_param *param) { void *cqc = param->cqc; MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size); MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec); MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts); mlx5e_build_common_cq_param(priv, param); } static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam) { memset(cparam, 0, sizeof(*cparam)); mlx5e_build_rq_param(priv, &cparam->rq); mlx5e_build_sq_param(priv, &cparam->sq); mlx5e_build_rx_cq_param(priv, &cparam->rx_cq); mlx5e_build_tx_cq_param(priv, &cparam->tx_cq); } static int mlx5e_open_channels(struct mlx5e_priv *priv) { struct mlx5e_channel_param cparam; void *ptr; int err; int i; int j; priv->channel = malloc(priv->params.num_channels * sizeof(struct mlx5e_channel *), M_MLX5EN, M_WAITOK | M_ZERO); if (priv->channel == NULL) return (-ENOMEM); mlx5e_build_channel_param(priv, &cparam); for (i = 0; i < priv->params.num_channels; i++) { err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]); if (err) goto err_close_channels; } for (j = 0; j < priv->params.num_channels; j++) { err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq); if (err) goto err_close_channels; } return (0); err_close_channels: for (i--; i >= 0; i--) { mlx5e_close_channel(&priv->channel[i]); mlx5e_close_channel_wait(&priv->channel[i]); } /* remove "volatile" attribute from "channel" pointer */ ptr = __DECONST(void *, priv->channel); priv->channel = NULL; free(ptr, M_MLX5EN); return (err); } static void mlx5e_close_channels(struct mlx5e_priv *priv) { void *ptr; int i; if (priv->channel == NULL) return; for (i = 0; i < priv->params.num_channels; i++) mlx5e_close_channel(&priv->channel[i]); for (i = 0; i < priv->params.num_channels; i++) mlx5e_close_channel_wait(&priv->channel[i]); /* remove "volatile" attribute from "channel" pointer */ ptr = __DECONST(void *, priv->channel); priv->channel = NULL; free(ptr, M_MLX5EN); } static int +mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq) +{ + return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq, + priv->params.tx_cq_moderation_usec, + priv->params.tx_cq_moderation_pkts)); +} + +static int +mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq) +{ + return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq, + priv->params.rx_cq_moderation_usec, + priv->params.rx_cq_moderation_pkts)); +} + +static int +mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c) +{ + int err; + int i; + + if (c == NULL) + return (EINVAL); + + err = mlx5e_refresh_rq_params(priv, &c->rq); + if (err) + goto done; + + for (i = 0; i != c->num_tc; i++) { + err = mlx5e_refresh_sq_params(priv, &c->sq[i]); + if (err) + goto done; + } +done: + return (err); +} + +int +mlx5e_refresh_channel_params(struct mlx5e_priv *priv) +{ + int i; + + if (priv->channel == NULL) + return (EINVAL); + + for (i = 0; i < priv->params.num_channels; i++) { + int err; + + err = mlx5e_refresh_channel_params_sub(priv, priv->channel[i]); + if (err) + return (err); + } + return (0); +} + +static int mlx5e_open_tis(struct mlx5e_priv *priv, int tc) { struct mlx5_core_dev *mdev = priv->mdev; u32 in[MLX5_ST_SZ_DW(create_tis_in)]; void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); memset(in, 0, sizeof(in)); MLX5_SET(tisc, tisc, prio, tc); MLX5_SET(tisc, tisc, transport_domain, priv->tdn); return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc])); } static void mlx5e_close_tis(struct mlx5e_priv *priv, int tc) { mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]); } static int mlx5e_open_tises(struct mlx5e_priv *priv) { int num_tc = priv->num_tc; int err; int tc; for (tc = 0; tc < num_tc; tc++) { err = mlx5e_open_tis(priv, tc); if (err) goto err_close_tises; } return (0); err_close_tises: for (tc--; tc >= 0; tc--) mlx5e_close_tis(priv, tc); return (err); } static void mlx5e_close_tises(struct mlx5e_priv *priv) { int num_tc = priv->num_tc; int tc; for (tc = 0; tc < num_tc; tc++) mlx5e_close_tis(priv, tc); } static int mlx5e_open_rqt(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; u32 *in; u32 out[MLX5_ST_SZ_DW(create_rqt_out)]; void *rqtc; int inlen; int err; int sz; int i; sz = 1 << priv->params.rx_hash_log_tbl_sz; inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); MLX5_SET(rqtc, rqtc, rqt_max_size, sz); for (i = 0; i < sz; i++) { int ix; #ifdef RSS ix = rss_get_indirection_to_bucket(i); #else ix = i; #endif /* ensure we don't overflow */ ix %= priv->params.num_channels; MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn); } MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); memset(out, 0, sizeof(out)); err = mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out)); if (!err) priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn); kvfree(in); return (err); } static void mlx5e_close_rqt(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]; u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)]; memset(in, 0, sizeof(in)); MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn); mlx5_cmd_exec_check_status(priv->mdev, in, sizeof(in), out, sizeof(out)); } static void mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt) { void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); __be32 *hkey; MLX5_SET(tirc, tirc, transport_domain, priv->tdn); #define ROUGH_MAX_L2_L3_HDR_SZ 256 #define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\ MLX5_HASH_FIELD_SEL_DST_IP) #define MLX5_HASH_ALL (MLX5_HASH_FIELD_SEL_SRC_IP |\ MLX5_HASH_FIELD_SEL_DST_IP |\ MLX5_HASH_FIELD_SEL_L4_SPORT |\ MLX5_HASH_FIELD_SEL_L4_DPORT) #define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\ MLX5_HASH_FIELD_SEL_DST_IP |\ MLX5_HASH_FIELD_SEL_IPSEC_SPI) if (priv->params.hw_lro_en) { MLX5_SET(tirc, tirc, lro_enable_mask, MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); MLX5_SET(tirc, tirc, lro_max_msg_sz, (priv->params.lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); /* TODO: add the option to choose timer value dynamically */ MLX5_SET(tirc, tirc, lro_timeout_period_usecs, MLX5_CAP_ETH(priv->mdev, lro_timer_supported_periods[2])); } /* setup parameters for hashing TIR type, if any */ switch (tt) { case MLX5E_TT_ANY: MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); MLX5_SET(tirc, tirc, inline_rqn, priv->channel[0]->rq.rqn); break; default: MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); MLX5_SET(tirc, tirc, indirect_table, priv->rqtn); MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ); hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); #ifdef RSS /* * The FreeBSD RSS implementation does currently not * support symmetric Toeplitz hashes: */ MLX5_SET(tirc, tirc, rx_hash_symmetric, 0); rss_getkey((uint8_t *)hkey); #else MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); hkey[0] = cpu_to_be32(0xD181C62C); hkey[1] = cpu_to_be32(0xF7F4DB5B); hkey[2] = cpu_to_be32(0x1983A2FC); hkey[3] = cpu_to_be32(0x943E1ADB); hkey[4] = cpu_to_be32(0xD9389E6B); hkey[5] = cpu_to_be32(0xD1039C2C); hkey[6] = cpu_to_be32(0xA74499AD); hkey[7] = cpu_to_be32(0x593D56D9); hkey[8] = cpu_to_be32(0xF3253C06); hkey[9] = cpu_to_be32(0x2ADC1FFC); #endif break; } switch (tt) { case MLX5E_TT_IPV4_TCP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); #ifdef RSS if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) { MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); } else #endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; case MLX5E_TT_IPV6_TCP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); #ifdef RSS if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) { MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); } else #endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; case MLX5E_TT_IPV4_UDP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_UDP); #ifdef RSS if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) { MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); } else #endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; case MLX5E_TT_IPV6_UDP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_UDP); #ifdef RSS if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) { MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); } else #endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; case MLX5E_TT_IPV4_IPSEC_AH: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP_IPSEC_SPI); break; case MLX5E_TT_IPV6_IPSEC_AH: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP_IPSEC_SPI); break; case MLX5E_TT_IPV4_IPSEC_ESP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP_IPSEC_SPI); break; case MLX5E_TT_IPV6_IPSEC_ESP: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP_IPSEC_SPI); break; case MLX5E_TT_IPV4: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); break; case MLX5E_TT_IPV6: MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_IP); break; default: break; } } static int mlx5e_open_tir(struct mlx5e_priv *priv, int tt) { struct mlx5_core_dev *mdev = priv->mdev; u32 *in; void *tirc; int inlen; int err; inlen = MLX5_ST_SZ_BYTES(create_tir_in); in = mlx5_vzalloc(inlen); if (in == NULL) return (-ENOMEM); tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context); mlx5e_build_tir_ctx(priv, tirc, tt); err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]); kvfree(in); return (err); } static void mlx5e_close_tir(struct mlx5e_priv *priv, int tt) { mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]); } static int mlx5e_open_tirs(struct mlx5e_priv *priv) { int err; int i; for (i = 0; i < MLX5E_NUM_TT; i++) { err = mlx5e_open_tir(priv, i); if (err) goto err_close_tirs; } return (0); err_close_tirs: for (i--; i >= 0; i--) mlx5e_close_tir(priv, i); return (err); } static void mlx5e_close_tirs(struct mlx5e_priv *priv) { int i; for (i = 0; i < MLX5E_NUM_TT; i++) mlx5e_close_tir(priv, i); } /* * SW MTU does not include headers, * HW MTU includes all headers and checksums. */ static int mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu) { struct mlx5e_priv *priv = ifp->if_softc; struct mlx5_core_dev *mdev = priv->mdev; int hw_mtu; int err; err = mlx5_set_port_mtu(mdev, MLX5E_SW2HW_MTU(sw_mtu)); if (err) { if_printf(ifp, "%s: mlx5_set_port_mtu failed setting %d, err=%d\n", __func__, sw_mtu, err); return (err); } err = mlx5_query_port_oper_mtu(mdev, &hw_mtu); if (!err) { ifp->if_mtu = MLX5E_HW2SW_MTU(hw_mtu); if (ifp->if_mtu != sw_mtu) { if_printf(ifp, "Port MTU %d is different than " "ifp mtu %d\n", sw_mtu, (int)ifp->if_mtu); } } else { if_printf(ifp, "Query port MTU, after setting new " "MTU value, failed\n"); ifp->if_mtu = sw_mtu; } return (0); } int mlx5e_open_locked(struct ifnet *ifp) { struct mlx5e_priv *priv = ifp->if_softc; int err; /* check if already opened */ if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0) return (0); #ifdef RSS if (rss_getnumbuckets() > priv->params.num_channels) { if_printf(ifp, "NOTE: There are more RSS buckets(%u) than " "channels(%u) available\n", rss_getnumbuckets(), priv->params.num_channels); } #endif err = mlx5e_open_tises(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n", __func__, err); return (err); } err = mlx5_vport_alloc_q_counter(priv->mdev, &priv->counter_set_id); if (err) { if_printf(priv->ifp, "%s: mlx5_vport_alloc_q_counter failed: %d\n", __func__, err); goto err_close_tises; } err = mlx5e_open_channels(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n", __func__, err); goto err_dalloc_q_counter; } err = mlx5e_open_rqt(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n", __func__, err); goto err_close_channels; } err = mlx5e_open_tirs(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n", __func__, err); goto err_close_rqls; } err = mlx5e_open_flow_table(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n", __func__, err); goto err_close_tirs; } err = mlx5e_add_all_vlan_rules(priv); if (err) { if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n", __func__, err); goto err_close_flow_table; } set_bit(MLX5E_STATE_OPENED, &priv->state); mlx5e_update_carrier(priv); mlx5e_set_rx_mode_core(priv); return (0); err_close_flow_table: mlx5e_close_flow_table(priv); err_close_tirs: mlx5e_close_tirs(priv); err_close_rqls: mlx5e_close_rqt(priv); err_close_channels: mlx5e_close_channels(priv); err_dalloc_q_counter: mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id); err_close_tises: mlx5e_close_tises(priv); return (err); } static void mlx5e_open(void *arg) { struct mlx5e_priv *priv = arg; PRIV_LOCK(priv); if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP)) if_printf(priv->ifp, "%s: Setting port status to up failed\n", __func__); mlx5e_open_locked(priv->ifp); priv->ifp->if_drv_flags |= IFF_DRV_RUNNING; PRIV_UNLOCK(priv); } int mlx5e_close_locked(struct ifnet *ifp) { struct mlx5e_priv *priv = ifp->if_softc; /* check if already closed */ if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) return (0); clear_bit(MLX5E_STATE_OPENED, &priv->state); mlx5e_set_rx_mode_core(priv); mlx5e_del_all_vlan_rules(priv); if_link_state_change(priv->ifp, LINK_STATE_DOWN); mlx5e_close_flow_table(priv); mlx5e_close_tirs(priv); mlx5e_close_rqt(priv); mlx5e_close_channels(priv); mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id); mlx5e_close_tises(priv); return (0); } #if (__FreeBSD_version >= 1100000) static uint64_t mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt) { struct mlx5e_priv *priv = ifp->if_softc; u64 retval; /* PRIV_LOCK(priv); XXX not allowed */ switch (cnt) { case IFCOUNTER_IPACKETS: retval = priv->stats.vport.rx_packets; break; case IFCOUNTER_IERRORS: retval = priv->stats.vport.rx_error_packets; break; case IFCOUNTER_IQDROPS: retval = priv->stats.vport.rx_out_of_buffer; break; case IFCOUNTER_OPACKETS: retval = priv->stats.vport.tx_packets; break; case IFCOUNTER_OERRORS: retval = priv->stats.vport.tx_error_packets; break; case IFCOUNTER_IBYTES: retval = priv->stats.vport.rx_bytes; break; case IFCOUNTER_OBYTES: retval = priv->stats.vport.tx_bytes; break; case IFCOUNTER_IMCASTS: retval = priv->stats.vport.rx_multicast_packets; break; case IFCOUNTER_OMCASTS: retval = priv->stats.vport.tx_multicast_packets; break; case IFCOUNTER_OQDROPS: retval = priv->stats.vport.tx_queue_dropped; break; default: retval = if_get_counter_default(ifp, cnt); break; } /* PRIV_UNLOCK(priv); XXX not allowed */ return (retval); } #endif static void mlx5e_set_rx_mode(struct ifnet *ifp) { struct mlx5e_priv *priv = ifp->if_softc; schedule_work(&priv->set_rx_mode_work); } static int mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct mlx5e_priv *priv; struct ifreq *ifr; struct ifi2creq i2c; int error = 0; int mask = 0; int size_read = 0; int module_num; int max_mtu; + uint8_t read_addr; priv = ifp->if_softc; /* check if detaching */ if (priv == NULL || priv->gone != 0) return (ENXIO); switch (command) { case SIOCSIFMTU: ifr = (struct ifreq *)data; PRIV_LOCK(priv); mlx5_query_port_max_mtu(priv->mdev, &max_mtu); if (ifr->ifr_mtu >= MLX5E_MTU_MIN && ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) { int was_opened; was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); if (was_opened) mlx5e_close_locked(ifp); /* set new MTU */ mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu); if (was_opened) mlx5e_open_locked(ifp); } else { error = EINVAL; if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n", MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu)); } PRIV_UNLOCK(priv); break; case SIOCSIFFLAGS: if ((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { mlx5e_set_rx_mode(ifp); break; } PRIV_LOCK(priv); if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) mlx5e_open_locked(ifp); ifp->if_drv_flags |= IFF_DRV_RUNNING; mlx5_set_port_status(priv->mdev, MLX5_PORT_UP); } } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { mlx5_set_port_status(priv->mdev, MLX5_PORT_DOWN); if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0) mlx5e_close_locked(ifp); mlx5e_update_carrier(priv); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; } } PRIV_UNLOCK(priv); break; case SIOCADDMULTI: case SIOCDELMULTI: mlx5e_set_rx_mode(ifp); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: case SIOCGIFXMEDIA: ifr = (struct ifreq *)data; error = ifmedia_ioctl(ifp, ifr, &priv->media, command); break; case SIOCSIFCAP: ifr = (struct ifreq *)data; PRIV_LOCK(priv); mask = ifr->ifr_reqcap ^ ifp->if_capenable; if (mask & IFCAP_TXCSUM) { ifp->if_capenable ^= IFCAP_TXCSUM; ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); if (IFCAP_TSO4 & ifp->if_capenable && !(IFCAP_TXCSUM & ifp->if_capenable)) { ifp->if_capenable &= ~IFCAP_TSO4; ifp->if_hwassist &= ~CSUM_IP_TSO; if_printf(ifp, "tso4 disabled due to -txcsum.\n"); } } if (mask & IFCAP_TXCSUM_IPV6) { ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); if (IFCAP_TSO6 & ifp->if_capenable && !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { ifp->if_capenable &= ~IFCAP_TSO6; ifp->if_hwassist &= ~CSUM_IP6_TSO; if_printf(ifp, "tso6 disabled due to -txcsum6.\n"); } } if (mask & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; if (mask & IFCAP_RXCSUM_IPV6) ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; if (mask & IFCAP_TSO4) { if (!(IFCAP_TSO4 & ifp->if_capenable) && !(IFCAP_TXCSUM & ifp->if_capenable)) { if_printf(ifp, "enable txcsum first.\n"); error = EAGAIN; goto out; } ifp->if_capenable ^= IFCAP_TSO4; ifp->if_hwassist ^= CSUM_IP_TSO; } if (mask & IFCAP_TSO6) { if (!(IFCAP_TSO6 & ifp->if_capenable) && !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { if_printf(ifp, "enable txcsum6 first.\n"); error = EAGAIN; goto out; } ifp->if_capenable ^= IFCAP_TSO6; ifp->if_hwassist ^= CSUM_IP6_TSO; } if (mask & IFCAP_VLAN_HWFILTER) { if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) mlx5e_disable_vlan_filter(priv); else mlx5e_enable_vlan_filter(priv); ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; } if (mask & IFCAP_VLAN_HWTAGGING) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (mask & IFCAP_WOL_MAGIC) ifp->if_capenable ^= IFCAP_WOL_MAGIC; VLAN_CAPABILITIES(ifp); /* turn off LRO means also turn of HW LRO - if it's on */ if (mask & IFCAP_LRO) { int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); bool need_restart = false; ifp->if_capenable ^= IFCAP_LRO; if (!(ifp->if_capenable & IFCAP_LRO)) { if (priv->params.hw_lro_en) { priv->params.hw_lro_en = false; need_restart = true; /* Not sure this is the correct way */ priv->params_ethtool.hw_lro = priv->params.hw_lro_en; } } if (was_opened && need_restart) { mlx5e_close_locked(ifp); mlx5e_open_locked(ifp); } } out: PRIV_UNLOCK(priv); break; case SIOCGI2C: ifr = (struct ifreq *)data; /* * Copy from the user-space address ifr_data to the * kernel-space address i2c */ error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); if (error) break; if (i2c.len > sizeof(i2c.data)) { error = EINVAL; break; } PRIV_LOCK(priv); /* Get module_num which is required for the query_eeprom */ error = mlx5_query_module_num(priv->mdev, &module_num); if (error) { if_printf(ifp, "Query module num failed, eeprom " "reading is not supported\n"); goto err_i2c; } /* - * Note that we ignore i2c.addr here. The driver hardcodes - * the address to 0x50, while standard expects it to be 0xA0. + * Currently 0XA0 and 0xA2 are the only addresses permitted. + * The internal conversion is as follows: */ + if (i2c.dev_addr == 0xA0) + read_addr = MLX5E_I2C_ADDR_LOW; + else if (i2c.dev_addr == 0xA2) + read_addr = MLX5E_I2C_ADDR_HIGH; + else { + if_printf(ifp, "Query eeprom failed, " + "Invalid Address: %X\n", i2c.dev_addr); + error = EINVAL; + goto err_i2c; + } error = mlx5_query_eeprom(priv->mdev, - MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE, + read_addr, MLX5E_EEPROM_LOW_PAGE, (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num, (uint32_t *)i2c.data, &size_read); if (error) { if_printf(ifp, "Query eeprom failed, eeprom " "reading is not supported\n"); goto err_i2c; } if (i2c.len > MLX5_EEPROM_MAX_BYTES) { error = mlx5_query_eeprom(priv->mdev, - MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE, + read_addr, MLX5E_EEPROM_LOW_PAGE, (uint32_t)(i2c.offset + size_read), (uint32_t)(i2c.len - size_read), module_num, (uint32_t *)(i2c.data + size_read), &size_read); } if (error) { if_printf(ifp, "Query eeprom failed, eeprom " "reading is not supported\n"); goto err_i2c; } error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); err_i2c: PRIV_UNLOCK(priv); break; default: error = ether_ioctl(ifp, command, data); break; } return (error); } static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) { /* * TODO: uncoment once FW really sets all these bits if * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap || * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap || * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return * -ENOTSUPP; */ /* TODO: add more must-to-have features */ return (0); } static void mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, int num_comp_vectors) { /* * TODO: Consider link speed for setting "log_sq_size", * "log_rq_size" and "cq_moderation_xxx": */ priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; priv->params.rx_cq_moderation_usec = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE : MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; priv->params.rx_cq_moderation_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0; priv->params.rx_cq_moderation_pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; priv->params.tx_cq_moderation_usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; priv->params.tx_cq_moderation_pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; priv->params.min_rx_wqes = MLX5E_PARAMS_DEFAULT_MIN_RX_WQES; priv->params.rx_hash_log_tbl_sz = (order_base_2(num_comp_vectors) > MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ? order_base_2(num_comp_vectors) : MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ; priv->params.num_tc = 1; priv->params.default_vlan_prio = 0; priv->counter_set_id = -1; /* * hw lro is currently defaulted to off. when it won't anymore we * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)" */ priv->params.hw_lro_en = false; priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; priv->params.cqe_zipping_en = !!MLX5_CAP_GEN(mdev, cqe_compression); priv->mdev = mdev; priv->params.num_channels = num_comp_vectors; priv->order_base_2_num_channels = order_base_2(num_comp_vectors); priv->queue_mapping_channel_mask = roundup_pow_of_two(num_comp_vectors) - 1; priv->num_tc = priv->params.num_tc; priv->default_vlan_prio = priv->params.default_vlan_prio; INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); } static int mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn, struct mlx5_core_mr *mr) { struct ifnet *ifp = priv->ifp; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_create_mkey_mbox_in *in; int err; in = mlx5_vzalloc(sizeof(*in)); if (in == NULL) { if_printf(ifp, "%s: failed to allocate inbox\n", __func__); return (-ENOMEM); } in->seg.flags = MLX5_PERM_LOCAL_WRITE | MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA; in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64); in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); err = mlx5_core_create_mkey(mdev, mr, in, sizeof(*in), NULL, NULL, NULL); if (err) if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n", __func__, err); kvfree(in); return (err); } static const char *mlx5e_vport_stats_desc[] = { MLX5E_VPORT_STATS(MLX5E_STATS_DESC) }; static const char *mlx5e_pport_stats_desc[] = { MLX5E_PPORT_STATS(MLX5E_STATS_DESC) }; static void mlx5e_priv_mtx_init(struct mlx5e_priv *priv) { mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF); sx_init(&priv->state_lock, "mlx5state"); callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0); } static void mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv) { mtx_destroy(&priv->async_events_mtx); sx_destroy(&priv->state_lock); } static int sysctl_firmware(SYSCTL_HANDLER_ARGS) { /* * %d.%d%.d the string format. * fw_rev_{maj,min,sub} return u16, 2^16 = 65536. * We need at most 5 chars to store that. * It also has: two "." and NULL at the end, which means we need 18 * (5*3 + 3) chars at most. */ char fw[18]; struct mlx5e_priv *priv = arg1; int error; snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev), fw_rev_sub(priv->mdev)); error = sysctl_handle_string(oidp, fw, sizeof(fw), req); return (error); } static void mlx5e_add_hw_stats(struct mlx5e_priv *priv) { SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw), OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0, sysctl_firmware, "A", "HCA firmware version"); SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw), OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0, "Board ID"); } static void * mlx5e_create_ifp(struct mlx5_core_dev *mdev) { static volatile int mlx5_en_unit; struct ifnet *ifp; struct mlx5e_priv *priv; u8 dev_addr[ETHER_ADDR_LEN] __aligned(4); struct sysctl_oid_list *child; int ncv = mdev->priv.eq_table.num_comp_vectors; char unit[16]; int err; int i; u32 eth_proto_cap; if (mlx5e_check_required_hca_cap(mdev)) { mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n"); return (NULL); } priv = malloc(sizeof(*priv), M_MLX5EN, M_WAITOK | M_ZERO); if (priv == NULL) { mlx5_core_err(mdev, "malloc() failed\n"); return (NULL); } mlx5e_priv_mtx_init(priv); ifp = priv->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { mlx5_core_err(mdev, "if_alloc() failed\n"); goto err_free_priv; } ifp->if_softc = priv; if_initname(ifp, "mce", atomic_fetchadd_int(&mlx5_en_unit, 1)); ifp->if_mtu = ETHERMTU; ifp->if_init = mlx5e_open; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = mlx5e_ioctl; ifp->if_transmit = mlx5e_xmit; ifp->if_qflush = if_qflush; #if (__FreeBSD_version >= 1100000) ifp->if_get_counter = mlx5e_get_counter; #endif ifp->if_snd.ifq_maxlen = ifqmaxlen; /* * Set driver features */ ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; ifp->if_capabilities |= IFCAP_LRO; ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO; /* set TSO limits so that we don't have to drop TX packets */ ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */; ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE; ifp->if_capenable = ifp->if_capabilities; ifp->if_hwassist = 0; if (ifp->if_capenable & IFCAP_TSO) ifp->if_hwassist |= CSUM_TSO; if (ifp->if_capenable & IFCAP_TXCSUM) ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); /* ifnet sysctl tree */ sysctl_ctx_init(&priv->sysctl_ctx); priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev), OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name"); if (priv->sysctl_ifnet == NULL) { mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); goto err_free_sysctl; } snprintf(unit, sizeof(unit), "%d", ifp->if_dunit); priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit"); if (priv->sysctl_ifnet == NULL) { mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); goto err_free_sysctl; } /* HW sysctl tree */ child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev)); priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child, OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw"); if (priv->sysctl_hw == NULL) { mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); goto err_free_sysctl; } mlx5e_build_ifp_priv(mdev, priv, ncv); err = mlx5_alloc_map_uar(mdev, &priv->cq_uar); if (err) { if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n", __func__, err); goto err_free_sysctl; } err = mlx5_core_alloc_pd(mdev, &priv->pdn); if (err) { if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n", __func__, err); goto err_unmap_free_uar; } err = mlx5_alloc_transport_domain(mdev, &priv->tdn); if (err) { if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n", __func__, err); goto err_dealloc_pd; } err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr); if (err) { if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n", __func__, err); goto err_dealloc_transport_domain; } mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr); /* set default MTU */ mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu); /* Set desc */ device_set_desc(mdev->pdev->dev.bsddev, mlx5e_version); /* Set default media status */ priv->media_status_last = IFM_AVALID; priv->media_active_last = IFM_ETHER | IFM_AUTO; /* Pauseframes are enabled by default */ priv->params_ethtool.tx_pauseframe_control = 1; priv->params_ethtool.rx_pauseframe_control = 1; err = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN); if (err) { eth_proto_cap = 0; if_printf(ifp, "%s: Query port media capability failed, %d\n", __func__, err); } /* Setup supported medias */ ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, mlx5e_media_change, mlx5e_media_status); for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) { if (mlx5e_mode_table[i].baudrate == 0) continue; if (MLX5E_PROT_MASK(i) & eth_proto_cap) ifmedia_add(&priv->media, IFM_ETHER | mlx5e_mode_table[i].subtype | IFM_FDX, 0, NULL); } ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); ether_ifattach(ifp, dev_addr); /* Register for VLAN events */ priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); /* Link is down by default */ if_link_state_change(ifp, LINK_STATE_DOWN); mlx5e_enable_async_events(priv); mlx5e_add_hw_stats(priv); mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM, priv->stats.vport.arg); mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM, priv->stats.pport.arg); mlx5e_create_ethtool(priv); mtx_lock(&priv->async_events_mtx); mlx5e_update_stats(priv); mtx_unlock(&priv->async_events_mtx); return (priv); err_dealloc_transport_domain: mlx5_dealloc_transport_domain(mdev, priv->tdn); err_dealloc_pd: mlx5_core_dealloc_pd(mdev, priv->pdn); err_unmap_free_uar: mlx5_unmap_free_uar(mdev, &priv->cq_uar); err_free_sysctl: sysctl_ctx_free(&priv->sysctl_ctx); if_free(ifp); err_free_priv: mlx5e_priv_mtx_destroy(priv); free(priv, M_MLX5EN); return (NULL); } static void mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv) { struct mlx5e_priv *priv = vpriv; struct ifnet *ifp = priv->ifp; /* don't allow more IOCTLs */ priv->gone = 1; /* XXX wait a bit to allow IOCTL handlers to complete */ pause("W", hz); /* stop watchdog timer */ callout_drain(&priv->watchdog); if (priv->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); if (priv->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); /* make sure device gets closed */ PRIV_LOCK(priv); mlx5e_close_locked(ifp); PRIV_UNLOCK(priv); /* unregister device */ ifmedia_removeall(&priv->media); ether_ifdetach(ifp); if_free(ifp); /* destroy all remaining sysctl nodes */ if (priv->sysctl_debug) sysctl_ctx_free(&priv->stats.port_stats_debug.ctx); sysctl_ctx_free(&priv->stats.vport.ctx); sysctl_ctx_free(&priv->stats.pport.ctx); sysctl_ctx_free(&priv->sysctl_ctx); mlx5_core_destroy_mkey(priv->mdev, &priv->mr); mlx5_dealloc_transport_domain(priv->mdev, priv->tdn); mlx5_core_dealloc_pd(priv->mdev, priv->pdn); mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar); mlx5e_disable_async_events(priv); flush_scheduled_work(); mlx5e_priv_mtx_destroy(priv); free(priv, M_MLX5EN); } static void * mlx5e_get_ifp(void *vpriv) { struct mlx5e_priv *priv = vpriv; return (priv->ifp); } static struct mlx5_interface mlx5e_interface = { .add = mlx5e_create_ifp, .remove = mlx5e_destroy_ifp, .event = mlx5e_async_event, .protocol = MLX5_INTERFACE_PROTOCOL_ETH, .get_dev = mlx5e_get_ifp, }; void mlx5e_init(void) { mlx5_register_interface(&mlx5e_interface); } void mlx5e_cleanup(void) { mlx5_unregister_interface(&mlx5e_interface); } module_init_order(mlx5e_init, SI_ORDER_THIRD); module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD); #if (__FreeBSD_version >= 1100000) MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1); #endif MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1); MODULE_VERSION(mlx5en, 1); Index: projects/clang380-import/sys/kern/kern_ktr.c =================================================================== --- projects/clang380-import/sys/kern/kern_ktr.c (revision 292950) +++ projects/clang380-import/sys/kern/kern_ktr.c (revision 292951) @@ -1,477 +1,474 @@ /*- * Copyright (c) 2000 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This module holds the global variables used by KTR and the ktr_tracepoint() * function that does the actual tracing. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_ktr.h" #include "opt_alq.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#ifdef __sparc64__ -#include -#endif #ifdef DDB #include #include #endif #ifndef KTR_BOOT_ENTRIES #define KTR_BOOT_ENTRIES 1024 #endif #ifndef KTR_ENTRIES #define KTR_ENTRIES 1024 #endif /* Limit the allocations to something manageable. */ #define KTR_ENTRIES_MAX (8 * 1024 * 1024) #ifndef KTR_MASK #define KTR_MASK (0) #endif #ifndef KTR_CPUMASK #define KTR_CPUMASK CPUSET_FSET #endif #ifndef KTR_TIME #define KTR_TIME get_cyclecount() #endif #ifndef KTR_CPU #define KTR_CPU PCPU_GET(cpuid) #endif static MALLOC_DEFINE(M_KTR, "KTR", "KTR"); FEATURE(ktr, "Kernel support for KTR kernel tracing facility"); volatile int ktr_idx = 0; uint64_t ktr_mask = KTR_MASK; uint64_t ktr_compile = KTR_COMPILE; int ktr_entries = KTR_BOOT_ENTRIES; int ktr_version = KTR_VERSION; struct ktr_entry ktr_buf_init[KTR_BOOT_ENTRIES]; struct ktr_entry *ktr_buf = ktr_buf_init; cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK); static SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options"); SYSCTL_INT(_debug_ktr, OID_AUTO, version, CTLFLAG_RD, &ktr_version, 0, "Version of the KTR interface"); SYSCTL_UQUAD(_debug_ktr, OID_AUTO, compile, CTLFLAG_RD, &ktr_compile, 0, "Bitmask of KTR event classes compiled into the kernel"); static int sysctl_debug_ktr_cpumask(SYSCTL_HANDLER_ARGS) { char lktr_cpumask_str[CPUSETBUFSIZ]; cpuset_t imask; int error; cpusetobj_strprint(lktr_cpumask_str, &ktr_cpumask); error = sysctl_handle_string(oidp, lktr_cpumask_str, sizeof(lktr_cpumask_str), req); if (error != 0 || req->newptr == NULL) return (error); if (cpusetobj_strscan(&imask, lktr_cpumask_str) == -1) return (EINVAL); CPU_COPY(&imask, &ktr_cpumask); return (error); } SYSCTL_PROC(_debug_ktr, OID_AUTO, cpumask, CTLFLAG_RWTUN | CTLFLAG_MPSAFE | CTLTYPE_STRING, NULL, 0, sysctl_debug_ktr_cpumask, "S", "Bitmask of CPUs on which KTR logging is enabled"); static int sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS) { int clear, error; clear = 0; error = sysctl_handle_int(oidp, &clear, 0, req); if (error || !req->newptr) return (error); if (clear) { bzero(ktr_buf, sizeof(*ktr_buf) * ktr_entries); ktr_idx = 0; } return (error); } SYSCTL_PROC(_debug_ktr, OID_AUTO, clear, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_debug_ktr_clear, "I", "Clear KTR Buffer"); /* * This is a sysctl proc so that it is serialized as !MPSAFE along with * the other ktr sysctl procs. */ static int sysctl_debug_ktr_mask(SYSCTL_HANDLER_ARGS) { uint64_t mask; int error; mask = ktr_mask; error = sysctl_handle_64(oidp, &mask, 0, req); if (error || !req->newptr) return (error); ktr_mask = mask; return (error); } SYSCTL_PROC(_debug_ktr, OID_AUTO, mask, CTLTYPE_U64 | CTLFLAG_RWTUN, 0, 0, sysctl_debug_ktr_mask, "QU", "Bitmask of KTR event classes for which logging is enabled"); #if KTR_ENTRIES > KTR_BOOT_ENTRIES /* * A simplified version of sysctl_debug_ktr_entries. * No need to care about SMP, scheduling, etc. */ static void ktr_entries_initializer(void *dummy __unused) { uint64_t mask; /* Temporarily disable ktr in case malloc() is being traced. */ mask = ktr_mask; ktr_mask = 0; ktr_buf = malloc(sizeof(*ktr_buf) * KTR_ENTRIES, M_KTR, M_WAITOK | M_ZERO); memcpy(ktr_buf, ktr_buf_init + ktr_idx, (KTR_BOOT_ENTRIES - ktr_idx) * sizeof(*ktr_buf)); if (ktr_idx != 0) { memcpy(ktr_buf + KTR_BOOT_ENTRIES - ktr_idx, ktr_buf_init, ktr_idx * sizeof(*ktr_buf)); ktr_idx = KTR_BOOT_ENTRIES; } ktr_entries = KTR_ENTRIES; ktr_mask = mask; } SYSINIT(ktr_entries_initializer, SI_SUB_KMEM, SI_ORDER_ANY, ktr_entries_initializer, NULL); #endif static int sysctl_debug_ktr_entries(SYSCTL_HANDLER_ARGS) { uint64_t mask; int entries, error; struct ktr_entry *buf, *oldbuf; entries = ktr_entries; error = sysctl_handle_int(oidp, &entries, 0, req); if (error || !req->newptr) return (error); if (entries > KTR_ENTRIES_MAX) return (ERANGE); /* Disable ktr temporarily. */ mask = ktr_mask; ktr_mask = 0; /* Wait for threads to go idle. */ if ((error = quiesce_all_cpus("ktrent", PCATCH)) != 0) { ktr_mask = mask; return (error); } if (ktr_buf != ktr_buf_init) oldbuf = ktr_buf; else oldbuf = NULL; /* Allocate a new buffer. */ buf = malloc(sizeof(*buf) * entries, M_KTR, M_WAITOK | M_ZERO); /* Install the new buffer and restart ktr. */ ktr_buf = buf; ktr_entries = entries; ktr_idx = 0; ktr_mask = mask; if (oldbuf != NULL) free(oldbuf, M_KTR); return (error); } SYSCTL_PROC(_debug_ktr, OID_AUTO, entries, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_debug_ktr_entries, "I", "Number of entries in the KTR buffer"); #ifdef KTR_VERBOSE int ktr_verbose = KTR_VERBOSE; TUNABLE_INT("debug.ktr.verbose", &ktr_verbose); SYSCTL_INT(_debug_ktr, OID_AUTO, verbose, CTLFLAG_RW, &ktr_verbose, 0, ""); #endif #ifdef KTR_ALQ struct alq *ktr_alq; char ktr_alq_file[MAXPATHLEN] = "/tmp/ktr.out"; int ktr_alq_cnt = 0; int ktr_alq_depth = KTR_ENTRIES; int ktr_alq_enabled = 0; int ktr_alq_failed = 0; int ktr_alq_max = 0; SYSCTL_INT(_debug_ktr, OID_AUTO, alq_max, CTLFLAG_RW, &ktr_alq_max, 0, "Maximum number of entries to write"); SYSCTL_INT(_debug_ktr, OID_AUTO, alq_cnt, CTLFLAG_RD, &ktr_alq_cnt, 0, "Current number of written entries"); SYSCTL_INT(_debug_ktr, OID_AUTO, alq_failed, CTLFLAG_RD, &ktr_alq_failed, 0, "Number of times we overran the buffer"); SYSCTL_INT(_debug_ktr, OID_AUTO, alq_depth, CTLFLAG_RW, &ktr_alq_depth, 0, "Number of items in the write buffer"); SYSCTL_STRING(_debug_ktr, OID_AUTO, alq_file, CTLFLAG_RW, ktr_alq_file, sizeof(ktr_alq_file), "KTR logging file"); static int sysctl_debug_ktr_alq_enable(SYSCTL_HANDLER_ARGS) { int error; int enable; enable = ktr_alq_enabled; error = sysctl_handle_int(oidp, &enable, 0, req); if (error || !req->newptr) return (error); if (enable) { if (ktr_alq_enabled) return (0); error = alq_open(&ktr_alq, (const char *)ktr_alq_file, req->td->td_ucred, ALQ_DEFAULT_CMODE, sizeof(struct ktr_entry), ktr_alq_depth); if (error == 0) { ktr_alq_cnt = 0; ktr_alq_failed = 0; ktr_alq_enabled = 1; } } else { if (ktr_alq_enabled == 0) return (0); ktr_alq_enabled = 0; alq_close(ktr_alq); ktr_alq = NULL; } return (error); } SYSCTL_PROC(_debug_ktr, OID_AUTO, alq_enable, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_debug_ktr_alq_enable, "I", "Enable KTR logging"); #endif void ktr_tracepoint(uint64_t mask, const char *file, int line, const char *format, u_long arg1, u_long arg2, u_long arg3, u_long arg4, u_long arg5, u_long arg6) { struct ktr_entry *entry; #ifdef KTR_ALQ struct ale *ale = NULL; #endif int newindex, saveindex; #if defined(KTR_VERBOSE) || defined(KTR_ALQ) struct thread *td; #endif int cpu; if (panicstr || kdb_active) return; if ((ktr_mask & mask) == 0 || ktr_buf == NULL) return; cpu = KTR_CPU; if (!CPU_ISSET(cpu, &ktr_cpumask)) return; #if defined(KTR_VERBOSE) || defined(KTR_ALQ) td = curthread; if (td->td_pflags & TDP_INKTR) return; td->td_pflags |= TDP_INKTR; #endif #ifdef KTR_ALQ if (ktr_alq_enabled) { if (td->td_critnest == 0 && (td->td_flags & TDF_IDLETD) == 0 && td != ald_thread) { if (ktr_alq_max && ktr_alq_cnt > ktr_alq_max) goto done; if ((ale = alq_get(ktr_alq, ALQ_NOWAIT)) == NULL) { ktr_alq_failed++; goto done; } ktr_alq_cnt++; entry = (struct ktr_entry *)ale->ae_data; } else { goto done; } } else #endif { do { saveindex = ktr_idx; newindex = (saveindex + 1) % ktr_entries; } while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0); entry = &ktr_buf[saveindex]; } entry->ktr_timestamp = KTR_TIME; entry->ktr_cpu = cpu; entry->ktr_thread = curthread; if (file != NULL) while (strncmp(file, "../", 3) == 0) file += 3; entry->ktr_file = file; entry->ktr_line = line; #ifdef KTR_VERBOSE if (ktr_verbose) { #ifdef SMP printf("cpu%d ", cpu); #endif if (ktr_verbose > 1) { printf("%s.%d\t", entry->ktr_file, entry->ktr_line); } printf(format, arg1, arg2, arg3, arg4, arg5, arg6); printf("\n"); } #endif entry->ktr_desc = format; entry->ktr_parms[0] = arg1; entry->ktr_parms[1] = arg2; entry->ktr_parms[2] = arg3; entry->ktr_parms[3] = arg4; entry->ktr_parms[4] = arg5; entry->ktr_parms[5] = arg6; #ifdef KTR_ALQ if (ktr_alq_enabled && ale) alq_post(ktr_alq, ale); done: #endif #if defined(KTR_VERBOSE) || defined(KTR_ALQ) td->td_pflags &= ~TDP_INKTR; #endif } #ifdef DDB struct tstate { int cur; int first; }; static struct tstate tstate; static int db_ktr_verbose; static int db_mach_vtrace(void); DB_SHOW_COMMAND(ktr, db_ktr_all) { tstate.cur = (ktr_idx - 1) % ktr_entries; tstate.first = -1; db_ktr_verbose = 0; db_ktr_verbose |= (strchr(modif, 'v') != NULL) ? 2 : 0; db_ktr_verbose |= (strchr(modif, 'V') != NULL) ? 1 : 0; /* just timestap please */ if (strchr(modif, 'a') != NULL) { db_disable_pager(); while (cncheckc() != -1) if (db_mach_vtrace() == 0) break; } else { while (!db_pager_quit) if (db_mach_vtrace() == 0) break; } } static int db_mach_vtrace(void) { struct ktr_entry *kp; if (tstate.cur == tstate.first || ktr_buf == NULL) { db_printf("--- End of trace buffer ---\n"); return (0); } kp = &ktr_buf[tstate.cur]; /* Skip over unused entries. */ if (kp->ktr_desc == NULL) { db_printf("--- End of trace buffer ---\n"); return (0); } db_printf("%d (%p", tstate.cur, kp->ktr_thread); #ifdef SMP db_printf(":cpu%d", kp->ktr_cpu); #endif db_printf(")"); if (db_ktr_verbose >= 1) { db_printf(" %10.10lld", (long long)kp->ktr_timestamp); } if (db_ktr_verbose >= 2) { db_printf(" %s.%d", kp->ktr_file, kp->ktr_line); } db_printf(": "); db_printf(kp->ktr_desc, kp->ktr_parms[0], kp->ktr_parms[1], kp->ktr_parms[2], kp->ktr_parms[3], kp->ktr_parms[4], kp->ktr_parms[5]); db_printf("\n"); if (tstate.first == -1) tstate.first = tstate.cur; if (--tstate.cur < 0) tstate.cur = ktr_entries - 1; return (1); } #endif /* DDB */ Index: projects/clang380-import/sys/sparc64/include/ktr.h =================================================================== --- projects/clang380-import/sys/sparc64/include/ktr.h (revision 292950) +++ projects/clang380-import/sys/sparc64/include/ktr.h (revision 292951) @@ -1,108 +1,99 @@ /*- * Copyright (c) 1996 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI $Id: ktr.h,v 1.10.2.7 2000/03/16 21:44:42 cp Exp $ * $FreeBSD$ */ #ifndef _MACHINE_KTR_H_ #define _MACHINE_KTR_H_ #include -#ifndef LOCORE +#ifdef LOCORE -#define KTR_CPU PCPU_GET(mid) - -#else - /* - * XXX could really use another register... + * XXX could really use another register ... */ #define ATR(desc, r1, r2, r3, l1, l2) \ .sect .rodata ; \ l1: .asciz desc ; \ .previous ; \ SET(ktr_idx, r2, r1) ; \ lduw [r1], r2 ; \ l2: add r2, 1, r3 ; \ set KTR_ENTRIES - 1, r1 ; \ and r3, r1, r3 ; \ set ktr_idx, r1 ; \ casa [r1] ASI_N, r2, r3 ; \ cmp r2, r3 ; \ bne %icc, l2 ## b ; \ mov r3, r2 ; \ SET(ktr_buf, r3, r1) ; \ ldx [r1], r1 ; \ mulx r2, KTR_SIZEOF, r2 ; \ add r1, r2, r1 ; \ rd %tick, r2 ; \ stx r2, [r1 + KTR_TIMESTAMP] ; \ lduw [PCPU(MID)], r2 ; \ stw r2, [r1 + KTR_CPU] ; \ stw %g0, [r1 + KTR_LINE] ; \ stx %g0, [r1 + KTR_FILE] ; \ SET(l1 ## b, r3, r2) ; \ stx r2, [r1 + KTR_DESC] -/* - * NB: this clobbers %y. - */ #define CATR(mask, desc, r1, r2, r3, l1, l2, l3) \ setx mask, r3, r1 ; \ setx ktr_mask, r3, r2 ; \ ldx [r2], r2 ; \ and r2, r1, r1 ; \ brz r1, l3 ## f ; \ nop ; \ lduw [PCPU(CPUID)], r2 ; \ mov _NCPUBITS, r3 ; \ - mov %g0, %y ; \ - udiv r2, r3, r2 ; \ + udivx r2, r3, r2 ; \ srl r2, 0, r2 ; \ sllx r2, PTR_SHIFT, r2 ; \ SET(ktr_cpumask, r3, r1) ; \ ldx [r1 + r2], r1 ; \ lduw [PCPU(CPUID)], r2 ; \ mov _NCPUBITS, r3 ; \ - mov %g0, %y ; \ - udiv r2, r3, r2 ; \ + udivx r2, r3, r2 ; \ srl r2, 0, r2 ; \ smul r2, r3, r3 ; \ lduw [PCPU(CPUID)], r2 ; \ sub r2, r3, r3 ; \ mov 1, r2 ; \ sllx r2, r3, r2 ; \ andn r1, r2, r1 ; \ brz r1, l3 ## f ; \ nop ; \ ATR(desc, r1, r2, r3, l1, l2) #endif /* LOCORE */ #endif /* !_MACHINE_KTR_H_ */ Index: projects/clang380-import/sys/sparc64/sparc64/exception.S =================================================================== --- projects/clang380-import/sys/sparc64/sparc64/exception.S (revision 292950) +++ projects/clang380-import/sys/sparc64/sparc64/exception.S (revision 292951) @@ -1,3083 +1,3069 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * BSDI $Id: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp $ */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "assym.s" #define TSB_ASI 0x0 #define TSB_KERNEL 0x0 #define TSB_KERNEL_MASK 0x0 #define TSB_KERNEL_PHYS 0x0 #define TSB_KERNEL_PHYS_END 0x0 #define TSB_QUAD_LDD 0x0 .register %g2,#ignore .register %g3,#ignore .register %g6,#ignore .register %g7,#ignore /* * Atomically set a bit in a TTE. */ #define TTE_SET_BIT(r1, r2, r3, bit, a, asi) \ add r1, TTE_DATA, r1 ; \ LD(x, a) [r1] asi, r2 ; \ 9: or r2, bit, r3 ; \ CAS(x, a) [r1] asi, r2, r3 ; \ cmp r2, r3 ; \ bne,pn %xcc, 9b ; \ mov r3, r2 #define TTE_SET_REF(r1, r2, r3, a, asi) TTE_SET_BIT(r1, r2, r3, TD_REF, a, asi) #define TTE_SET_W(r1, r2, r3, a, asi) TTE_SET_BIT(r1, r2, r3, TD_W, a, asi) /* * Macros for spilling and filling live windows. * * NOTE: These macros use exactly 16 instructions, and it is assumed that the * handler will not use more than 24 instructions total, to leave room for * resume vectors which occupy the last 8 instructions. */ #define SPILL(storer, base, size, asi) \ storer %l0, [base + (0 * size)] asi ; \ storer %l1, [base + (1 * size)] asi ; \ storer %l2, [base + (2 * size)] asi ; \ storer %l3, [base + (3 * size)] asi ; \ storer %l4, [base + (4 * size)] asi ; \ storer %l5, [base + (5 * size)] asi ; \ storer %l6, [base + (6 * size)] asi ; \ storer %l7, [base + (7 * size)] asi ; \ storer %i0, [base + (8 * size)] asi ; \ storer %i1, [base + (9 * size)] asi ; \ storer %i2, [base + (10 * size)] asi ; \ storer %i3, [base + (11 * size)] asi ; \ storer %i4, [base + (12 * size)] asi ; \ storer %i5, [base + (13 * size)] asi ; \ storer %i6, [base + (14 * size)] asi ; \ storer %i7, [base + (15 * size)] asi #define FILL(loader, base, size, asi) \ loader [base + (0 * size)] asi, %l0 ; \ loader [base + (1 * size)] asi, %l1 ; \ loader [base + (2 * size)] asi, %l2 ; \ loader [base + (3 * size)] asi, %l3 ; \ loader [base + (4 * size)] asi, %l4 ; \ loader [base + (5 * size)] asi, %l5 ; \ loader [base + (6 * size)] asi, %l6 ; \ loader [base + (7 * size)] asi, %l7 ; \ loader [base + (8 * size)] asi, %i0 ; \ loader [base + (9 * size)] asi, %i1 ; \ loader [base + (10 * size)] asi, %i2 ; \ loader [base + (11 * size)] asi, %i3 ; \ loader [base + (12 * size)] asi, %i4 ; \ loader [base + (13 * size)] asi, %i5 ; \ loader [base + (14 * size)] asi, %i6 ; \ loader [base + (15 * size)] asi, %i7 #define ERRATUM50(reg) mov reg, reg #define KSTACK_SLOP 1024 /* * Sanity check the kernel stack and bail out if it's wrong. * XXX: doesn't handle being on the panic stack. */ #define KSTACK_CHECK \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ add %sp, SPOFF, %g1 ; \ andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ bnz,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [PCPU(CURTHREAD)], %g2 ; \ ldx [%g2 + TD_KSTACK], %g2 ; \ add %g2, KSTACK_SLOP, %g2 ; \ subcc %g1, %g2, %g1 ; \ ble,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ cmp %g1, %g2 ; \ bgt,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG .globl tl_text_begin tl_text_begin: nop ENTRY(tl1_kstack_fault) rdpr %tl, %g1 1: cmp %g1, 2 be,a 2f nop #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %tl, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM1] 9: #endif sub %g1, 1, %g1 wrpr %g1, 0, %tl ba,a %xcc, 1b nop 2: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" , %g1, %g2, %g3, 7, 8, 9) add %sp, SPOFF, %g2 stx %g2, [%g1 + KTR_PARM1] ldx [PCPU(CURTHREAD)], %g2 ldx [%g2 + TD_KSTACK], %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %canrestore, %g2 stx %g2, [%g1 + KTR_PARM3] rdpr %cansave, %g2 stx %g2, [%g1 + KTR_PARM4] rdpr %otherwin, %g2 stx %g2, [%g1 + KTR_PARM5] rdpr %wstate, %g2 stx %g2, [%g1 + KTR_PARM6] 9: #endif wrpr %g0, 0, %canrestore wrpr %g0, 6, %cansave wrpr %g0, 0, %otherwin wrpr %g0, WSTATE_KERNEL, %wstate sub ASP_REG, SPOFF + CCFSZ, %sp clr %fp set trap, %o2 ba %xcc, tl1_trap mov T_KSTACK_FAULT | T_KERNEL, %o0 END(tl1_kstack_fault) /* * Magic to resume from a spill or fill trap. If we get an alignment or an * MMU fault during a spill or a fill, this macro will detect the fault and * resume at a set instruction offset in the trap handler. * * To check if the previous trap was a spill/fill we convert the trapped pc * to a trap type and verify that it is in the range of spill/fill vectors. * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the * tl bit allows us to detect both ranges with one test. * * This is: * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 * * To calculate the new pc we take advantage of the xor feature of wrpr. * Forcing all the low bits of the trapped pc on we can produce any offset * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. * * 0x7f ^ 0x1f == 0x60 * 0x1f == (0x80 - 0x60) - 1 * * Which are the offset and xor value used to resume from alignment faults. */ /* * Determine if we have trapped inside of a spill/fill vector, and if so resume * at a fixed instruction offset in the trap vector. Must be called on * alternate globals. */ #define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ rdpr %tpc, %g1 ; \ ERRATUM50(%g1) ; \ rdpr %tba, %g2 ; \ sub %g1, %g2, %g2 ; \ srlx %g2, 5, %g2 ; \ andn %g2, 0x200, %g2 ; \ cmp %g2, 0x80 ; \ blu,pt %xcc, 9f ; \ cmp %g2, 0x100 ; \ bgeu,pt %xcc, 9f ; \ or %g1, 0x7f, %g1 ; \ wrpr %g1, xor, %tnpc ; \ stxa_g0_sfsr ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ; \ done ; \ 9: ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG /* * For certain faults we need to clear the SFSR MMU register before returning. */ #define RSF_CLR_SFSR \ wr %g0, ASI_DMMU, %asi ; \ stxa %g0, [%g0 + AA_DMMU_SFSR] %asi #define RSF_XOR(off) ((0x80 - off) - 1) /* * Instruction offsets in spill and fill trap handlers for handling certain * nested traps, and corresponding xor constants for wrpr. */ #define RSF_OFF_ALIGN 0x60 #define RSF_OFF_MMU 0x70 #define RESUME_SPILLFILL_ALIGN \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) #define RESUME_SPILLFILL_MMU \ RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) #define RESUME_SPILLFILL_MMU_CLR_SFSR \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) /* * Constant to add to %tnpc when taking a fill trap just before returning to * user mode. */ #define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill /* * Generate a T_SPILL or T_FILL trap if the window operation fails. */ #define RSF_TRAP(type) \ ba %xcc, tl0_sftrap ; \ mov type, %g2 ; \ .align 16 /* * Game over if the window operation fails. */ #define RSF_FATAL(type) \ ba %xcc, rsf_fatal ; \ mov type, %g2 ; \ .align 16 /* * Magic to resume from a failed fill a few instructions after the corrsponding * restore. This is used on return from the kernel to usermode. */ #define RSF_FILL_MAGIC \ rdpr %tnpc, %g1 ; \ add %g1, RSF_FILL_INC, %g1 ; \ wrpr %g1, 0, %tnpc ; \ done ; \ .align 16 /* * Spill to the pcb if a spill to the user stack in kernel mode fails. */ #define RSF_SPILL_TOPCB \ ba,a %xcc, tl1_spill_topcb ; \ nop ; \ .align 16 ENTRY(rsf_fatal) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" , %g1, %g3, %g4, 7, 8, 9) rdpr %tt, %g3 stx %g3, [%g1 + KTR_PARM1] stx %g2, [%g1 + KTR_PARM2] 9: #endif KSTACK_CHECK sir END(rsf_fatal) .data _ALIGN_DATA .globl intrnames, sintrnames intrnames: .space (IV_MAX + PIL_MAX) * (MAXCOMLEN + 1) sintrnames: .quad (IV_MAX + PIL_MAX) * (MAXCOMLEN + 1) .globl intrcnt, sintrcnt intrcnt: .space (IV_MAX + PIL_MAX) * 8 sintrcnt: .quad (IV_MAX + PIL_MAX) * 8 .text /* * Trap table and associated macros * * Due to its size a trap table is an inherently hard thing to represent in * code in a clean way. There are approximately 1024 vectors, of 8 or 32 * instructions each, many of which are identical. The way that this is * laid out is the instructions (8 or 32) for the actual trap vector appear * as an AS macro. In general this code branches to tl0_trap or tl1_trap, * but if not supporting code can be placed just after the definition of the * macro. The macros are then instantiated in a different section (.trap), * which is setup to be placed by the linker at the beginning of .text, and the * code around the macros is moved to the end of trap table. In this way the * code that must be sequential in memory can be split up, and located near * its supporting code so that it is easier to follow. */ /* * Clean window traps occur when %cleanwin is zero to ensure that data * is not leaked between address spaces in registers. */ .macro clean_window clr %o0 clr %o1 clr %o2 clr %o3 clr %o4 clr %o5 clr %o6 clr %o7 clr %l0 clr %l1 clr %l2 clr %l3 clr %l4 clr %l5 clr %l6 rdpr %cleanwin, %l7 inc %l7 wrpr %l7, 0, %cleanwin clr %l7 retry .align 128 .endm /* * Stack fixups for entry from user mode. We are still running on the * user stack, and with its live registers, so we must save soon. We * are on alternate globals so we do have some registers. Set the * transitional window state, and do the save. If this traps we * attempt to spill a window to the user stack. If this fails, we * spill the window to the pcb and continue. Spilling to the pcb * must not fail. * * NOTE: Must be called with alternate globals and clobbers %g1. */ .macro tl0_split rdpr %wstate, %g1 wrpr %g1, WSTATE_TRANSITION, %wstate save .endm .macro tl0_setup type tl0_split clr %o1 set trap, %o2 ba %xcc, tl0_utrap mov \type, %o0 .endm /* * Generic trap type. Call trap() with the specified type. */ .macro tl0_gen type tl0_setup \type .align 32 .endm /* * This is used to suck up the massive swaths of reserved trap types. * Generates count "reserved" trap vectors. */ .macro tl0_reserved count .rept \count tl0_gen T_RESERVED .endr .endm .macro tl1_split rdpr %wstate, %g1 wrpr %g1, WSTATE_NESTED, %wstate save %sp, -(CCFSZ + TF_SIZEOF), %sp .endm .macro tl1_setup type tl1_split clr %o1 set trap, %o2 ba %xcc, tl1_trap mov \type | T_KERNEL, %o0 .endm .macro tl1_gen type tl1_setup \type .align 32 .endm .macro tl1_reserved count .rept \count tl1_gen T_RESERVED .endr .endm .macro tl0_insn_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 /* * XXX in theory, a store to AA_IMMU_SFSR must be immediately * followed by a DONE, FLUSH or RETRY for USIII. In practice, * this triggers a RED state exception though. */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_INSTRUCTION_EXCEPTION, %g2 .align 32 .endm .macro tl0_data_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_DATA_EXCEPTION, %g2 .align 32 .endm .macro tl0_align wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 .align 32 .endm ENTRY(tl0_sfsr_trap) tl0_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl0_utrap mov %g2, %o0 END(tl0_sfsr_trap) .macro tl0_intr level, mask tl0_split set \mask, %o1 ba %xcc, tl0_intr mov \level, %o0 .align 32 .endm #define INTR(level, traplvl) \ tl ## traplvl ## _intr level, 1 << level #define TICK(traplvl) \ tl ## traplvl ## _intr PIL_TICK, 0x10001 #define INTR_LEVEL(tl) \ INTR(1, tl) ; \ INTR(2, tl) ; \ INTR(3, tl) ; \ INTR(4, tl) ; \ INTR(5, tl) ; \ INTR(6, tl) ; \ INTR(7, tl) ; \ INTR(8, tl) ; \ INTR(9, tl) ; \ INTR(10, tl) ; \ INTR(11, tl) ; \ INTR(12, tl) ; \ INTR(13, tl) ; \ TICK(tl) ; \ INTR(15, tl) ; .macro tl0_intr_level INTR_LEVEL(0) .endm .macro intr_vector ldxa [%g0] ASI_INTR_RECEIVE, %g1 andcc %g1, IRSR_BUSY, %g0 bnz,a,pt %xcc, intr_vector nop ba,a,pt %xcc, intr_vector_stray nop .align 32 .endm .macro tl0_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and executable and that the TTE tags match. */ brgez,pn %g7, 3f andcc %g7, TD_EXEC, %g0 bz,pn %xcc, 3f cmp %g3, %g6 bne,pn %xcc, 3f EMPTY /* * We matched a TTE, load the TLB. */ /* * Set the reference bit, if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl0_immu_miss_set_ref nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_IMMU_TAR] %asi stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG retry /* * Advance to the next TTE in this bucket, and check the low bits * of the bucket pointer to see if we've finished the bucket. */ 3: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_immu_miss_trap .align 128 .endm ENTRY(tl0_immu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g4, %g2, %g3, a, ASI_N) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 1f nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_IMMU_TAR] %asi stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl0_immu_miss_set_ref) ENTRY(tl0_immu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ sethi %hi(KERNBASE), %g2 stxa %g1, [%g0 + AA_IMMU_TAR] %asi flush %g2 /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 /* * Save the tag access register, and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl0_utrap mov T_INSTRUCTION_MISS, %o0 END(tl0_immu_miss_trap) .macro tl0_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ tl1_dmmu_miss_user: mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and that the virtual page numbers match. */ brgez,pn %g7, 3f cmp %g3, %g6 bne,pn %xcc, 3f EMPTY /* * We matched a TTE, load the TLB. */ /* * Set the reference bit, if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl0_dmmu_miss_set_ref nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG retry /* * Advance to the next TTE in this bucket, and check the low bits * of the bucket pointer to see if we've finished the bucket. */ 3: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_dmmu_miss_trap .align 128 .endm ENTRY(tl0_dmmu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g4, %g2, %g3, a, ASI_N) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 1f nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl0_dmmu_miss_set_ref) ENTRY(tl0_dmmu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Check if we actually came from the kernel. */ rdpr %tl, %g1 cmp %g1, 1 bgt,a,pn %xcc, 1f nop /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Save the tag access register and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl0_utrap mov T_DATA_MISS, %o0 /* * Handle faults during window spill/fill. */ 1: RESUME_SPILLFILL_MMU /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl0_dmmu_miss_trap) .macro tl0_dmmu_prot ba,a %xcc, tl0_dmmu_prot_1 nop .align 128 .endm ENTRY(tl0_dmmu_prot_1) /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ tl1_dmmu_prot_user: mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and writable and that the virtual page * numbers match. */ brgez,pn %g7, 4f andcc %g7, TD_SW, %g0 bz,pn %xcc, 4f cmp %g3, %g6 bne,pn %xcc, 4f nop /* * Set the hardware write bit. */ TTE_SET_W(%g4, %g2, %g3, a, ASI_N) /* * Delete the old TLB entry and clear the SFSR. */ srlx %g1, PAGE_SHIFT, %g3 sllx %g3, PAGE_SHIFT, %g3 stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 3f or %g2, TD_W, %g2 /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 3: retry /* * Check the low bits to see if we've finished the bucket. */ 4: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_dmmu_prot_trap nop END(tl0_dmmu_prot_1) ENTRY(tl0_dmmu_prot_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Check if we actually came from the kernel. */ rdpr %tl, %g1 cmp %g1, 1 bgt,a,pn %xcc, 1f nop /* * Load the SFAR, SFSR and TAR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Save the MMU registers and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl0_utrap mov T_DATA_PROTECTION, %o0 /* * Handle faults during window spill/fill. */ 1: RESUME_SPILLFILL_MMU_CLR_SFSR /* * Load the SFAR, SFSR and TAR. Clear the SFSR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl0_dmmu_prot_trap) .macro tl0_spill_0_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_spill_1_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_fill_0_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm .macro tl0_fill_1_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm ENTRY(tl0_sftrap) rdpr %tstate, %g1 and %g1, TSTATE_CWP_MASK, %g1 wrpr %g1, 0, %cwp tl0_split clr %o1 set trap, %o2 ba %xcc, tl0_trap mov %g2, %o0 END(tl0_sftrap) .macro tl0_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_syscall tl0_split clr %o1 set syscall, %o2 ba %xcc, tl0_trap mov T_SYSCALL, %o0 .align 32 .endm .macro tl0_fp_restore ba,a %xcc, tl0_fp_restore nop .align 32 .endm ENTRY(tl0_fp_restore) ldx [PCB_REG + PCB_FLAGS], %g1 andn %g1, PCB_FEF, %g1 stx %g1, [PCB_REG + PCB_FLAGS] wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_UFP + (0 * 64)] %asi, %f0 ldda [PCB_REG + PCB_UFP + (1 * 64)] %asi, %f16 ldda [PCB_REG + PCB_UFP + (2 * 64)] %asi, %f32 ldda [PCB_REG + PCB_UFP + (3 * 64)] %asi, %f48 membar #Sync done END(tl0_fp_restore) .macro tl1_insn_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 /* * XXX in theory, a store to AA_IMMU_SFSR must be immediately * followed by a DONE, FLUSH or RETRY for USIII. In practice, * this triggers a RED state exception though. */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl1_insn_exceptn_trap mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 .align 32 .endm ENTRY(tl1_insn_exceptn_trap) tl1_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov %g2, %o0 END(tl1_insn_exceptn_trap) .macro tl1_fp_disabled ba,a %xcc, tl1_fp_disabled_1 nop .align 32 .endm ENTRY(tl1_fp_disabled_1) rdpr %tpc, %g1 set fpu_fault_begin, %g2 sub %g1, %g2, %g1 cmp %g1, fpu_fault_size bgeu,a,pn %xcc, 1f nop wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_KFP + (0 * 64)] %asi, %f0 ldda [PCB_REG + PCB_KFP + (1 * 64)] %asi, %f16 ldda [PCB_REG + PCB_KFP + (2 * 64)] %asi, %f32 ldda [PCB_REG + PCB_KFP + (3 * 64)] %asi, %f48 membar #Sync retry 1: tl1_split clr %o1 set trap, %o2 ba %xcc, tl1_trap mov T_FP_DISABLED | T_KERNEL, %o0 END(tl1_fp_disabled_1) .macro tl1_data_excptn wrpr %g0, PSTATE_ALT, %pstate ba,a %xcc, tl1_data_excptn_trap nop .align 32 .endm ENTRY(tl1_data_excptn_trap) RESUME_SPILLFILL_MMU_CLR_SFSR ba %xcc, tl1_sfsr_trap mov T_DATA_EXCEPTION | T_KERNEL, %g2 END(tl1_data_excptn_trap) .macro tl1_align wrpr %g0, PSTATE_ALT, %pstate ba,a %xcc, tl1_align_trap nop .align 32 .endm ENTRY(tl1_align_trap) RESUME_SPILLFILL_ALIGN ba %xcc, tl1_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 END(tl1_align_trap) ENTRY(tl1_sfsr_trap) wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov %g2, %o0 END(tl1_sfsr_trap) .macro tl1_intr level, mask tl1_split set \mask, %o1 ba %xcc, tl1_intr mov \level, %o0 .align 32 .endm .macro tl1_intr_level INTR_LEVEL(1) .endm .macro tl1_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_immu_miss_patch_tsb_1 tl1_immu_miss_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_immu_miss_patch_tsb_mask_1 tl1_immu_miss_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_immu_miss_patch_quad_ldd_1 tl1_immu_miss_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and executable and that the virtual page * numbers match. */ brgez,pn %g7, tl1_immu_miss_trap andcc %g7, TD_EXEC, %g0 bz,pn %xcc, tl1_immu_miss_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_immu_miss_trap EMPTY /* * Set the reference bit if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl1_immu_miss_set_ref nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG retry .align 128 .endm ENTRY(tl1_immu_miss_set_ref) /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_immu_miss_patch_tsb_2 tl1_immu_miss_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_immu_miss_patch_tsb_mask_2 tl1_immu_miss_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the reference bit. */ .globl tl1_immu_miss_patch_asi_1 tl1_immu_miss_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_REF(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl1_immu_miss_set_ref) ENTRY(tl1_immu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_INSTRUCTION_MISS | T_KERNEL, %o0 END(tl1_immu_miss_trap) .macro tl1_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 /* * Extract the context from the contents of the tag access register. * If it's non-zero this is a fault on a user address. Note that the * faulting address is passed in %g1. */ sllx %g5, 64 - TAR_VPN_SHIFT, %g6 brnz,a,pn %g6, tl1_dmmu_miss_user mov %g5, %g1 /* * Check for the direct mapped physical region. These addresses have * the high bit set so they are negative. */ brlz,pn %g5, tl1_dmmu_miss_direct EMPTY /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_dmmu_miss_patch_tsb_1 tl1_dmmu_miss_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_miss_patch_tsb_mask_1 tl1_dmmu_miss_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_dmmu_miss_patch_quad_ldd_1 tl1_dmmu_miss_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and that the virtual page numbers match. */ brgez,pn %g7, tl1_dmmu_miss_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_dmmu_miss_trap EMPTY /* * Set the reference bit if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pt %xcc, tl1_dmmu_miss_set_ref nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG retry .align 128 .endm ENTRY(tl1_dmmu_miss_set_ref) /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_dmmu_miss_patch_tsb_mask_2 tl1_dmmu_miss_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_miss_patch_tsb_2 tl1_dmmu_miss_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the reference bit. */ .globl tl1_dmmu_miss_patch_asi_1 tl1_dmmu_miss_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_REF(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl1_dmmu_miss_set_ref) ENTRY(tl1_dmmu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 KSTACK_CHECK tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_trap) ENTRY(tl1_dmmu_miss_direct) /* * Mask off the high bits of the virtual address to get the physical * address, and or in the TTE bits. The virtual address bits that * correspond to the TTE valid and page size bits are left set, so * they don't have to be included in the TTE bits below. We know they * are set because the virtual address is in the upper va hole. * NB: if we are taking advantage of the ASI_ATOMIC_QUAD_LDD_PHYS * and we get a miss on the directly accessed kernel TSB we must not * set TD_CV in order to access it uniformly bypassing the D$. */ setx TLB_DIRECT_ADDRESS_MASK, %g7, %g4 and %g5, %g4, %g4 setx TLB_DIRECT_TO_TTE_MASK, %g7, %g6 and %g5, %g6, %g5 .globl tl1_dmmu_miss_direct_patch_tsb_phys_1 tl1_dmmu_miss_direct_patch_tsb_phys_1: sethi %uhi(TSB_KERNEL_PHYS), %g3 or %g3, %ulo(TSB_KERNEL_PHYS), %g3 sllx %g3, 32, %g3 sethi %hi(TSB_KERNEL_PHYS), %g3 or %g7, %g3, %g7 cmp %g4, %g7 bl,pt %xcc, 1f or %g5, TD_CP | TD_W, %g5 .globl tl1_dmmu_miss_direct_patch_tsb_phys_end_1 tl1_dmmu_miss_direct_patch_tsb_phys_end_1: sethi %uhi(TSB_KERNEL_PHYS_END), %g3 or %g3, %ulo(TSB_KERNEL_PHYS_END), %g3 sllx %g3, 32, %g3 sethi %hi(TSB_KERNEL_PHYS_END), %g7 or %g7, %g3, %g7 cmp %g4, %g7 bg,a,pt %xcc, 1f nop ba,pt %xcc, 2f nop 1: or %g5, TD_CV, %g5 /* * Load the TTE data into the TLB and retry the instruction. */ 2: stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry END(tl1_dmmu_miss_direct) .macro tl1_dmmu_prot ba,a %xcc, tl1_dmmu_prot_1 nop .align 128 .endm ENTRY(tl1_dmmu_prot_1) /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 /* * Extract the context from the contents of the tag access register. * If it's non-zero this is a fault on a user address. Note that the * faulting address is passed in %g1. */ sllx %g5, 64 - TAR_VPN_SHIFT, %g6 brnz,a,pn %g6, tl1_dmmu_prot_user mov %g5, %g1 /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_dmmu_prot_patch_tsb_1 tl1_dmmu_prot_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_prot_patch_tsb_mask_1 tl1_dmmu_prot_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_dmmu_prot_patch_quad_ldd_1 tl1_dmmu_prot_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and writeable and that the virtual page * numbers match. */ brgez,pn %g7, tl1_dmmu_prot_trap andcc %g7, TD_SW, %g0 bz,pn %xcc, tl1_dmmu_prot_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_dmmu_prot_trap EMPTY /* * Delete the old TLB entry and clear the SFSR. */ sllx %g5, TAR_VPN_SHIFT, %g6 or %g6, TLB_DEMAP_NUCLEUS, %g6 stxa %g0, [%g6] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_dmmu_prot_patch_tsb_2 tl1_dmmu_prot_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_prot_patch_tsb_mask_2 tl1_dmmu_prot_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the hardware write bit. */ .globl tl1_dmmu_prot_patch_asi_1 tl1_dmmu_prot_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_W(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f or %g6, TD_W, %g6 /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl1_dmmu_prot_1) ENTRY(tl1_dmmu_prot_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the SFAR, SFSR and TAR. Clear the SFSR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl1_dmmu_prot_trap) .macro tl1_spill_0_n SPILL(stx, %sp + SPOFF, 8, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_2_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_3_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_7_n btst 1, %sp bnz,a,pn %xcc, tl1_spill_0_n nop srl %sp, 0, %sp SPILL(stw, %sp, 4, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_0_o wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_1_o wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_2_o RSF_SPILL_TOPCB .align 128 .endm .macro tl1_fill_0_n FILL(ldx, %sp + SPOFF, 8, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm .macro tl1_fill_2_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_3_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_7_n btst 1, %sp bnz,a,pt %xcc, tl1_fill_0_n nop srl %sp, 0, %sp FILL(lduw, %sp, 4, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm /* * This is used to spill windows that are still occupied with user * data on kernel entry to the pcb. */ ENTRY(tl1_spill_topcb) wrpr %g0, PSTATE_ALT, %pstate /* Free some globals for our use. */ dec 24, ASP_REG stx %g1, [ASP_REG + 0] stx %g2, [ASP_REG + 8] stx %g3, [ASP_REG + 16] ldx [PCB_REG + PCB_NSAVED], %g1 sllx %g1, PTR_SHIFT, %g2 add %g2, PCB_REG, %g2 stx %sp, [%g2 + PCB_RWSP] sllx %g1, RW_SHIFT, %g2 add %g2, PCB_REG, %g2 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) inc %g1 stx %g1, [PCB_REG + PCB_NSAVED] #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" , %g1, %g2, %g3, 7, 8, 9) rdpr %tpc, %g2 stx %g2, [%g1 + KTR_PARM1] rdpr %tnpc, %g2 stx %g2, [%g1 + KTR_PARM2] stx %sp, [%g1 + KTR_PARM3] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM4] 9: #endif saved ldx [ASP_REG + 16], %g3 ldx [ASP_REG + 8], %g2 ldx [ASP_REG + 0], %g1 inc 24, ASP_REG retry END(tl1_spill_topcb) .macro tl1_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_soft count .rept \count tl1_gen T_SOFT | T_KERNEL .endr .endm .sect .trap .globl tl_trap_begin tl_trap_begin: nop .align 0x8000 .globl tl0_base tl0_base: tl0_reserved 8 ! 0x0-0x7 tl0_insn_excptn: tl0_insn_excptn ! 0x8 tl0_reserved 1 ! 0x9 tl0_insn_error: tl0_gen T_INSTRUCTION_ERROR ! 0xa tl0_reserved 5 ! 0xb-0xf tl0_insn_illegal: tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 tl0_priv_opcode: tl0_gen T_PRIVILEGED_OPCODE ! 0x11 tl0_reserved 14 ! 0x12-0x1f tl0_fp_disabled: tl0_gen T_FP_DISABLED ! 0x20 tl0_fp_ieee: tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 tl0_fp_other: tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 tl0_tag_ovflw: tl0_gen T_TAG_OVERFLOW ! 0x23 tl0_clean_window: clean_window ! 0x24 tl0_divide: tl0_gen T_DIVISION_BY_ZERO ! 0x28 tl0_reserved 7 ! 0x29-0x2f tl0_data_excptn: tl0_data_excptn ! 0x30 tl0_reserved 1 ! 0x31 tl0_data_error: tl0_gen T_DATA_ERROR ! 0x32 tl0_reserved 1 ! 0x33 tl0_align: tl0_align ! 0x34 tl0_align_lddf: tl0_gen T_RESERVED ! 0x35 tl0_align_stdf: tl0_gen T_RESERVED ! 0x36 tl0_priv_action: tl0_gen T_PRIVILEGED_ACTION ! 0x37 tl0_reserved 9 ! 0x38-0x40 tl0_intr_level: tl0_intr_level ! 0x41-0x4f tl0_reserved 16 ! 0x50-0x5f tl0_intr_vector: intr_vector ! 0x60 tl0_watch_phys: tl0_gen T_PA_WATCHPOINT ! 0x61 tl0_watch_virt: tl0_gen T_VA_WATCHPOINT ! 0x62 tl0_ecc: tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 tl0_immu_miss: tl0_immu_miss ! 0x64 tl0_dmmu_miss: tl0_dmmu_miss ! 0x68 tl0_dmmu_prot: tl0_dmmu_prot ! 0x6c tl0_reserved 16 ! 0x70-0x7f tl0_spill_0_n: tl0_spill_0_n ! 0x80 tl0_spill_1_n: tl0_spill_1_n ! 0x84 tl0_spill_bad 14 ! 0x88-0xbf tl0_fill_0_n: tl0_fill_0_n ! 0xc0 tl0_fill_1_n: tl0_fill_1_n ! 0xc4 tl0_fill_bad 14 ! 0xc8-0xff tl0_soft: tl0_gen T_SYSCALL ! 0x100 tl0_gen T_BREAKPOINT ! 0x101 tl0_gen T_DIVISION_BY_ZERO ! 0x102 tl0_reserved 1 ! 0x103 tl0_gen T_CLEAN_WINDOW ! 0x104 tl0_gen T_RANGE_CHECK ! 0x105 tl0_gen T_FIX_ALIGNMENT ! 0x106 tl0_gen T_INTEGER_OVERFLOW ! 0x107 tl0_gen T_SYSCALL ! 0x108 tl0_gen T_SYSCALL ! 0x109 tl0_fp_restore ! 0x10a tl0_reserved 5 ! 0x10b-0x10f tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f tl0_reserved 32 ! 0x120-0x13f tl0_gen T_SYSCALL ! 0x140 tl0_syscall ! 0x141 tl0_gen T_SYSCALL ! 0x142 tl0_gen T_SYSCALL ! 0x143 tl0_reserved 188 ! 0x144-0x1ff tl1_base: tl1_reserved 8 ! 0x200-0x207 tl1_insn_excptn: tl1_insn_excptn ! 0x208 tl1_reserved 1 ! 0x209 tl1_insn_error: tl1_gen T_INSTRUCTION_ERROR ! 0x20a tl1_reserved 5 ! 0x20b-0x20f tl1_insn_illegal: tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 tl1_priv_opcode: tl1_gen T_PRIVILEGED_OPCODE ! 0x211 tl1_reserved 14 ! 0x212-0x21f tl1_fp_disabled: tl1_fp_disabled ! 0x220 tl1_fp_ieee: tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 tl1_fp_other: tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 tl1_tag_ovflw: tl1_gen T_TAG_OVERFLOW ! 0x223 tl1_clean_window: clean_window ! 0x224 tl1_divide: tl1_gen T_DIVISION_BY_ZERO ! 0x228 tl1_reserved 7 ! 0x229-0x22f tl1_data_excptn: tl1_data_excptn ! 0x230 tl1_reserved 1 ! 0x231 tl1_data_error: tl1_gen T_DATA_ERROR ! 0x232 tl1_reserved 1 ! 0x233 tl1_align: tl1_align ! 0x234 tl1_align_lddf: tl1_gen T_RESERVED ! 0x235 tl1_align_stdf: tl1_gen T_RESERVED ! 0x236 tl1_priv_action: tl1_gen T_PRIVILEGED_ACTION ! 0x237 tl1_reserved 9 ! 0x238-0x240 tl1_intr_level: tl1_intr_level ! 0x241-0x24f tl1_reserved 16 ! 0x250-0x25f tl1_intr_vector: intr_vector ! 0x260 tl1_watch_phys: tl1_gen T_PA_WATCHPOINT ! 0x261 tl1_watch_virt: tl1_gen T_VA_WATCHPOINT ! 0x262 tl1_ecc: tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 tl1_immu_miss: tl1_immu_miss ! 0x264 tl1_dmmu_miss: tl1_dmmu_miss ! 0x268 tl1_dmmu_prot: tl1_dmmu_prot ! 0x26c tl1_reserved 16 ! 0x270-0x27f tl1_spill_0_n: tl1_spill_0_n ! 0x280 tl1_spill_bad 1 ! 0x284 tl1_spill_2_n: tl1_spill_2_n ! 0x288 tl1_spill_3_n: tl1_spill_3_n ! 0x28c tl1_spill_bad 3 ! 0x290-0x29b tl1_spill_7_n: tl1_spill_7_n ! 0x29c tl1_spill_0_o: tl1_spill_0_o ! 0x2a0 tl1_spill_1_o: tl1_spill_1_o ! 0x2a4 tl1_spill_2_o: tl1_spill_2_o ! 0x2a8 tl1_spill_bad 5 ! 0x2ac-0x2bf tl1_fill_0_n: tl1_fill_0_n ! 0x2c0 tl1_fill_bad 1 ! 0x2c4 tl1_fill_2_n: tl1_fill_2_n ! 0x2c8 tl1_fill_3_n: tl1_fill_3_n ! 0x2cc tl1_fill_bad 3 ! 0x2d0-0x2db tl1_fill_7_n: tl1_fill_7_n ! 0x2dc tl1_fill_bad 8 ! 0x2e0-0x2ff tl1_reserved 1 ! 0x300 tl1_breakpoint: tl1_gen T_BREAKPOINT ! 0x301 tl1_gen T_RSTRWP_PHYS ! 0x302 tl1_gen T_RSTRWP_VIRT ! 0x303 tl1_reserved 252 ! 0x304-0x3ff .globl tl_trap_end tl_trap_end: nop /* * User trap entry point * * void tl0_utrap(u_long type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_long sfsr) * * This handles redirecting a trap back to usermode as a user trap. The user * program must have first registered a trap handler with the kernel using * sysarch(SPARC_UTRAP_INSTALL). The trap handler is passed enough state * for it to return to the trapping code directly, it will not return through * the kernel. The trap type is passed in %o0, all out registers must be * passed through to tl0_trap or to usermode untouched. Note that the * parameters passed in out registers may be used by the user trap handler. * Do not change the registers they are passed in or you will break the ABI. * * If the trap type allows user traps, setup state to execute the user trap * handler and bounce back to usermode, otherwise branch to tl0_trap. */ ENTRY(tl0_utrap) /* * Check if the trap type allows user traps. */ cmp %o0, UT_MAX bge,a,pt %xcc, tl0_trap nop /* * Load the user trap handler from the utrap table. */ ldx [PCPU(CURTHREAD)], %l0 ldx [%l0 + TD_PROC], %l0 ldx [%l0 + P_MD + MD_UTRAP], %l0 brz,pt %l0, tl0_trap sllx %o0, PTR_SHIFT, %l1 ldx [%l0 + %l1], %l0 brz,a,pt %l0, tl0_trap nop /* * If the save we did on entry to the kernel had to spill a window * to the pcb, pretend we took a spill trap instead. Any windows * that are in the pcb must be copied out or the fill handler will * not be able to find them, since the user trap handler returns * directly to the trapping code. Note that we only support precise * user traps, which implies that the condition that caused the trap * in the first place is still valid, so it will occur again when we * re-execute the trapping instruction. */ ldx [PCB_REG + PCB_NSAVED], %l1 brnz,a,pn %l1, tl0_trap mov T_SPILL, %o0 /* * Pass %fsr in %l4, %tstate in %l5, %tpc in %l6 and %tnpc in %l7. * The ABI specifies only %l6 and %l7, but we need to pass %fsr or * it may be clobbered by an interrupt before the user trap code * can read it, and we must pass %tstate in order to restore %ccr * and %asi. The %fsr must be stored to memory, so we use the * temporary stack for that. */ rd %fprs, %l1 or %l1, FPRS_FEF, %l2 wr %l2, 0, %fprs dec 8, ASP_REG stx %fsr, [ASP_REG] ldx [ASP_REG], %l4 inc 8, ASP_REG wr %l1, 0, %fprs rdpr %tstate, %l5 rdpr %tpc, %l6 rdpr %tnpc, %l7 /* * Setup %tnpc to return to. */ wrpr %l0, 0, %tnpc /* * Setup %wstate for return, clear WSTATE_TRANSITION. */ rdpr %wstate, %l1 and %l1, WSTATE_NORMAL_MASK, %l1 wrpr %l1, 0, %wstate /* * Setup %tstate for return, change the saved cwp to point to the * current window instead of the window at the time of the trap. */ andn %l5, TSTATE_CWP_MASK, %l1 rdpr %cwp, %l2 wrpr %l1, %l2, %tstate /* * Setup %sp. Userland processes will crash if this is not setup. */ sub %fp, CCFSZ, %sp /* * Execute the user trap handler. */ done END(tl0_utrap) /* * (Real) User trap entry point * * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfsr, * u_int sfsr) * * The following setup has been performed: * - the windows have been split and the active user window has been saved * (maybe just to the pcb) * - we are on alternate globals and interrupts are disabled * * We switch to the kernel stack, build a trapframe, switch to normal * globals, enable interrupts and call trap. * * NOTE: We must be very careful setting up the per-cpu pointer. We know that * it has been pre-set in alternate globals, so we read it from there and setup * the normal %g7 *before* enabling interrupts. This avoids any possibility * of cpu migration and using the wrong pcpup. */ ENTRY(tl0_trap) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif 1: and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] rd %gsr, %l6 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] set tl0_ret - 8, %o7 jmpl %o2, %g0 add %sp, CCFSZ + SPOFF, %o0 END(tl0_trap) /* * void tl0_intr(u_int level, u_int mask) */ ENTRY(tl0_intr) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %clear_softint and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] rd %gsr, %l6 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] wr %g0, 0, %fprs mov %o0, %l3 mov T_INTERRUPT, %o1 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] SET(intr_handlers, %l1, %l0) sllx %l3, IH_SHIFT, %l1 ldx [%l0 + %l1], %l1 KASSERT(%l1, "tl0_intr: ih null") call %l1 add %sp, CCFSZ + SPOFF, %o0 /* %l3 contains PIL */ SET(intrcnt, %l1, %l2) prefetcha [%l2] ASI_N, 1 SET(pil_countp, %l1, %l0) sllx %l3, 1, %l1 lduh [%l0 + %l1], %l0 sllx %l0, 3, %l0 add %l0, %l2, %l0 ldx [%l0], %l1 inc %l1 stx %l1, [%l0] lduw [PCPU(CNT) + V_INTR], %l0 inc %l0 stw %l0, [PCPU(CNT) + V_INTR] ba,a %xcc, tl0_ret nop END(tl0_intr) /* * Initiate return to usermode. * * Called with a trapframe on the stack. The window that was setup in * tl0_trap may have been used by "fast" trap handlers that pretend to be * leaf functions, so all ins and locals may have been clobbered since * then. * * This code is rather long and complicated. */ ENTRY(tl0_ret) /* * Check for pending asts atomically with returning. We must raise * the PIL before checking, and if no asts are found the PIL must * remain raised until the retry is executed, or we risk missing asts * caused by interrupts occurring after the test. If the PIL is * lowered, as it is when we call ast, the check must be re-executed. */ wrpr %g0, PIL_TICK, %pil ldx [PCPU(CURTHREAD)], %l0 lduw [%l0 + TD_FLAGS], %l1 set TDF_ASTPENDING | TDF_NEEDRESCHED, %l2 and %l1, %l2, %l1 brz,a,pt %l1, 1f nop /* * We have an AST. Re-enable interrupts and handle it, then restart * the return sequence. */ wrpr %g0, 0, %pil call ast add %sp, CCFSZ + SPOFF, %o0 ba,a %xcc, tl0_ret nop /* * Check for windows that were spilled to the pcb and need to be * copied out. This must be the last thing that is done before the * return to usermode. If there are still user windows in the cpu * and we call a nested function after this, which causes them to be * spilled to the pcb, they will not be copied out and the stack will * be inconsistent. */ 1: ldx [PCB_REG + PCB_NSAVED], %l1 brz,a,pt %l1, 2f nop wrpr %g0, 0, %pil mov T_SPILL, %o0 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, SPOFF + CCFSZ, %o0 ba,a %xcc, tl0_ret nop /* * Restore the out and most global registers from the trapframe. * The ins will become the outs when we restore below. */ 2: ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 /* * Load everything we need to restore below before disabling * interrupts. */ ldx [%sp + SPOFF + CCFSZ + TF_FPRS], %l0 ldx [%sp + SPOFF + CCFSZ + TF_GSR], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l3 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l4 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l5 ldx [%sp + SPOFF + CCFSZ + TF_WSTATE], %l6 /* * Disable interrupts to restore the special globals. They are not * saved and restored for all kernel traps, so an interrupt at the * wrong time would clobber them. */ wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 /* * Switch to alternate globals. This frees up some registers we * can use after the restore changes our window. */ wrpr %g0, PSTATE_ALT, %pstate /* * Drop %pil to zero. It must have been zero at the time of the * trap, since we were in usermode, but it was raised above in * order to check for asts atomically. We have interrupts disabled * so any interrupts will not be serviced until we complete the * return to usermode. */ wrpr %g0, 0, %pil /* * Save %fprs in an alternate global so it can be restored after the * restore instruction below. If we restore it before the restore, * and the restore traps we may run for a while with floating point * enabled in the kernel, which we want to avoid. */ mov %l0, %g1 /* * Restore %fsr and %gsr. These need floating point enabled in %fprs, * so we set it temporarily and then clear it. */ wr %g0, FPRS_FEF, %fprs ldx [%sp + SPOFF + CCFSZ + TF_FSR], %fsr wr %l1, 0, %gsr wr %g0, 0, %fprs /* * Restore program counters. This could be done after the restore * but we're out of alternate globals to store them in... */ wrpr %l2, 0, %tnpc wrpr %l3, 0, %tpc /* * Save %tstate in an alternate global and clear the %cwp field. %cwp * will be affected by the restore below and we need to make sure it * points to the current window at that time, not the window that was * active at the time of the trap. */ andn %l4, TSTATE_CWP_MASK, %g2 /* - * Save %y in an alternate global. + * Restore %y. Could also be below if we had more alternate globals. */ - mov %l5, %g4 + wr %l5, 0, %y /* * Setup %wstate for return. We need to restore the user window state * which we saved in wstate.other when we trapped. We also need to * set the transition bit so the restore will be handled specially * if it traps, use the xor feature of wrpr to do that. */ srlx %l6, WSTATE_OTHER_SHIFT, %g3 wrpr %g3, WSTATE_TRANSITION, %wstate /* * Setup window management registers for return. If not all user * windows were spilled in the kernel %otherwin will be non-zero, * so we need to transfer it to %canrestore to correctly restore * those windows. Otherwise everything gets set to zero and the * restore below will fill a window directly from the user stack. */ rdpr %otherwin, %o0 wrpr %o0, 0, %canrestore wrpr %g0, 0, %otherwin wrpr %o0, 0, %cleanwin /* * Now do the restore. If this instruction causes a fill trap which * fails to fill a window from the user stack, we will resume at * tl0_ret_fill_end and call back into the kernel. */ restore tl0_ret_fill: /* * We made it. We're back in the window that was active at the time * of the trap, and ready to return to usermode. */ /* * Restore %frps. This was saved in an alternate global above. */ wr %g1, 0, %fprs /* * Fixup %tstate so the saved %cwp points to the current window and * restore it. */ - rdpr %cwp, %g1 - wrpr %g2, %g1, %tstate + rdpr %cwp, %g4 + wrpr %g2, %g4, %tstate /* * Restore the user window state. The transition bit was set above * for special handling of the restore, this clears it. */ wrpr %g3, 0, %wstate #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" - , %g1, %g2, %g3, 7, 8, 9) - ldx [PCPU(CURTHREAD)], %g2 - stx %g2, [%g1 + KTR_PARM1] - rdpr %pil, %g2 - stx %g2, [%g1 + KTR_PARM2] - rdpr %tpc, %g2 - stx %g2, [%g1 + KTR_PARM3] - rdpr %tnpc, %g2 - stx %g2, [%g1 + KTR_PARM4] - stx %sp, [%g1 + KTR_PARM5] + , %g2, %g3, %g4, 7, 8, 9) + ldx [PCPU(CURTHREAD)], %g3 + stx %g3, [%g2 + KTR_PARM1] + rdpr %pil, %g3 + stx %g3, [%g2 + KTR_PARM2] + rdpr %tpc, %g3 + stx %g3, [%g2 + KTR_PARM3] + rdpr %tnpc, %g3 + stx %g3, [%g2 + KTR_PARM4] + stx %sp, [%g2 + KTR_PARM5] 9: #endif /* - * Restore %y. Note that the CATR above clobbered it. - */ - wr %g4, 0, %y - - /* * Return to usermode. */ retry tl0_ret_fill_end: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" , %l0, %l1, %l2, 7, 8, 9) rdpr %pstate, %l1 stx %l1, [%l0 + KTR_PARM1] stx %l6, [%l0 + KTR_PARM2] stx %sp, [%l0 + KTR_PARM3] 9: - - /* - * Restore %y clobbered by the CATR. This was saved in %l5 above. - */ - wr %l5, 0, %y #endif /* * The restore above caused a fill trap and the fill handler was * unable to fill a window from the user stack. The special fill * handler recognized this and punted, sending us here. We need * to carefully undo any state that was restored before the restore * was executed and call trap again. Trap will copyin a window * from the user stack which will fault in the page we need so the * restore above will succeed when we try again. If this fails * the process has trashed its stack, so we kill it. */ /* * Restore the kernel window state. This was saved in %l6 above, and * since the restore failed we're back in the same window. */ wrpr %l6, 0, %wstate /* * Restore the normal globals which have predefined values in the * kernel. We clobbered them above restoring the user's globals * so this is very important. * XXX PSTATE_ALT must already be set. */ wrpr %g0, PSTATE_ALT, %pstate mov PCB_REG, %o0 mov PCPU_REG, %o1 wrpr %g0, PSTATE_NORMAL, %pstate mov %o0, PCB_REG mov %o1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate /* * Simulate a fill trap and then start the whole return sequence over * again. This is special because it only copies in 1 window, not 2 * as we would for a normal failed fill. This may be the first time * the process has been run, so there may not be 2 windows worth of * stack to copyin. */ mov T_FILL_RET, %o0 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, SPOFF + CCFSZ, %o0 ba,a %xcc, tl0_ret nop END(tl0_ret) /* * Kernel trap entry point * * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * This is easy because the stack is already setup and the windows don't need * to be split. We build a trapframe and call trap(), the same as above, but * the outs don't need to be saved. */ ENTRY(tl1_trap) rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] set tl1_ret - 8, %o7 jmpl %o2, %g0 add %sp, CCFSZ + SPOFF, %o0 END(tl1_trap) ENTRY(tl1_ret) ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldx [%sp + SPOFF + CCFSZ + TF_PIL], %l3 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 set VM_MIN_PROM_ADDRESS, %l5 cmp %l1, %l5 bl,a,pt %xcc, 1f nop set VM_MAX_PROM_ADDRESS, %l5 cmp %l1, %l5 bg,a,pt %xcc, 1f nop wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 1: wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 - mov %l4, %g4 wrpr %l3, 0, %pil + wr %l4, 0, %y restore wrpr %g0, 2, %tl + rdpr %cwp, %g4 + wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc - rdpr %cwp, %g2 - wrpr %g1, %g2, %tstate #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_ret: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" - , %g1, %g2, %g3, 7, 8, 9) - ldx [PCPU(CURTHREAD)], %g2 - stx %g2, [%g1 + KTR_PARM1] - rdpr %pil, %g2 - stx %g2, [%g1 + KTR_PARM2] - rdpr %tstate, %g2 - stx %g2, [%g1 + KTR_PARM3] - rdpr %tpc, %g2 - stx %g2, [%g1 + KTR_PARM4] - stx %sp, [%g1 + KTR_PARM5] + , %g2, %g3, %g4, 7, 8, 9) + ldx [PCPU(CURTHREAD)], %g3 + stx %g3, [%g2 + KTR_PARM1] + rdpr %pil, %g3 + stx %g3, [%g2 + KTR_PARM2] + rdpr %tstate, %g3 + stx %g3, [%g2 + KTR_PARM3] + rdpr %tpc, %g3 + stx %g3, [%g2 + KTR_PARM4] + stx %sp, [%g2 + KTR_PARM5] 9: #endif - wr %g4, 0, %y - retry END(tl1_ret) /* * void tl1_intr(u_int level, u_int mask) */ ENTRY(tl1_intr) rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%p level=%#x pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %clear_softint wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov %o0, %l7 mov T_INTERRUPT | T_KERNEL, %o1 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate SET(intr_handlers, %l5, %l4) sllx %l7, IH_SHIFT, %l5 ldx [%l4 + %l5], %l5 KASSERT(%l5, "tl1_intr: ih null") call %l5 add %sp, CCFSZ + SPOFF, %o0 /* %l7 contains PIL */ SET(intrcnt, %l5, %l4) prefetcha [%l4] ASI_N, 1 SET(pil_countp, %l5, %l6) sllx %l7, 1, %l5 lduh [%l5 + %l6], %l5 sllx %l5, 3, %l5 add %l5, %l4, %l4 ldx [%l4], %l5 inc %l5 stx %l5, [%l4] lduw [PCPU(CNT) + V_INTR], %l4 inc %l4 stw %l4, [PCPU(CNT) + V_INTR] ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 - mov %l4, %g4 wrpr %l3, 0, %pil + wr %l4, 0, %y restore wrpr %g0, 2, %tl + rdpr %cwp, %g4 + wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc - rdpr %cwp, %g2 - wrpr %g1, %g2, %tstate #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%#x pil=%#lx ts=%#lx pc=%#lx sp=%#lx" - , %g1, %g2, %g3, 7, 8, 9) - ldx [PCPU(CURTHREAD)], %g2 - stx %g2, [%g1 + KTR_PARM1] - rdpr %pil, %g2 - stx %g2, [%g1 + KTR_PARM2] - rdpr %tstate, %g2 - stx %g2, [%g1 + KTR_PARM3] - rdpr %tpc, %g2 - stx %g2, [%g1 + KTR_PARM4] - stx %sp, [%g1 + KTR_PARM5] + , %g2, %g3, %g4, 7, 8, 9) + ldx [PCPU(CURTHREAD)], %g3 + stx %g3, [%g2 + KTR_PARM1] + rdpr %pil, %g3 + stx %g3, [%g2 + KTR_PARM2] + rdpr %tstate, %g3 + stx %g3, [%g2 + KTR_PARM3] + rdpr %tpc, %g3 + stx %g3, [%g2 + KTR_PARM4] + stx %sp, [%g2 + KTR_PARM5] 9: #endif - - wr %g4, 0, %y retry END(tl1_intr) .globl tl_text_end tl_text_end: nop /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move * them to the outs. */ ENTRY(fork_trampoline) #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %cwp, %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif mov %l0, %o0 mov %l1, %o1 call fork_exit mov %l2, %o2 ba,a %xcc, tl0_ret nop END(fork_trampoline) Index: projects/clang380-import/sys/sparc64/sparc64/mp_exception.S =================================================================== --- projects/clang380-import/sys/sparc64/sparc64/mp_exception.S (revision 292950) +++ projects/clang380-import/sys/sparc64/sparc64/mp_exception.S (revision 292951) @@ -1,312 +1,309 @@ /*- * Copyright (c) 2002 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "assym.s" .register %g2, #ignore .register %g3, #ignore -#define IPI_DONE(r1, r2, r3, r4, r5, r6) \ - rd %y, r6 ; \ +#define IPI_DONE(r1, r2, r3, r4, r5) \ lduw [PCPU(CPUID)], r2 ; \ mov _NCPUBITS, r3 ; \ - mov %g0, %y ; \ - udiv r2, r3, r4 ; \ + udivx r2, r3, r4 ; \ srl r4, 0, r5 ; \ sllx r5, PTR_SHIFT, r5 ; \ add r1, r5, r1 ; \ smul r4, r3, r3 ; \ sub r2, r3, r3 ; \ mov 1, r4 ; \ sllx r4, r3, r4 ; \ - wr r6, %y ; \ ATOMIC_CLEAR_LONG(r1, r2, r3, r4) /* * Invalidate a physical page in the data cache. For UltraSPARC I and II. */ ENTRY(tl_ipi_spitfire_dcache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g6 srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 lduw [PCPU(CACHE) + DC_SIZE], %g3 lduw [PCPU(CACHE) + DC_LINESIZE], %g4 sub %g3, %g4, %g2 1: ldxa [%g2] ASI_DCACHE_TAG, %g1 srlx %g1, DC_VALID_SHIFT, %g3 andcc %g3, DC_VALID_MASK, %g0 bz,pt %xcc, 2f set DC_TAG_MASK, %g3 sllx %g3, DC_TAG_SHIFT, %g3 and %g1, %g3, %g1 cmp %g1, %g6 bne,a,pt %xcc, 2f nop stxa %g1, [%g2] ASI_DCACHE_TAG membar #Sync 2: brgz,pt %g2, 1b sub %g2, %g4, %g2 - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_spitfire_dcache_page_inval) /* * Invalidate a physical page in the instruction cache. For UltraSPARC I and * II. */ ENTRY(tl_ipi_spitfire_icache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g6 srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 lduw [PCPU(CACHE) + IC_SIZE], %g3 lduw [PCPU(CACHE) + IC_LINESIZE], %g4 sub %g3, %g4, %g2 1: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ srlx %g1, IC_VALID_SHIFT, %g3 andcc %g3, IC_VALID_MASK, %g0 bz,pt %xcc, 2f set IC_TAG_MASK, %g3 sllx %g3, IC_TAG_SHIFT, %g3 and %g1, %g3, %g1 cmp %g1, %g6 bne,a,pt %xcc, 2f nop stxa %g1, [%g2] ASI_ICACHE_TAG membar #Sync 2: brgz,pt %g2, 1b sub %g2, %g4, %g2 - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_spitfire_icache_page_inval) /* * Invalidate a physical page in the data cache. For UltraSPARC III. */ ENTRY(tl_ipi_cheetah_dcache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g1 set PAGE_SIZE, %g2 add %g1, %g2, %g3 lduw [PCPU(CACHE) + DC_LINESIZE], %g2 1: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE membar #Sync add %g1, %g2, %g1 cmp %g1, %g3 blt,a,pt %xcc, 1b nop - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_cheetah_dcache_page_inval) /* * Trigger a softint at the desired level. */ ENTRY(tl_ipi_level) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx" , %g1, %g2, %g3, 7, 8, 9) lduw [PCPU(CPUID)], %g2 stx %g2, [%g1 + KTR_PARM1] lduw [PCPU(MID)], %g2 stx %g2, [%g1 + KTR_PARM2] stx %g4, [%g1 + KTR_PARM3] stx %g5, [%g1 + KTR_PARM4] 9: #endif mov 1, %g1 sllx %g1, %g5, %g1 wr %g1, 0, %set_softint retry END(tl_ipi_level) /* * Demap a page from the dtlb and/or itlb. */ ENTRY(tl_ipi_tlb_page_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_VA], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif ldx [%g5 + ITA_PMAP], %g1 SET(kernel_pmap_store, %g3, %g2) mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 cmp %g1, %g2 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 ldx [%g5 + ITA_VA], %g2 or %g2, %g3, %g2 sethi %hi(KERNBASE), %g3 stxa %g0, [%g2] ASI_DMMU_DEMAP stxa %g0, [%g2] ASI_IMMU_DEMAP flush %g3 - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_page_demap) /* * Demap a range of pages from the dtlb and itlb. */ ENTRY(tl_ipi_tlb_range_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_START], %g2 stx %g2, [%g1 + KTR_PARM2] ldx [%g5 + ITA_END], %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif ldx [%g5 + ITA_PMAP], %g1 SET(kernel_pmap_store, %g3, %g2) mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 cmp %g1, %g2 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 ldx [%g5 + ITA_START], %g1 ldx [%g5 + ITA_END], %g2 sethi %hi(KERNBASE), %g6 1: or %g1, %g3, %g4 stxa %g0, [%g4] ASI_DMMU_DEMAP stxa %g0, [%g4] ASI_IMMU_DEMAP flush %g6 set PAGE_SIZE, %g6 add %g1, %g6, %g1 cmp %g1, %g2 blt,a,pt %xcc, 1b sethi %hi(KERNBASE), %g6 - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_range_demap) /* * Demap the primary context from the dtlb and itlb. */ ENTRY(tl_ipi_tlb_context_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_VA], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 sethi %hi(KERNBASE), %g3 stxa %g0, [%g1] ASI_DMMU_DEMAP stxa %g0, [%g1] ASI_IMMU_DEMAP flush %g3 - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_context_demap) /* * Read %stick. */ ENTRY(tl_ipi_stick_rd) ldx [%g5 + IRA_VAL], %g1 rd %asr24, %g2 stx %g2, [%g1] - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_stick_rd) /* * Read %tick. */ ENTRY(tl_ipi_tick_rd) ldx [%g5 + IRA_VAL], %g1 rd %tick, %g2 stx %g2, [%g1] - IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6) + IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tick_rd) Index: projects/clang380-import/sys/sparc64/sparc64/pmap.c =================================================================== --- projects/clang380-import/sys/sparc64/sparc64/pmap.c (revision 292950) +++ projects/clang380-import/sys/sparc64/sparc64/pmap.c (revision 292951) @@ -1,2347 +1,2354 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is also stored by the * logical address mapping module, this module may throw away valid virtual * to physical mappings at almost any time. However, invalidations of * mappings must be done as requested. * * In order to cope with hardware architectures which make virtual to * physical map invalidates expensive, this module may delay invalidate * reduced protection operations until such time as they are actually * necessary. This module is given full information as to which processors * are currently using which maps, and to when physical maps must be made * correct. */ #include "opt_kstack_pages.h" #include "opt_pmap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Virtual address of message buffer */ struct msgbuf *msgbufp; /* * Map of physical memory reagions */ vm_paddr_t phys_avail[128]; static struct ofw_mem_region mra[128]; struct ofw_mem_region sparc64_memreg[128]; int sparc64_nmemreg; static struct ofw_map translations[128]; static int translations_size; static vm_offset_t pmap_idle_map; static vm_offset_t pmap_temp_map_1; static vm_offset_t pmap_temp_map_2; /* * First and last available kernel virtual addresses */ vm_offset_t virtual_avail; vm_offset_t virtual_end; vm_offset_t kernel_vm_end; vm_offset_t vm_max_kernel_address; /* * Kernel pmap */ struct pmap kernel_pmap_store; struct rwlock_padalign tte_list_global_lock; /* * Allocate physical memory for use in pmap_bootstrap. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size, uint32_t colors); static void pmap_bootstrap_set_tte(struct tte *tp, u_long vpn, u_long data); static void pmap_cache_remove(vm_page_t m, vm_offset_t va); static int pmap_protect_tte(struct pmap *pm1, struct pmap *pm2, struct tte *tp, vm_offset_t va); static int pmap_unwire_tte(pmap_t pm, pmap_t pm2, struct tte *tp, vm_offset_t va); static void pmap_init_qpages(void); /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static int pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind); extern int tl1_dmmu_miss_direct_patch_tsb_phys_1[]; extern int tl1_dmmu_miss_direct_patch_tsb_phys_end_1[]; extern int tl1_dmmu_miss_patch_asi_1[]; extern int tl1_dmmu_miss_patch_quad_ldd_1[]; extern int tl1_dmmu_miss_patch_tsb_1[]; extern int tl1_dmmu_miss_patch_tsb_2[]; extern int tl1_dmmu_miss_patch_tsb_mask_1[]; extern int tl1_dmmu_miss_patch_tsb_mask_2[]; extern int tl1_dmmu_prot_patch_asi_1[]; extern int tl1_dmmu_prot_patch_quad_ldd_1[]; extern int tl1_dmmu_prot_patch_tsb_1[]; extern int tl1_dmmu_prot_patch_tsb_2[]; extern int tl1_dmmu_prot_patch_tsb_mask_1[]; extern int tl1_dmmu_prot_patch_tsb_mask_2[]; extern int tl1_immu_miss_patch_asi_1[]; extern int tl1_immu_miss_patch_quad_ldd_1[]; extern int tl1_immu_miss_patch_tsb_1[]; extern int tl1_immu_miss_patch_tsb_2[]; extern int tl1_immu_miss_patch_tsb_mask_1[]; extern int tl1_immu_miss_patch_tsb_mask_2[]; /* * If user pmap is processed with pmap_remove and with pmap_remove and the * resident count drops to 0, there are no more pages to remove, so we * need not continue. */ #define PMAP_REMOVE_DONE(pm) \ ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) /* * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() * and pmap_protect() instead of trying each virtual address. */ #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); PMAP_STATS_VAR(pmap_nenter); PMAP_STATS_VAR(pmap_nenter_update); PMAP_STATS_VAR(pmap_nenter_replace); PMAP_STATS_VAR(pmap_nenter_new); PMAP_STATS_VAR(pmap_nkenter); PMAP_STATS_VAR(pmap_nkenter_oc); PMAP_STATS_VAR(pmap_nkenter_stupid); PMAP_STATS_VAR(pmap_nkremove); PMAP_STATS_VAR(pmap_nqenter); PMAP_STATS_VAR(pmap_nqremove); PMAP_STATS_VAR(pmap_ncache_enter); PMAP_STATS_VAR(pmap_ncache_enter_c); PMAP_STATS_VAR(pmap_ncache_enter_oc); PMAP_STATS_VAR(pmap_ncache_enter_cc); PMAP_STATS_VAR(pmap_ncache_enter_coc); PMAP_STATS_VAR(pmap_ncache_enter_nc); PMAP_STATS_VAR(pmap_ncache_enter_cnc); PMAP_STATS_VAR(pmap_ncache_remove); PMAP_STATS_VAR(pmap_ncache_remove_c); PMAP_STATS_VAR(pmap_ncache_remove_oc); PMAP_STATS_VAR(pmap_ncache_remove_cc); PMAP_STATS_VAR(pmap_ncache_remove_coc); PMAP_STATS_VAR(pmap_ncache_remove_nc); PMAP_STATS_VAR(pmap_nzero_page); PMAP_STATS_VAR(pmap_nzero_page_c); PMAP_STATS_VAR(pmap_nzero_page_oc); PMAP_STATS_VAR(pmap_nzero_page_nc); PMAP_STATS_VAR(pmap_nzero_page_area); PMAP_STATS_VAR(pmap_nzero_page_area_c); PMAP_STATS_VAR(pmap_nzero_page_area_oc); PMAP_STATS_VAR(pmap_nzero_page_area_nc); PMAP_STATS_VAR(pmap_nzero_page_idle); PMAP_STATS_VAR(pmap_nzero_page_idle_c); PMAP_STATS_VAR(pmap_nzero_page_idle_oc); PMAP_STATS_VAR(pmap_nzero_page_idle_nc); PMAP_STATS_VAR(pmap_ncopy_page); PMAP_STATS_VAR(pmap_ncopy_page_c); PMAP_STATS_VAR(pmap_ncopy_page_oc); PMAP_STATS_VAR(pmap_ncopy_page_nc); PMAP_STATS_VAR(pmap_ncopy_page_dc); PMAP_STATS_VAR(pmap_ncopy_page_doc); PMAP_STATS_VAR(pmap_ncopy_page_sc); PMAP_STATS_VAR(pmap_ncopy_page_soc); PMAP_STATS_VAR(pmap_nnew_thread); PMAP_STATS_VAR(pmap_nnew_thread_oc); static inline u_long dtlb_get_data(u_int tlb, u_int slot); /* * Quick sort callout for comparing memory regions */ static int mr_cmp(const void *a, const void *b); static int om_cmp(const void *a, const void *b); static int mr_cmp(const void *a, const void *b) { const struct ofw_mem_region *mra; const struct ofw_mem_region *mrb; mra = a; mrb = b; if (mra->mr_start < mrb->mr_start) return (-1); else if (mra->mr_start > mrb->mr_start) return (1); else return (0); } static int om_cmp(const void *a, const void *b) { const struct ofw_map *oma; const struct ofw_map *omb; oma = a; omb = b; if (oma->om_start < omb->om_start) return (-1); else if (oma->om_start > omb->om_start) return (1); else return (0); } static inline u_long dtlb_get_data(u_int tlb, u_int slot) { u_long data; register_t s; slot = TLB_DAR_SLOT(tlb, slot); /* * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to * work around errata of USIII and beyond. */ s = intr_disable(); (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); intr_restore(s); return (data); } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(u_int cpu_impl) { struct pmap *pm; struct tte *tp; vm_offset_t off; vm_offset_t va; vm_paddr_t pa; vm_size_t physsz; vm_size_t virtsz; u_long data; u_long vpn; phandle_t pmem; phandle_t vmem; u_int dtlb_slots_avail; int i; int j; int sz; uint32_t asi; uint32_t colors; uint32_t ldd; /* * Set the kernel context. */ pmap_set_kctx(); colors = dcache_color_ignore != 0 ? 1 : DCACHE_COLORS; /* * Find out what physical memory is available from the PROM and * initialize the phys_avail array. This must be done before * pmap_bootstrap_alloc is called. */ if ((pmem = OF_finddevice("/memory")) == -1) OF_panic("%s: finddevice /memory", __func__); if ((sz = OF_getproplen(pmem, "available")) == -1) OF_panic("%s: getproplen /memory/available", __func__); if (sizeof(phys_avail) < sz) OF_panic("%s: phys_avail too small", __func__); if (sizeof(mra) < sz) OF_panic("%s: mra too small", __func__); bzero(mra, sz); if (OF_getprop(pmem, "available", mra, sz) == -1) OF_panic("%s: getprop /memory/available", __func__); sz /= sizeof(*mra); - CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); +#ifdef DIAGNOSTIC + OF_printf("pmap_bootstrap: physical memory\n"); +#endif qsort(mra, sz, sizeof (*mra), mr_cmp); physsz = 0; getenv_quad("hw.physmem", &physmem); physmem = btoc(physmem); for (i = 0, j = 0; i < sz; i++, j += 2) { - CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, +#ifdef DIAGNOSTIC + OF_printf("start=%#lx size=%#lx\n", mra[i].mr_start, mra[i].mr_size); +#endif if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { if (btoc(physsz) < physmem) { phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + (ctob(physmem) - physsz); physsz = ctob(physmem); } break; } phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; physsz += mra[i].mr_size; } physmem = btoc(physsz); /* * Calculate the size of kernel virtual memory, and the size and mask * for the kernel TSB based on the phsyical memory size but limited * by the amount of dTLB slots available for locked entries if we have * to lock the TSB in the TLB (given that for spitfire-class CPUs all * of the dt64 slots can hold locked entries but there is no large * dTLB for unlocked ones, we don't use more than half of it for the * TSB). * Note that for reasons unknown OpenSolaris doesn't take advantage of * ASI_ATOMIC_QUAD_LDD_PHYS on UltraSPARC-III. However, given that no * public documentation is available for these, the latter just might * not support it, yet. */ if (cpu_impl == CPU_IMPL_SPARC64V || cpu_impl >= CPU_IMPL_ULTRASPARCIIIp) { tsb_kernel_ldd_phys = 1; virtsz = roundup(5 / 3 * physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); } else { dtlb_slots_avail = 0; for (i = 0; i < dtlb_slots; i++) { data = dtlb_get_data(cpu_impl == CPU_IMPL_ULTRASPARCIII ? TLB_DAR_T16 : TLB_DAR_T32, i); if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) dtlb_slots_avail++; } #ifdef SMP dtlb_slots_avail -= PCPU_PAGES; #endif if (cpu_impl >= CPU_IMPL_ULTRASPARCI && cpu_impl < CPU_IMPL_ULTRASPARCIII) dtlb_slots_avail /= 2; virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); virtsz = MIN(virtsz, (dtlb_slots_avail * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); } vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; /* * Allocate the kernel TSB and lock it in the TLB if necessary. */ pa = pmap_bootstrap_alloc(tsb_kernel_size, colors); if (pa & PAGE_MASK_4M) OF_panic("%s: TSB unaligned", __func__); tsb_kernel_phys = pa; if (tsb_kernel_ldd_phys == 0) { tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); pmap_map_tsb(); bzero(tsb_kernel, tsb_kernel_size); } else { tsb_kernel = (struct tte *)TLB_PHYS_TO_DIRECT(tsb_kernel_phys); aszero(ASI_PHYS_USE_EC, tsb_kernel_phys, tsb_kernel_size); } /* * Allocate and map the dynamic per-CPU area for the BSP. */ pa = pmap_bootstrap_alloc(DPCPU_SIZE, colors); dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); /* * Allocate and map the message buffer. */ pa = pmap_bootstrap_alloc(msgbufsize, colors); msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); /* * Patch the TSB addresses and mask as well as the ASIs used to load * it into the trap table. */ #define LDDA_R_I_R(rd, imm_asi, rs1, rs2) \ (EIF_OP(IOP_LDST) | EIF_F3_RD(rd) | EIF_F3_OP3(INS3_LDDA) | \ EIF_F3_RS1(rs1) | EIF_F3_I(0) | EIF_F3_IMM_ASI(imm_asi) | \ EIF_F3_RS2(rs2)) #define OR_R_I_R(rd, imm13, rs1) \ (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) #define SETHI(rd, imm22) \ (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ EIF_IMM((imm22) >> 10, 22)) #define WR_R_I(rd, imm13, rs1) \ (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_WR) | \ EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) #define PATCH_ASI(addr, asi) do { \ if (addr[0] != WR_R_I(IF_F3_RD(addr[0]), 0x0, \ IF_F3_RS1(addr[0]))) \ OF_panic("%s: patched instructions have changed", \ __func__); \ addr[0] |= EIF_IMM((asi), 13); \ flush(addr); \ } while (0) #define PATCH_LDD(addr, asi) do { \ if (addr[0] != LDDA_R_I_R(IF_F3_RD(addr[0]), 0x0, \ IF_F3_RS1(addr[0]), IF_F3_RS2(addr[0]))) \ OF_panic("%s: patched instructions have changed", \ __func__); \ addr[0] |= EIF_F3_IMM_ASI(asi); \ flush(addr); \ } while (0) #define PATCH_TSB(addr, val) do { \ if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, \ IF_F3_RS1(addr[1])) || \ addr[3] != SETHI(IF_F2_RD(addr[3]), 0x0)) \ OF_panic("%s: patched instructions have changed", \ __func__); \ addr[0] |= EIF_IMM((val) >> 42, 22); \ addr[1] |= EIF_IMM((val) >> 32, 10); \ addr[3] |= EIF_IMM((val) >> 10, 22); \ flush(addr); \ flush(addr + 1); \ flush(addr + 3); \ } while (0) #define PATCH_TSB_MASK(addr, val) do { \ if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, \ IF_F3_RS1(addr[1]))) \ OF_panic("%s: patched instructions have changed", \ __func__); \ addr[0] |= EIF_IMM((val) >> 10, 22); \ addr[1] |= EIF_IMM((val), 10); \ flush(addr); \ flush(addr + 1); \ } while (0) if (tsb_kernel_ldd_phys == 0) { asi = ASI_N; ldd = ASI_NUCLEUS_QUAD_LDD; off = (vm_offset_t)tsb_kernel; } else { asi = ASI_PHYS_USE_EC; ldd = ASI_ATOMIC_QUAD_LDD_PHYS; off = (vm_offset_t)tsb_kernel_phys; } PATCH_TSB(tl1_dmmu_miss_direct_patch_tsb_phys_1, tsb_kernel_phys); PATCH_TSB(tl1_dmmu_miss_direct_patch_tsb_phys_end_1, tsb_kernel_phys + tsb_kernel_size - 1); PATCH_ASI(tl1_dmmu_miss_patch_asi_1, asi); PATCH_LDD(tl1_dmmu_miss_patch_quad_ldd_1, ldd); PATCH_TSB(tl1_dmmu_miss_patch_tsb_1, off); PATCH_TSB(tl1_dmmu_miss_patch_tsb_2, off); PATCH_TSB_MASK(tl1_dmmu_miss_patch_tsb_mask_1, tsb_kernel_mask); PATCH_TSB_MASK(tl1_dmmu_miss_patch_tsb_mask_2, tsb_kernel_mask); PATCH_ASI(tl1_dmmu_prot_patch_asi_1, asi); PATCH_LDD(tl1_dmmu_prot_patch_quad_ldd_1, ldd); PATCH_TSB(tl1_dmmu_prot_patch_tsb_1, off); PATCH_TSB(tl1_dmmu_prot_patch_tsb_2, off); PATCH_TSB_MASK(tl1_dmmu_prot_patch_tsb_mask_1, tsb_kernel_mask); PATCH_TSB_MASK(tl1_dmmu_prot_patch_tsb_mask_2, tsb_kernel_mask); PATCH_ASI(tl1_immu_miss_patch_asi_1, asi); PATCH_LDD(tl1_immu_miss_patch_quad_ldd_1, ldd); PATCH_TSB(tl1_immu_miss_patch_tsb_1, off); PATCH_TSB(tl1_immu_miss_patch_tsb_2, off); PATCH_TSB_MASK(tl1_immu_miss_patch_tsb_mask_1, tsb_kernel_mask); PATCH_TSB_MASK(tl1_immu_miss_patch_tsb_mask_2, tsb_kernel_mask); /* * Enter fake 8k pages for the 4MB kernel pages, so that * pmap_kextract() will work for them. */ for (i = 0; i < kernel_tlb_slots; i++) { pa = kernel_tlbs[i].te_pa; va = kernel_tlbs[i].te_va; for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { tp = tsb_kvtotte(va + off); vpn = TV_VPN(va + off, TS_8K); data = TD_V | TD_8K | TD_PA(pa + off) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; pmap_bootstrap_set_tte(tp, vpn, data); } } /* * Set the start and end of KVA. The kernel is loaded starting * at the first available 4MB super page, so we advance to the * end of the last one used for it. */ virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; virtual_end = vm_max_kernel_address; kernel_vm_end = vm_max_kernel_address; /* * Allocate kva space for temporary mappings. */ pmap_idle_map = virtual_avail; virtual_avail += PAGE_SIZE * colors; pmap_temp_map_1 = virtual_avail; virtual_avail += PAGE_SIZE * colors; pmap_temp_map_2 = virtual_avail; virtual_avail += PAGE_SIZE * colors; /* * Allocate a kernel stack with guard page for thread0 and map it * into the kernel TSB. We must ensure that the virtual address is * colored properly for corresponding CPUs, since we're allocating * from phys_avail so the memory won't have an associated vm_page_t. */ pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, colors); kstack0_phys = pa; virtual_avail += roundup(KSTACK_GUARD_PAGES, colors) * PAGE_SIZE; kstack0 = virtual_avail; virtual_avail += roundup(KSTACK_PAGES, colors) * PAGE_SIZE; if (dcache_color_ignore == 0) KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), ("pmap_bootstrap: kstack0 miscolored")); for (i = 0; i < KSTACK_PAGES; i++) { pa = kstack0_phys + i * PAGE_SIZE; va = kstack0 + i * PAGE_SIZE; tp = tsb_kvtotte(va); vpn = TV_VPN(va, TS_8K); data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; pmap_bootstrap_set_tte(tp, vpn, data); } /* * Calculate the last available physical address. */ for (i = 0; phys_avail[i + 2] != 0; i += 2) ; Maxmem = sparc64_btop(phys_avail[i + 1]); /* * Add the PROM mappings to the kernel TSB. */ if ((vmem = OF_finddevice("/virtual-memory")) == -1) OF_panic("%s: finddevice /virtual-memory", __func__); if ((sz = OF_getproplen(vmem, "translations")) == -1) OF_panic("%s: getproplen translations", __func__); if (sizeof(translations) < sz) OF_panic("%s: translations too small", __func__); bzero(translations, sz); if (OF_getprop(vmem, "translations", translations, sz) == -1) OF_panic("%s: getprop /virtual-memory/translations", __func__); sz /= sizeof(*translations); translations_size = sz; - CTR0(KTR_PMAP, "pmap_bootstrap: translations"); +#ifdef DIAGNOSTIC + OF_printf("pmap_bootstrap: translations\n"); +#endif qsort(translations, sz, sizeof (*translations), om_cmp); for (i = 0; i < sz; i++) { - CTR3(KTR_PMAP, - "translation: start=%#lx size=%#lx tte=%#lx", +#ifdef DIAGNOSTIC + OF_printf("translation: start=%#lx size=%#lx tte=%#lx\n", translations[i].om_start, translations[i].om_size, translations[i].om_tte); +#endif if ((translations[i].om_tte & TD_V) == 0) continue; if (translations[i].om_start < VM_MIN_PROM_ADDRESS || translations[i].om_start > VM_MAX_PROM_ADDRESS) continue; for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { va = translations[i].om_start + off; tp = tsb_kvtotte(va); vpn = TV_VPN(va, TS_8K); data = ((translations[i].om_tte & ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | (cpu_impl >= CPU_IMPL_ULTRASPARCI && cpu_impl < CPU_IMPL_ULTRASPARCIII ? (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + off; pmap_bootstrap_set_tte(tp, vpn, data); } } /* * Get the available physical memory ranges from /memory/reg. These * are only used for kernel dumps, but it may not be wise to do PROM * calls in that situation. */ if ((sz = OF_getproplen(pmem, "reg")) == -1) OF_panic("%s: getproplen /memory/reg", __func__); if (sizeof(sparc64_memreg) < sz) OF_panic("%s: sparc64_memreg too small", __func__); if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) OF_panic("%s: getprop /memory/reg", __func__); sparc64_nmemreg = sz / sizeof(*sparc64_memreg); /* * Initialize the kernel pmap (which is statically allocated). */ pm = kernel_pmap; PMAP_LOCK_INIT(pm); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = TLB_CTX_KERNEL; CPU_FILL(&pm->pm_active); /* * Initialize the global tte list lock, which is more commonly * known as the pmap pv global lock. */ rw_init(&tte_list_global_lock, "pmap pv global"); /* * Flush all non-locked TLB entries possibly left over by the * firmware. */ tlb_flush_nonlocked(); } static void pmap_init_qpages(void) { struct pcpu *pc; int i; if (dcache_color_ignore != 0) return; CPU_FOREACH(i) { pc = pcpu_find(i); pc->pc_qmap_addr = kva_alloc(PAGE_SIZE * DCACHE_COLORS); if (pc->pc_qmap_addr == 0) panic("pmap_init_qpages: unable to allocate KVA"); } } SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL); /* * Map the 4MB kernel TSB pages. */ void pmap_map_tsb(void) { vm_offset_t va; vm_paddr_t pa; u_long data; int i; for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { va = (vm_offset_t)tsb_kernel + i; pa = tsb_kernel_phys + i; data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); } } /* * Set the secondary context to be the kernel context (needed for FP block * operations in the kernel). */ void pmap_set_kctx(void) { stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & TLB_CXR_PGSZ_MASK) | TLB_CTX_KERNEL); flush(KERNBASE); } /* * Allocate a physical page of memory directly from the phys_avail map. * Can only be called from pmap_bootstrap before avail start and end are * calculated. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size, uint32_t colors) { vm_paddr_t pa; int i; size = roundup(size, PAGE_SIZE * colors); for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (phys_avail[i + 1] - phys_avail[i] < size) continue; pa = phys_avail[i]; phys_avail[i] += size; return (pa); } OF_panic("%s: no suitable region found", __func__); } /* * Set a TTE. This function is intended as a helper when tsb_kernel is * direct-mapped but we haven't taken over the trap table, yet, as it's the * case when we are taking advantage of ASI_ATOMIC_QUAD_LDD_PHYS to access * the kernel TSB. */ void pmap_bootstrap_set_tte(struct tte *tp, u_long vpn, u_long data) { if (tsb_kernel_ldd_phys == 0) { tp->tte_vpn = vpn; tp->tte_data = data; } else { stxa((vm_paddr_t)tp + offsetof(struct tte, tte_vpn), ASI_PHYS_USE_EC, vpn); stxa((vm_paddr_t)tp + offsetof(struct tte, tte_data), ASI_PHYS_USE_EC, data); } } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.tte_list); m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); m->md.pmap = NULL; } /* * Initialize the pmap module. */ void pmap_init(void) { vm_offset_t addr; vm_size_t size; int result; int i; for (i = 0; i < translations_size; i++) { addr = translations[i].om_start; size = translations[i].om_size; if ((translations[i].om_tte & TD_V) == 0) continue; if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) continue; result = vm_map_find(kernel_map, NULL, 0, &addr, size, 0, VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); if (result != KERN_SUCCESS || addr != translations[i].om_start) panic("pmap_init: vm_map_find"); } } /* * Extract the physical page address associated with the given * map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pm, vm_offset_t va) { struct tte *tp; vm_paddr_t pa; if (pm == kernel_pmap) return (pmap_kextract(va)); PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); if (tp == NULL) pa = 0; else pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); PMAP_UNLOCK(pm); return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. */ vm_page_t pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) { struct tte *tp; vm_page_t m; vm_paddr_t pa; m = NULL; pa = 0; PMAP_LOCK(pm); retry: if (pm == kernel_pmap) { if (va >= VM_MIN_DIRECT_ADDRESS) { tp = NULL; m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); (void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va), &pa); vm_page_hold(m); } else { tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) tp = NULL; } } else tp = tsb_tte_lookup(pm, va); if (tp != NULL && ((tp->tte_data & TD_SW) || (prot & VM_PROT_WRITE) == 0)) { if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa)) goto retry; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); vm_page_hold(m); } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pm); return (m); } /* * Extract the physical page address associated with the given kernel virtual * address. */ vm_paddr_t pmap_kextract(vm_offset_t va) { struct tte *tp; if (va >= VM_MIN_DIRECT_ADDRESS) return (TLB_DIRECT_TO_PHYS(va)); tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) return (0); return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); } int pmap_cache_enter(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; rw_assert(&tte_list_global_lock, RA_WLOCKED); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_enter: fake page")); PMAP_STATS_INC(pmap_ncache_enter); if (dcache_color_ignore != 0) return (1); /* * Find the color for this virtual address and note the added mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]++; /* * If all existing mappings have the same color, the mapping is * cacheable. */ if (m->md.color == color) { KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, ("pmap_cache_enter: cacheable, mappings of other color")); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_c); else PMAP_STATS_INC(pmap_ncache_enter_oc); return (1); } /* * If there are no mappings of the other color, and the page still has * the wrong color, this must be a new mapping. Change the color to * match the new mapping, which is cacheable. We must flush the page * from the cache now. */ if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { KASSERT(m->md.colors[color] == 1, ("pmap_cache_enter: changing color, not new mapping")); dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = color; if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_cc); else PMAP_STATS_INC(pmap_ncache_enter_coc); return (1); } /* * If the mapping is already non-cacheable, just return. */ if (m->md.color == -1) { PMAP_STATS_INC(pmap_ncache_enter_nc); return (0); } PMAP_STATS_INC(pmap_ncache_enter_cnc); /* * Mark all mappings as uncacheable, flush any lines with the other * color out of the dcache, and set the color to none (-1). */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_clear_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = -1; return (0); } static void pmap_cache_remove(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; rw_assert(&tte_list_global_lock, RA_WLOCKED); CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, m->md.colors[DCACHE_COLOR(va)]); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_remove: fake page")); PMAP_STATS_INC(pmap_ncache_remove); if (dcache_color_ignore != 0) return; KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, ("pmap_cache_remove: no mappings %d <= 0", m->md.colors[DCACHE_COLOR(va)])); /* * Find the color for this virtual address and note the removal of * the mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]--; /* * If the page is cacheable, just return and keep the same color, even * if there are no longer any mappings. */ if (m->md.color != -1) { if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_c); else PMAP_STATS_INC(pmap_ncache_remove_oc); return; } KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, ("pmap_cache_remove: uncacheable, no mappings of other color")); /* * If the page is not cacheable (color is -1), and the number of * mappings for this color is not zero, just return. There are * mappings of the other color still, so remain non-cacheable. */ if (m->md.colors[color] != 0) { PMAP_STATS_INC(pmap_ncache_remove_nc); return; } /* * The number of mappings for this color is now zero. Recache the * other colored mappings, and change the page color to the other * color. There should be no lines in the data cache for this page, * so flushing should not be needed. */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_set_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } m->md.color = DCACHE_OTHER_COLOR(color); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_cc); else PMAP_STATS_INC(pmap_ncache_remove_coc); } /* * Map a wired page into kernel virtual address space. */ void pmap_kenter(vm_offset_t va, vm_page_t m) { vm_offset_t ova; struct tte *tp; vm_page_t om; u_long data; rw_assert(&tte_list_global_lock, RA_WLOCKED); PMAP_STATS_INC(pmap_nkenter); tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { CTR5(KTR_SPARE2, "pmap_kenter: off color va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", va, VM_PAGE_TO_PHYS(m), m->object, m->object ? m->object->type : -1, m->pindex); PMAP_STATS_INC(pmap_nkenter_oc); } if ((tp->tte_data & TD_V) != 0) { om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); ova = TTE_GET_VA(tp); if (m == om && va == ova) { PMAP_STATS_INC(pmap_nkenter_stupid); return; } TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); pmap_cache_remove(om, ova); if (va != ova) tlb_page_demap(kernel_pmap, ova); } data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; if (pmap_cache_enter(m, va) != 0) data |= TD_CV; tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = data; TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); } /* * Map a wired page into kernel virtual address space. This additionally * takes a flag argument which is or'ed to the TTE data. This is used by * sparc64_bus_mem_map(). * NOTE: if the mapping is non-cacheable, it's the caller's responsibility * to flush entries that might still be in the cache, if applicable. */ void pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) { struct tte *tp; tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", va, pa, tp, tp->tte_data); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; } /* * Remove a wired page from kernel virtual address space. */ void pmap_kremove(vm_offset_t va) { struct tte *tp; vm_page_t m; rw_assert(&tte_list_global_lock, RA_WLOCKED); PMAP_STATS_INC(pmap_nkremove); tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); if ((tp->tte_data & TD_V) == 0) return; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pmap_cache_remove(m, va); TTE_ZERO(tp); } /* * Inverse of pmap_kenter_flags, used by bus_space_unmap(). */ void pmap_kremove_flags(vm_offset_t va) { struct tte *tp; tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); TTE_ZERO(tp); } /* * Map a range of physical addresses into kernel virtual address space. * * The value passed in *virt is a suggested virtual address for the mapping. * Architectures which can support a direct-mapped physical to virtual region * can return the appropriate address within that region, leaving '*virt' * unchanged. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return (TLB_PHYS_TO_DIRECT(start)); } /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ void pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) { vm_offset_t va; PMAP_STATS_INC(pmap_nqenter); va = sva; rw_wlock(&tte_list_global_lock); while (count-- > 0) { pmap_kenter(va, *m); va += PAGE_SIZE; m++; } rw_wunlock(&tte_list_global_lock); tlb_range_demap(kernel_pmap, sva, va); } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by pmap_qenter. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; PMAP_STATS_INC(pmap_nqremove); va = sva; rw_wlock(&tte_list_global_lock); while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } rw_wunlock(&tte_list_global_lock); tlb_range_demap(kernel_pmap, sva, va); } /* * Initialize the pmap associated with process 0. */ void pmap_pinit0(pmap_t pm) { int i; PMAP_LOCK_INIT(pm); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = TLB_CTX_KERNEL; CPU_ZERO(&pm->pm_active); pm->pm_tsb = NULL; pm->pm_tsb_obj = NULL; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } /* * Initialize a preallocated and zeroed pmap structure, such as one in a * vmspace structure. */ int pmap_pinit(pmap_t pm) { vm_page_t ma[TSB_PAGES]; vm_page_t m; int i; /* * Allocate KVA space for the TSB. */ if (pm->pm_tsb == NULL) { pm->pm_tsb = (struct tte *)kva_alloc(TSB_BSIZE); if (pm->pm_tsb == NULL) return (0); } /* * Allocate an object for it. */ if (pm->pm_tsb_obj == NULL) pm->pm_tsb_obj = vm_object_allocate(OBJT_PHYS, TSB_PAGES); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = -1; CPU_ZERO(&pm->pm_active); VM_OBJECT_WLOCK(pm->pm_tsb_obj); for (i = 0; i < TSB_PAGES; i++) { m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); m->valid = VM_PAGE_BITS_ALL; m->md.pmap = pm; ma[i] = m; } VM_OBJECT_WUNLOCK(pm->pm_tsb_obj); pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); bzero(&pm->pm_stats, sizeof(pm->pm_stats)); return (1); } /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pm) { vm_object_t obj; vm_page_t m; #ifdef SMP struct pcpu *pc; #endif CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", pm->pm_context[curcpu], pm->pm_tsb); KASSERT(pmap_resident_count(pm) == 0, ("pmap_release: resident pages %ld != 0", pmap_resident_count(pm))); /* * After the pmap was freed, it might be reallocated to a new process. * When switching, this might lead us to wrongly assume that we need * not switch contexts because old and new pmap pointer are equal. * Therefore, make sure that this pmap is not referenced by any PCPU * pointer any more. This could happen in two cases: * - A process that referenced the pmap is currently exiting on a CPU. * However, it is guaranteed to not switch in any more after setting * its state to PRS_ZOMBIE. * - A process that referenced this pmap ran on a CPU, but we switched * to a kernel thread, leaving the pmap pointer unchanged. */ #ifdef SMP sched_pin(); STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) atomic_cmpset_rel_ptr((uintptr_t *)&pc->pc_pmap, (uintptr_t)pm, (uintptr_t)NULL); sched_unpin(); #else critical_enter(); if (PCPU_GET(pmap) == pm) PCPU_SET(pmap, NULL); critical_exit(); #endif pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); obj = pm->pm_tsb_obj; VM_OBJECT_WLOCK(obj); KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); while (!TAILQ_EMPTY(&obj->memq)) { m = TAILQ_FIRST(&obj->memq); m->md.pmap = NULL; m->wire_count--; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } VM_OBJECT_WUNLOCK(obj); } /* * Grow the number of kernel page table entries. Unneeded. */ void pmap_growkernel(vm_offset_t addr) { panic("pmap_growkernel: can't grow kernel"); } int pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { vm_page_t m; u_long data; rw_assert(&tte_list_global_lock, RA_WLOCKED); data = atomic_readandclear_long(&tp->tte_data); if ((data & TD_FAKE) == 0) { m = PHYS_TO_VM_PAGE(TD_PA(data)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); if ((data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((data & TD_PV) != 0) { if ((data & TD_W) != 0) vm_page_dirty(m); if ((data & TD_REF) != 0) vm_page_aflag_set(m, PGA_REFERENCED); if (TAILQ_EMPTY(&m->md.tte_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); pm->pm_stats.resident_count--; } pmap_cache_remove(m, va); } TTE_ZERO(tp); if (PMAP_REMOVE_DONE(pm)) return (0); return (1); } /* * Remove the given range of addresses from the specified map. */ void pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) { struct tte *tp; vm_offset_t va; CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", pm->pm_context[curcpu], start, end); if (PMAP_REMOVE_DONE(pm)) return; rw_wlock(&tte_list_global_lock); PMAP_LOCK(pm); if (end - start > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, start, end, pmap_remove_tte); tlb_context_demap(pm); } else { for (va = start; va < end; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL && !pmap_remove_tte(pm, NULL, tp, va)) break; tlb_range_demap(pm, start, end - 1); } PMAP_UNLOCK(pm); rw_wunlock(&tte_list_global_lock); } void pmap_remove_all(vm_page_t m) { struct pmap *pm; struct tte *tpn; struct tte *tp; vm_offset_t va; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); rw_wlock(&tte_list_global_lock); for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { tpn = TAILQ_NEXT(tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; pm = TTE_GET_PMAP(tp); va = TTE_GET_VA(tp); PMAP_LOCK(pm); if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((tp->tte_data & TD_REF) != 0) vm_page_aflag_set(m, PGA_REFERENCED); if ((tp->tte_data & TD_W) != 0) vm_page_dirty(m); tp->tte_data &= ~TD_V; tlb_page_demap(pm, va); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pm->pm_stats.resident_count--; pmap_cache_remove(m, va); TTE_ZERO(tp); PMAP_UNLOCK(pm); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&tte_list_global_lock); } static int pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { u_long data; vm_page_t m; PMAP_LOCK_ASSERT(pm, MA_OWNED); data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); if ((data & (TD_PV | TD_W)) == (TD_PV | TD_W)) { m = PHYS_TO_VM_PAGE(TD_PA(data)); vm_page_dirty(m); } return (1); } /* * Set the physical protection on the specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va; struct tte *tp; CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", pm->pm_context[curcpu], sva, eva, prot); if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) return; PMAP_LOCK(pm); if (eva - sva > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); tlb_context_demap(pm); } else { for (va = sva; va < eva; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL) pmap_protect_tte(pm, NULL, tp, va); tlb_range_demap(pm, sva, eva - 1); } PMAP_UNLOCK(pm); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. */ int pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind) { int rv; rw_wlock(&tte_list_global_lock); PMAP_LOCK(pm); rv = pmap_enter_locked(pm, va, m, prot, flags, psind); rw_wunlock(&tte_list_global_lock); PMAP_UNLOCK(pm); return (rv); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static int pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { struct tte *tp; vm_paddr_t pa; vm_page_t real; u_long data; boolean_t wired; rw_assert(&tte_list_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pm, MA_OWNED); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); PMAP_STATS_INC(pmap_nenter); pa = VM_PAGE_TO_PHYS(m); wired = (flags & PMAP_ENTER_WIRED) != 0; /* * If this is a fake page from the device_pager, but it covers actual * physical memory, convert to the real backing page. */ if ((m->flags & PG_FICTITIOUS) != 0) { real = vm_phys_paddr_to_vm_page(pa); if (real != NULL) m = real; } CTR6(KTR_PMAP, "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", pm->pm_context[curcpu], m, va, pa, prot, wired); /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. */ if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { CTR0(KTR_PMAP, "pmap_enter_locked: update"); PMAP_STATS_INC(pmap_nenter_update); /* * Wiring change, just update stats. */ if (wired) { if ((tp->tte_data & TD_WIRED) == 0) { tp->tte_data |= TD_WIRED; pm->pm_stats.wired_count++; } } else { if ((tp->tte_data & TD_WIRED) != 0) { tp->tte_data &= ~TD_WIRED; pm->pm_stats.wired_count--; } } /* * Save the old bits and clear the ones we're interested in. */ data = tp->tte_data; tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); /* * If we're turning off write permissions, sense modify status. */ if ((prot & VM_PROT_WRITE) != 0) { tp->tte_data |= TD_SW; if (wired) tp->tte_data |= TD_W; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_aflag_set(m, PGA_WRITEABLE); } else if ((data & TD_W) != 0) vm_page_dirty(m); /* * If we're turning on execute permissions, flush the icache. */ if ((prot & VM_PROT_EXECUTE) != 0) { if ((data & TD_EXEC) == 0) icache_page_inval(pa); tp->tte_data |= TD_EXEC; } /* * Delete the old mapping. */ tlb_page_demap(pm, TTE_GET_VA(tp)); } else { /* * If there is an existing mapping, but its for a different * physical address, delete the old mapping. */ if (tp != NULL) { CTR0(KTR_PMAP, "pmap_enter_locked: replace"); PMAP_STATS_INC(pmap_nenter_replace); pmap_remove_tte(pm, NULL, tp, va); tlb_page_demap(pm, va); } else { CTR0(KTR_PMAP, "pmap_enter_locked: new"); PMAP_STATS_INC(pmap_nenter_new); } /* * Now set up the data and install the new mapping. */ data = TD_V | TD_8K | TD_PA(pa); if (pm == kernel_pmap) data |= TD_P; if ((prot & VM_PROT_WRITE) != 0) { data |= TD_SW; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_aflag_set(m, PGA_WRITEABLE); } if (prot & VM_PROT_EXECUTE) { data |= TD_EXEC; icache_page_inval(pa); } /* * If its wired update stats. We also don't need reference or * modify tracking for wired mappings, so set the bits now. */ if (wired) { pm->pm_stats.wired_count++; data |= TD_REF | TD_WIRED; if ((prot & VM_PROT_WRITE) != 0) data |= TD_W; } tsb_tte_enter(pm, m, va, TS_8K, data); } return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); m = m_start; rw_wlock(&tte_list_global_lock); PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 0, 0); m = TAILQ_NEXT(m, listq); } rw_wunlock(&tte_list_global_lock); PMAP_UNLOCK(pm); } void pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&tte_list_global_lock); PMAP_LOCK(pm); pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 0, 0); rw_wunlock(&tte_list_global_lock); PMAP_UNLOCK(pm); } void pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } static int pmap_unwire_tte(pmap_t pm, pmap_t pm2, struct tte *tp, vm_offset_t va) { PMAP_LOCK_ASSERT(pm, MA_OWNED); if ((tp->tte_data & TD_WIRED) == 0) panic("pmap_unwire_tte: tp %p is missing TD_WIRED", tp); atomic_clear_long(&tp->tte_data, TD_WIRED); pm->pm_stats.wired_count--; return (1); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range must * have the wired attribute set. In contrast, invalid mappings cannot have * the wired attribute set, so they are ignored. * * The wired attribute of the translation table entry is not a hardware * feature, so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pm, vm_offset_t sva, vm_offset_t eva) { vm_offset_t va; struct tte *tp; PMAP_LOCK(pm); if (eva - sva > PMAP_TSB_THRESH) tsb_foreach(pm, NULL, sva, eva, pmap_unwire_tte); else { for (va = sva; va < eva; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL) pmap_unwire_tte(pm, NULL, tp, va); } PMAP_UNLOCK(pm); } static int pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) { vm_page_t m; u_long data; if ((tp->tte_data & TD_FAKE) != 0) return (1); if (tsb_tte_lookup(dst_pmap, va) == NULL) { data = tp->tte_data & ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); tsb_tte_enter(dst_pmap, m, va, TS_8K, data); } return (1); } void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct tte *tp; vm_offset_t va; if (dst_addr != src_addr) return; rw_wlock(&tte_list_global_lock); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } if (len > PMAP_TSB_THRESH) { tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, pmap_copy_tte); tlb_context_demap(dst_pmap); } else { for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) pmap_copy_tte(src_pmap, dst_pmap, tp, va); tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); } rw_wunlock(&tte_list_global_lock); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } void pmap_zero_page(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page: fake page")); PMAP_STATS_INC(pmap_nzero_page); pa = VM_PAGE_TO_PHYS(m); if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_area(vm_page_t m, int off, int size) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_area: fake page")); KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); PMAP_STATS_INC(pmap_nzero_page_area); pa = VM_PAGE_TO_PHYS(m); if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_area_c); va = TLB_PHYS_TO_DIRECT(pa); bzero((void *)(va + off), size); } else if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_area_nc); aszero(ASI_PHYS_USE_EC, pa + off, size); } else { PMAP_STATS_INC(pmap_nzero_page_area_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); bzero((void *)(va + off), size); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_idle(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_idle: fake page")); PMAP_STATS_INC(pmap_nzero_page_idle); pa = VM_PAGE_TO_PHYS(m); if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_idle_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_idle_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_idle_oc); va = pmap_idle_map + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); } } void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t vdst; vm_offset_t vsrc; vm_paddr_t pdst; vm_paddr_t psrc; struct tte *tp; KASSERT((mdst->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake dst page")); KASSERT((msrc->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake src page")); PMAP_STATS_INC(pmap_ncopy_page); pdst = VM_PAGE_TO_PHYS(mdst); psrc = VM_PAGE_TO_PHYS(msrc); if (dcache_color_ignore != 0 || (msrc->md.color == DCACHE_COLOR(psrc) && mdst->md.color == DCACHE_COLOR(pdst))) { PMAP_STATS_INC(pmap_ncopy_page_c); vdst = TLB_PHYS_TO_DIRECT(pdst); vsrc = TLB_PHYS_TO_DIRECT(psrc); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); } else if (msrc->md.color == -1 && mdst->md.color == -1) { PMAP_STATS_INC(pmap_ncopy_page_nc); ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); } else if (msrc->md.color == -1) { if (mdst->md.color == DCACHE_COLOR(pdst)) { PMAP_STATS_INC(pmap_ncopy_page_dc); vdst = TLB_PHYS_TO_DIRECT(pdst); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_doc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); PMAP_UNLOCK(kernel_pmap); } } else if (mdst->md.color == -1) { if (msrc->md.color == DCACHE_COLOR(psrc)) { PMAP_STATS_INC(pmap_ncopy_page_sc); vsrc = TLB_PHYS_TO_DIRECT(psrc); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_soc); PMAP_LOCK(kernel_pmap); vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } else { PMAP_STATS_INC(pmap_ncopy_page_oc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { vm_paddr_t pa; vm_offset_t qaddr; struct tte *tp; pa = VM_PAGE_TO_PHYS(m); if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) return (TLB_PHYS_TO_DIRECT(pa)); critical_enter(); qaddr = PCPU_GET(qmap_addr); qaddr += (PAGE_SIZE * ((DCACHE_COLORS + DCACHE_COLOR(pa) - DCACHE_COLOR(qaddr)) % DCACHE_COLORS)); tp = tsb_kvtotte(qaddr); KASSERT(tp->tte_data == 0, ("pmap_quick_enter_page: PTE busy")); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(qaddr, TS_8K); return (qaddr); } void pmap_quick_remove_page(vm_offset_t addr) { vm_offset_t qaddr; struct tte *tp; if (addr >= VM_MIN_DIRECT_ADDRESS) return; tp = tsb_kvtotte(addr); qaddr = PCPU_GET(qmap_addr); KASSERT((addr >= qaddr) && (addr < (qaddr + (PAGE_SIZE * DCACHE_COLORS))), ("pmap_quick_remove_page: invalid address")); KASSERT(tp->tte_data != 0, ("pmap_quick_remove_page: PTE not in use")); stxa(TLB_DEMAP_VA(addr) | TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(addr) | TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, ASI_IMMU_DEMAP, 0); flush(KERNBASE); TTE_ZERO(tp); critical_exit(); } int unmapped_buf_allowed; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { panic("pmap_copy_pages: not implemented"); } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pm, vm_page_t m) { struct tte *tp; int loops; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); loops = 0; rv = FALSE; rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if (TTE_GET_PMAP(tp) == pm) { rv = TRUE; break; } if (++loops >= 16) break; } rw_wunlock(&tte_list_global_lock); return (rv); } /* * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct tte *tp; int count; count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) count++; rw_wunlock(&tte_list_global_lock); return (count); } /* * Remove all pages from specified address space, this aids process exit * speeds. This is much faster than pmap_remove in the case of running down * an entire address space. Only works for the current pmap. */ void pmap_remove_pages(pmap_t pm) { } /* * Returns TRUE if the given page has a managed mapping. */ boolean_t pmap_page_is_mapped(vm_page_t m) { struct tte *tp; boolean_t rv; rv = FALSE; if ((m->oflags & VPO_UNMANAGED) != 0) return (rv); rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & TD_PV) != 0) { rv = TRUE; break; } rw_wunlock(&tte_list_global_lock); return (rv); } /* * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct tte *tpf; struct tte *tpn; struct tte *tp; u_long data; int count; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); count = 0; rw_wlock(&tte_list_global_lock); if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { tpn = TAILQ_NEXT(tp, tte_link); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_REF); if ((data & TD_REF) != 0 && ++count > 4) break; } while ((tp = tpn) != NULL && tp != tpf); } rw_wunlock(&tte_list_global_lock); return (count); } boolean_t pmap_is_modified(vm_page_t m) { struct tte *tp; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); rv = FALSE; /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no TTEs can have TD_W set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (rv); rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_W) != 0) { rv = TRUE; break; } } rw_wunlock(&tte_list_global_lock); return (rv); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is elgible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { boolean_t rv; PMAP_LOCK(pmap); rv = tsb_tte_lookup(pmap, addr) == NULL; PMAP_UNLOCK(pmap); return (rv); } /* * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { struct tte *tp; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); rv = FALSE; rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_REF) != 0) { rv = TRUE; break; } } rw_wunlock(&tte_list_global_lock); return (rv); } /* * This function is advisory. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { } void pmap_clear_modify(vm_page_t m) { struct tte *tp; u_long data; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no TTEs can have TD_W set. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_W); if ((data & TD_W) != 0) tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } rw_wunlock(&tte_list_global_lock); } void pmap_remove_write(vm_page_t m) { struct tte *tp; u_long data; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&tte_list_global_lock); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); if ((data & TD_W) != 0) { vm_page_dirty(m); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&tte_list_global_lock); } int pmap_mincore(pmap_t pm, vm_offset_t addr, vm_paddr_t *locked_pa) { /* TODO; */ return (0); } /* * Activate a user pmap. The pmap must be activated before its address space * can be accessed in any way. */ void pmap_activate(struct thread *td) { struct vmspace *vm; struct pmap *pm; int context; critical_enter(); vm = td->td_proc->p_vmspace; pm = vmspace_pmap(vm); context = PCPU_GET(tlb_ctx); if (context == PCPU_GET(tlb_ctx_max)) { tlb_flush_user(); context = PCPU_GET(tlb_ctx_min); } PCPU_SET(tlb_ctx, context + 1); pm->pm_context[curcpu] = context; #ifdef SMP CPU_SET_ATOMIC(PCPU_GET(cpuid), &pm->pm_active); atomic_store_acq_ptr((uintptr_t *)PCPU_PTR(pmap), (uintptr_t)pm); #else CPU_SET(PCPU_GET(cpuid), &pm->pm_active); PCPU_SET(pmap, pm); #endif stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & TLB_CXR_PGSZ_MASK) | context); flush(KERNBASE); critical_exit(); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } Index: projects/clang380-import/sys/sparc64/sparc64/swtch.S =================================================================== --- projects/clang380-import/sys/sparc64/sparc64/swtch.S (revision 292950) +++ projects/clang380-import/sys/sparc64/sparc64/swtch.S (revision 292951) @@ -1,346 +1,344 @@ /*- * Copyright (c) 2001 Jake Burkholder. * Copyright (c) 2011 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include "assym.s" #include "opt_sched.h" .register %g2, #ignore .register %g3, #ignore /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) save %sp, -CCFSZ, %sp flushw ba %xcc, .Lsw1 mov %g0, %i2 END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) */ ENTRY(cpu_switch) save %sp, -CCFSZ, %sp /* * If the current thread was using floating point in the kernel, save * its context. The userland floating point context has already been * saved in that case. */ rd %fprs, %l2 andcc %l2, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop call savefpctx add PCB_REG, PCB_KFP, %o0 ba,a,pt %xcc, 2f nop /* * If the current thread was using floating point in userland, save * its context. */ 1: sub PCB_REG, TF_SIZEOF, %l2 ldx [%l2 + TF_FPRS], %l3 andcc %l3, FPRS_FEF, %g0 bz,a,pt %xcc, 2f nop call savefpctx add PCB_REG, PCB_UFP, %o0 andn %l3, FPRS_FEF, %l3 stx %l3, [%l2 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %l3 or %l3, PCB_FEF, %l3 stx %l3, [PCB_REG + PCB_FLAGS] /* * Flush the windows out to the stack and save the current frame * pointer and program counter. */ 2: flushw wrpr %g0, 0, %cleanwin stx %fp, [PCB_REG + PCB_SP] stx %i7, [PCB_REG + PCB_PC] /* * Load the new thread's frame pointer and program counter, and set * the current thread and pcb. */ .Lsw1: #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx" , %g1, %g2, %g3, 8, 9, 10) stx %i1, [%g1 + KTR_PARM1] ldx [%i1 + TD_PCB], %g2 ldx [%g2 + PCB_PC], %g3 stx %g3, [%g1 + KTR_PARM2] ldx [%g2 + PCB_SP], %g3 stx %g3, [%g1 + KTR_PARM3] 10: #endif ldx [%i1 + TD_PCB], %l0 stx %i1, [PCPU(CURTHREAD)] stx %l0, [PCPU(CURPCB)] wrpr %g0, PSTATE_NORMAL, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_ALT, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_KERNEL, %pstate ldx [PCB_REG + PCB_SP], %fp ldx [PCB_REG + PCB_PC], %i7 sub %fp, CCFSZ, %sp /* * Point to the pmaps of the new process, and of the last non-kernel * process to run. */ ldx [%i1 + TD_PROC], %l1 ldx [PCPU(PMAP)], %l2 ldx [%l1 + P_VMSPACE], %i5 add %i5, VM_PMAP, %l1 #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p" , %g1, %g2, %g3, 8, 9, 10) stx %l1, [%g1 + KTR_PARM1] stx %l2, [%g1 + KTR_PARM2] 10: #endif /* * If they are the same we are done. */ cmp %l2, %l1 be,a,pn %xcc, 8f nop /* * If the new process is a kernel thread we can just leave the old * context active and avoid recycling its context number. */ SET(vmspace0, %i4, %i3) cmp %i5, %i3 be,a,pn %xcc, 8f nop /* * If there was no non-kernel pmap, don't try to deactivate it. */ brz,pn %l2, 3f lduw [PCPU(CPUID)], %l3 /* * Mark the pmap of the last non-kernel vmspace to run as no longer * active on this CPU. */ mov _NCPUBITS, %l5 - mov %g0, %y - udiv %l3, %l5, %l6 + udivx %l3, %l5, %l6 srl %l6, 0, %l4 sllx %l4, PTR_SHIFT, %l4 add %l4, PM_ACTIVE, %l4 smul %l6, %l5, %l5 sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 #ifdef SMP add %l2, %l4, %l4 membar #LoadStore | #StoreStore ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5) #else ldx [%l2 + %l4], %l6 andn %l6, %l5, %l6 stx %l6, [%l2 + %l4] #endif /* * Take away its context number. */ sllx %l3, INT_SHIFT, %l3 add %l2, PM_CONTEXT, %l4 mov -1, %l5 stw %l5, [%l3 + %l4] 3: cmp %i2, %g0 be,pn %xcc, 4f add %i0, TD_LOCK, %l4 #if defined(SCHED_ULE) && defined(SMP) membar #LoadStore | #StoreStore ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) #else stx %i2, [%l4] #endif /* * Find a new TLB context. If we've run out we have to flush all * user mappings from the TLB and reset the context numbers. */ 4: lduw [PCPU(TLB_CTX)], %i3 lduw [PCPU(TLB_CTX_MAX)], %i4 cmp %i3, %i4 bne,a,pt %xcc, 5f nop SET(tlb_flush_user, %i5, %i4) ldx [%i4], %i5 call %i5 lduw [PCPU(TLB_CTX_MIN)], %i3 /* * Advance next free context. */ 5: add %i3, 1, %i4 stw %i4, [PCPU(TLB_CTX)] /* * Set the new context number in the pmap. */ lduw [PCPU(CPUID)], %l3 sllx %l3, INT_SHIFT, %i4 add %l1, PM_CONTEXT, %i5 stw %i3, [%i4 + %i5] /* * Mark the pmap as active on this CPU. */ mov _NCPUBITS, %l5 - mov %g0, %y - udiv %l3, %l5, %l6 + udivx %l3, %l5, %l6 srl %l6, 0, %l4 sllx %l4, PTR_SHIFT, %l4 add %l4, PM_ACTIVE, %l4 smul %l6, %l5, %l5 sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 #ifdef SMP add %l1, %l4, %l4 ATOMIC_SET_LONG(%l4, %l6, %l7, %l5) #else ldx [%l1 + %l4], %l6 or %l6, %l5, %l6 stx %l6, [%l1 + %l4] #endif /* * Make note of the change in pmap. */ #ifdef SMP PCPU_ADDR(PMAP, %l4) ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1) #else stx %l1, [PCPU(PMAP)] #endif /* * Fiddle the hardware bits. Set the TSB registers and install the * new context number in the CPU. */ ldx [%l1 + PM_TSB], %i4 mov AA_DMMU_TSB, %i5 stxa %i4, [%i5] ASI_DMMU mov AA_IMMU_TSB, %i5 stxa %i4, [%i5] ASI_IMMU setx TLB_CXR_PGSZ_MASK, %i5, %i4 mov AA_DMMU_PCXR, %i5 ldxa [%i5] ASI_DMMU, %l1 and %l1, %i4, %l1 or %i3, %l1, %i3 sethi %hi(KERNBASE), %i4 stxa %i3, [%i5] ASI_DMMU flush %i4 6: #if defined(SCHED_ULE) && defined(SMP) SET(blocked_lock, %l2, %l1) add %i1, TD_LOCK, %l2 7: ATOMIC_LOAD_LONG(%l2, %l3) cmp %l1, %l3 be,a,pn %xcc, 7b nop #endif /* * Done, return and load the new process's window from the stack. */ ret restore 8: cmp %i2, %g0 be,pn %xcc, 6b add %i0, TD_LOCK, %l4 #if defined(SCHED_ULE) && defined(SMP) membar #LoadStore | #StoreStore ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) ba,pt %xcc, 6b nop #else ba,pt %xcc, 6b stx %i2, [%l4] #endif END(cpu_switch) ENTRY(savectx) save %sp, -CCFSZ, %sp flushw call savefpctx add %i0, PCB_UFP, %o0 stx %fp, [%i0 + PCB_SP] stx %i7, [%i0 + PCB_PC] ret restore %g0, 0, %o0 END(savectx) /* * void savefpctx(uint32_t *); */ ENTRY(savefpctx) wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi stda %f0, [%o0 + (0 * 64)] %asi stda %f16, [%o0 + (1 * 64)] %asi stda %f32, [%o0 + (2 * 64)] %asi stda %f48, [%o0 + (3 * 64)] %asi membar #Sync retl wr %g0, 0, %fprs END(savefpctx) Index: projects/clang380-import/sys =================================================================== --- projects/clang380-import/sys (revision 292950) +++ projects/clang380-import/sys (revision 292951) Property changes on: projects/clang380-import/sys ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys:r292913-292950 Index: projects/clang380-import/usr.bin/clang/clang/CC.sh =================================================================== --- projects/clang380-import/usr.bin/clang/clang/CC.sh (nonexistent) +++ projects/clang380-import/usr.bin/clang/clang/CC.sh (revision 292951) @@ -0,0 +1,4 @@ +#!/bin/sh +# $FreeBSD$ +# This file is in the public domain. +exec /usr/bin/c++ "$@" Property changes on: projects/clang380-import/usr.bin/clang/clang/CC.sh ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/clang380-import/usr.bin/clang/clang/Makefile =================================================================== --- projects/clang380-import/usr.bin/clang/clang/Makefile (revision 292950) +++ projects/clang380-import/usr.bin/clang/clang/Makefile (revision 292951) @@ -1,126 +1,128 @@ # $FreeBSD$ .include PROG_CXX=clang SRCDIR= tools/clang/tools/driver SRCS= cc1_main.cpp \ cc1as_main.cpp \ driver.cpp .if ${MK_SHARED_TOOLCHAIN} == "no" NO_SHARED?= yes .endif LINKS= ${BINDIR}/clang ${BINDIR}/clang++ \ ${BINDIR}/clang ${BINDIR}/clang-cpp MLINKS= clang.1 clang++.1 \ clang.1 clang-cpp.1 .if ${MK_CLANG_IS_CC} != "no" +SCRIPTS=CC.sh +SCRIPTSNAME=CC + LINKS+= ${BINDIR}/clang ${BINDIR}/cc \ ${BINDIR}/clang ${BINDIR}/c++ \ - ${BINDIR}/clang ${BINDIR}/CC \ ${BINDIR}/clang ${BINDIR}/cpp MLINKS+= clang.1 cc.1 \ clang.1 c++.1 \ clang.1 CC.1 \ clang.1 cpp.1 .endif TGHDRS= DiagnosticCommonKinds \ DiagnosticDriverKinds \ DiagnosticFrontendKinds \ DiagnosticLexKinds \ DiagnosticSemaKinds \ Options .if ${MK_CLANG_FULL} != "no" _clangstaticanalyzer= \ clangstaticanalyzerfrontend \ clangstaticanalyzercheckers \ clangstaticanalyzercore _clangarcmigrate= \ clangarcmigrate .endif # MK_CLANG_FULL LIBDEPS=clangfrontendtool \ clangfrontend \ clangdriver \ clangserialization \ clangcodegen \ clangparse \ clangsema \ clangrewritefrontend \ clangrewrite \ ${_clangstaticanalyzer} \ ${_clangarcmigrate} \ clanganalysis \ clangedit \ clangast \ clanglex \ clangbasic \ llvmoption \ llvmobjcarcopts \ llvmlinker \ llvmirreader \ llvmipo \ llvmvectorize \ llvmbitwriter \ llvmasmparser \ llvmaarch64codegen \ llvmaarch64asmparser \ llvmaarch64desc \ llvmaarch64info \ llvmaarch64instprinter \ llvmaarch64utils \ llvmarmdisassembler \ llvmarmcodegen \ llvmarmasmparser \ llvmarmdesc \ llvmarminfo \ llvmarminstprinter \ llvmmipscodegen \ llvmmipsasmparser \ llvmmipsdesc \ llvmmipsinfo \ llvmmipsinstprinter \ llvmpowerpcdisassembler \ llvmpowerpccodegen \ llvmpowerpcasmparser \ llvmpowerpcdesc \ llvmpowerpcinfo \ llvmpowerpcinstprinter \ llvmsparccodegen \ llvmsparcasmparser \ llvmsparcdesc \ llvmsparcinfo \ llvmsparcinstprinter \ llvmx86asmparser \ llvmx86codegen \ llvmselectiondag \ llvmasmprinter \ llvmcodegen \ llvmtarget \ llvmscalaropts \ llvmprofiledata \ llvminstcombine \ llvminstrumentation \ llvmtransformutils \ llvmipa \ llvmanalysis \ llvmx86desc \ llvmobject \ llvmmcparser \ llvmbitreader \ llvmmcdisassembler \ llvmx86info \ llvmx86instprinter \ llvmmc \ llvmx86utils \ llvmcore \ llvmsupport LIBADD+= z .include "../clang.prog.mk" Index: projects/clang380-import =================================================================== --- projects/clang380-import (revision 292950) +++ projects/clang380-import (revision 292951) Property changes on: projects/clang380-import ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r292936-292950