Index: projects/import-googletest-1.8.1/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp (revision 345026) @@ -1,512 +1,518 @@ //==- TargetRegisterInfo.cpp - Target Register Information Implementation --==// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file implements the TargetRegisterInfo interface. // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Function.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Printable.h" #include "llvm/Support/raw_ostream.h" #include #include #define DEBUG_TYPE "target-reg-info" using namespace llvm; TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID, regclass_iterator RCB, regclass_iterator RCE, const char *const *SRINames, const LaneBitmask *SRILaneMasks, LaneBitmask SRICoveringLanes, const RegClassInfo *const RCIs, unsigned Mode) : InfoDesc(ID), SubRegIndexNames(SRINames), SubRegIndexLaneMasks(SRILaneMasks), RegClassBegin(RCB), RegClassEnd(RCE), CoveringLanes(SRICoveringLanes), RCInfos(RCIs), HwMode(Mode) { } TargetRegisterInfo::~TargetRegisterInfo() = default; void TargetRegisterInfo::markSuperRegs(BitVector &RegisterSet, unsigned Reg) const { for (MCSuperRegIterator AI(Reg, this, true); AI.isValid(); ++AI) RegisterSet.set(*AI); } bool TargetRegisterInfo::checkAllSuperRegsMarked(const BitVector &RegisterSet, ArrayRef Exceptions) const { // Check that all super registers of reserved regs are reserved as well. BitVector Checked(getNumRegs()); for (unsigned Reg : RegisterSet.set_bits()) { if (Checked[Reg]) continue; for (MCSuperRegIterator SR(Reg, this); SR.isValid(); ++SR) { if (!RegisterSet[*SR] && !is_contained(Exceptions, Reg)) { dbgs() << "Error: Super register " << printReg(*SR, this) << " of reserved register " << printReg(Reg, this) << " is not reserved.\n"; return false; } // We transitively check superregs. So we can remember this for later // to avoid compiletime explosion in deep register hierarchies. Checked.set(*SR); } } return true; } namespace llvm { Printable printReg(unsigned Reg, const TargetRegisterInfo *TRI, unsigned SubIdx, const MachineRegisterInfo *MRI) { return Printable([Reg, TRI, SubIdx, MRI](raw_ostream &OS) { if (!Reg) OS << "$noreg"; else if (TargetRegisterInfo::isStackSlot(Reg)) OS << "SS#" << TargetRegisterInfo::stackSlot2Index(Reg); else if (TargetRegisterInfo::isVirtualRegister(Reg)) { StringRef Name = MRI ? MRI->getVRegName(Reg) : ""; if (Name != "") { OS << '%' << Name; } else { OS << '%' << TargetRegisterInfo::virtReg2Index(Reg); } } else if (!TRI) OS << '$' << "physreg" << Reg; else if (Reg < TRI->getNumRegs()) { OS << '$'; printLowerCase(TRI->getName(Reg), OS); } else llvm_unreachable("Register kind is unsupported."); if (SubIdx) { if (TRI) OS << ':' << TRI->getSubRegIndexName(SubIdx); else OS << ":sub(" << SubIdx << ')'; } }); } Printable printRegUnit(unsigned Unit, const TargetRegisterInfo *TRI) { return Printable([Unit, TRI](raw_ostream &OS) { // Generic printout when TRI is missing. if (!TRI) { OS << "Unit~" << Unit; return; } // Check for invalid register units. if (Unit >= TRI->getNumRegUnits()) { OS << "BadUnit~" << Unit; return; } // Normal units have at least one root. MCRegUnitRootIterator Roots(Unit, TRI); assert(Roots.isValid() && "Unit has no roots."); OS << TRI->getName(*Roots); for (++Roots; Roots.isValid(); ++Roots) OS << '~' << TRI->getName(*Roots); }); } Printable printVRegOrUnit(unsigned Unit, const TargetRegisterInfo *TRI) { return Printable([Unit, TRI](raw_ostream &OS) { if (TRI && TRI->isVirtualRegister(Unit)) { OS << '%' << TargetRegisterInfo::virtReg2Index(Unit); } else { OS << printRegUnit(Unit, TRI); } }); } Printable printRegClassOrBank(unsigned Reg, const MachineRegisterInfo &RegInfo, const TargetRegisterInfo *TRI) { return Printable([Reg, &RegInfo, TRI](raw_ostream &OS) { if (RegInfo.getRegClassOrNull(Reg)) OS << StringRef(TRI->getRegClassName(RegInfo.getRegClass(Reg))).lower(); else if (RegInfo.getRegBankOrNull(Reg)) OS << StringRef(RegInfo.getRegBankOrNull(Reg)->getName()).lower(); else { OS << "_"; assert((RegInfo.def_empty(Reg) || RegInfo.getType(Reg).isValid()) && "Generic registers must have a valid type"); } }); } } // end namespace llvm /// getAllocatableClass - Return the maximal subclass of the given register /// class that is alloctable, or NULL. const TargetRegisterClass * TargetRegisterInfo::getAllocatableClass(const TargetRegisterClass *RC) const { if (!RC || RC->isAllocatable()) return RC; for (BitMaskClassIterator It(RC->getSubClassMask(), *this); It.isValid(); ++It) { const TargetRegisterClass *SubRC = getRegClass(It.getID()); if (SubRC->isAllocatable()) return SubRC; } return nullptr; } /// getMinimalPhysRegClass - Returns the Register Class of a physical /// register of the given type, picking the most sub register class of /// the right type that contains this physreg. const TargetRegisterClass * TargetRegisterInfo::getMinimalPhysRegClass(unsigned reg, MVT VT) const { assert(isPhysicalRegister(reg) && "reg must be a physical register"); // Pick the most sub register class of the right type that contains // this physreg. const TargetRegisterClass* BestRC = nullptr; for (const TargetRegisterClass* RC : regclasses()) { if ((VT == MVT::Other || isTypeLegalForClass(*RC, VT)) && RC->contains(reg) && (!BestRC || BestRC->hasSubClass(RC))) BestRC = RC; } assert(BestRC && "Couldn't find the register class"); return BestRC; } /// getAllocatableSetForRC - Toggle the bits that represent allocatable /// registers for the specific register class. static void getAllocatableSetForRC(const MachineFunction &MF, const TargetRegisterClass *RC, BitVector &R){ assert(RC->isAllocatable() && "invalid for nonallocatable sets"); ArrayRef Order = RC->getRawAllocationOrder(MF); for (unsigned i = 0; i != Order.size(); ++i) R.set(Order[i]); } BitVector TargetRegisterInfo::getAllocatableSet(const MachineFunction &MF, const TargetRegisterClass *RC) const { BitVector Allocatable(getNumRegs()); if (RC) { // A register class with no allocatable subclass returns an empty set. const TargetRegisterClass *SubClass = getAllocatableClass(RC); if (SubClass) getAllocatableSetForRC(MF, SubClass, Allocatable); } else { for (const TargetRegisterClass *C : regclasses()) if (C->isAllocatable()) getAllocatableSetForRC(MF, C, Allocatable); } // Mask out the reserved registers BitVector Reserved = getReservedRegs(MF); Allocatable &= Reserved.flip(); return Allocatable; } static inline const TargetRegisterClass *firstCommonClass(const uint32_t *A, const uint32_t *B, const TargetRegisterInfo *TRI, const MVT::SimpleValueType SVT = MVT::SimpleValueType::Any) { const MVT VT(SVT); for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; I += 32) if (unsigned Common = *A++ & *B++) { const TargetRegisterClass *RC = TRI->getRegClass(I + countTrailingZeros(Common)); if (SVT == MVT::SimpleValueType::Any || TRI->isTypeLegalForClass(*RC, VT)) return RC; } return nullptr; } const TargetRegisterClass * TargetRegisterInfo::getCommonSubClass(const TargetRegisterClass *A, const TargetRegisterClass *B, const MVT::SimpleValueType SVT) const { // First take care of the trivial cases. if (A == B) return A; if (!A || !B) return nullptr; // Register classes are ordered topologically, so the largest common // sub-class it the common sub-class with the smallest ID. return firstCommonClass(A->getSubClassMask(), B->getSubClassMask(), this, SVT); } const TargetRegisterClass * TargetRegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A, const TargetRegisterClass *B, unsigned Idx) const { assert(A && B && "Missing register class"); assert(Idx && "Bad sub-register index"); // Find Idx in the list of super-register indices. for (SuperRegClassIterator RCI(B, this); RCI.isValid(); ++RCI) if (RCI.getSubReg() == Idx) // The bit mask contains all register classes that are projected into B // by Idx. Find a class that is also a sub-class of A. return firstCommonClass(RCI.getMask(), A->getSubClassMask(), this); return nullptr; } const TargetRegisterClass *TargetRegisterInfo:: getCommonSuperRegClass(const TargetRegisterClass *RCA, unsigned SubA, const TargetRegisterClass *RCB, unsigned SubB, unsigned &PreA, unsigned &PreB) const { assert(RCA && SubA && RCB && SubB && "Invalid arguments"); // Search all pairs of sub-register indices that project into RCA and RCB // respectively. This is quadratic, but usually the sets are very small. On // most targets like X86, there will only be a single sub-register index // (e.g., sub_16bit projecting into GR16). // // The worst case is a register class like DPR on ARM. // We have indices dsub_0..dsub_7 projecting into that class. // // It is very common that one register class is a sub-register of the other. // Arrange for RCA to be the larger register so the answer will be found in // the first iteration. This makes the search linear for the most common // case. const TargetRegisterClass *BestRC = nullptr; unsigned *BestPreA = &PreA; unsigned *BestPreB = &PreB; if (getRegSizeInBits(*RCA) < getRegSizeInBits(*RCB)) { std::swap(RCA, RCB); std::swap(SubA, SubB); std::swap(BestPreA, BestPreB); } // Also terminate the search one we have found a register class as small as // RCA. unsigned MinSize = getRegSizeInBits(*RCA); for (SuperRegClassIterator IA(RCA, this, true); IA.isValid(); ++IA) { unsigned FinalA = composeSubRegIndices(IA.getSubReg(), SubA); for (SuperRegClassIterator IB(RCB, this, true); IB.isValid(); ++IB) { // Check if a common super-register class exists for this index pair. const TargetRegisterClass *RC = firstCommonClass(IA.getMask(), IB.getMask(), this); if (!RC || getRegSizeInBits(*RC) < MinSize) continue; // The indexes must compose identically: PreA+SubA == PreB+SubB. unsigned FinalB = composeSubRegIndices(IB.getSubReg(), SubB); if (FinalA != FinalB) continue; // Is RC a better candidate than BestRC? if (BestRC && getRegSizeInBits(*RC) >= getRegSizeInBits(*BestRC)) continue; // Yes, RC is the smallest super-register seen so far. BestRC = RC; *BestPreA = IA.getSubReg(); *BestPreB = IB.getSubReg(); // Bail early if we reached MinSize. We won't find a better candidate. if (getRegSizeInBits(*BestRC) == MinSize) return BestRC; } } return BestRC; } /// Check if the registers defined by the pair (RegisterClass, SubReg) /// share the same register file. static bool shareSameRegisterFile(const TargetRegisterInfo &TRI, const TargetRegisterClass *DefRC, unsigned DefSubReg, const TargetRegisterClass *SrcRC, unsigned SrcSubReg) { // Same register class. if (DefRC == SrcRC) return true; // Both operands are sub registers. Check if they share a register class. unsigned SrcIdx, DefIdx; if (SrcSubReg && DefSubReg) { return TRI.getCommonSuperRegClass(SrcRC, SrcSubReg, DefRC, DefSubReg, SrcIdx, DefIdx) != nullptr; } // At most one of the register is a sub register, make it Src to avoid // duplicating the test. if (!SrcSubReg) { std::swap(DefSubReg, SrcSubReg); std::swap(DefRC, SrcRC); } // One of the register is a sub register, check if we can get a superclass. if (SrcSubReg) return TRI.getMatchingSuperRegClass(SrcRC, DefRC, SrcSubReg) != nullptr; // Plain copy. return TRI.getCommonSubClass(DefRC, SrcRC) != nullptr; } bool TargetRegisterInfo::shouldRewriteCopySrc(const TargetRegisterClass *DefRC, unsigned DefSubReg, const TargetRegisterClass *SrcRC, unsigned SrcSubReg) const { // If this source does not incur a cross register bank copy, use it. return shareSameRegisterFile(*this, DefRC, DefSubReg, SrcRC, SrcSubReg); } // Compute target-independent register allocator hints to help eliminate copies. bool TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg, ArrayRef Order, SmallVectorImpl &Hints, const MachineFunction &MF, const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const { const MachineRegisterInfo &MRI = MF.getRegInfo(); const std::pair> &Hints_MRI = MRI.getRegAllocationHints(VirtReg); + SmallSet HintedRegs; // First hint may be a target hint. bool Skip = (Hints_MRI.first != 0); for (auto Reg : Hints_MRI.second) { if (Skip) { Skip = false; continue; } // Target-independent hints are either a physical or a virtual register. unsigned Phys = Reg; if (VRM && isVirtualRegister(Phys)) Phys = VRM->getPhys(Phys); + // Don't add the same reg twice (Hints_MRI may contain multiple virtual + // registers allocated to the same physreg). + if (!HintedRegs.insert(Phys).second) + continue; // Check that Phys is a valid hint in VirtReg's register class. if (!isPhysicalRegister(Phys)) continue; if (MRI.isReserved(Phys)) continue; // Check that Phys is in the allocation order. We shouldn't heed hints // from VirtReg's register class if they aren't in the allocation order. The // target probably has a reason for removing the register. if (!is_contained(Order, Phys)) continue; // All clear, tell the register allocator to prefer this register. Hints.push_back(Phys); } return false; } bool TargetRegisterInfo::canRealignStack(const MachineFunction &MF) const { return !MF.getFunction().hasFnAttribute("no-realign-stack"); } bool TargetRegisterInfo::needsStackRealignment( const MachineFunction &MF) const { const MachineFrameInfo &MFI = MF.getFrameInfo(); const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); const Function &F = MF.getFunction(); unsigned StackAlign = TFI->getStackAlignment(); bool requiresRealignment = ((MFI.getMaxAlignment() > StackAlign) || F.hasFnAttribute(Attribute::StackAlignment)); if (F.hasFnAttribute("stackrealign") || requiresRealignment) { if (canRealignStack(MF)) return true; LLVM_DEBUG(dbgs() << "Can't realign function's stack: " << F.getName() << "\n"); } return false; } bool TargetRegisterInfo::regmaskSubsetEqual(const uint32_t *mask0, const uint32_t *mask1) const { unsigned N = (getNumRegs()+31) / 32; for (unsigned I = 0; I < N; ++I) if ((mask0[I] & mask1[I]) != mask0[I]) return false; return true; } unsigned TargetRegisterInfo::getRegSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI) const { const TargetRegisterClass *RC{}; if (isPhysicalRegister(Reg)) { // The size is not directly available for physical registers. // Instead, we need to access a register class that contains Reg and // get the size of that register class. RC = getMinimalPhysRegClass(Reg); } else { LLT Ty = MRI.getType(Reg); unsigned RegSize = Ty.isValid() ? Ty.getSizeInBits() : 0; // If Reg is not a generic register, query the register class to // get its size. if (RegSize) return RegSize; // Since Reg is not a generic register, it must have a register class. RC = MRI.getRegClass(Reg); } assert(RC && "Unable to deduce the register class"); return getRegSizeInBits(*RC); } unsigned TargetRegisterInfo::lookThruCopyLike(unsigned SrcReg, const MachineRegisterInfo *MRI) const { while (true) { const MachineInstr *MI = MRI->getVRegDef(SrcReg); if (!MI->isCopyLike()) return SrcReg; unsigned CopySrcReg; if (MI->isCopy()) CopySrcReg = MI->getOperand(1).getReg(); else { assert(MI->isSubregToReg() && "Bad opcode for lookThruCopyLike"); CopySrcReg = MI->getOperand(2).getReg(); } if (!isVirtualRegister(CopySrcReg)) return CopySrcReg; SrcReg = CopySrcReg; } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void TargetRegisterInfo::dumpReg(unsigned Reg, unsigned SubRegIndex, const TargetRegisterInfo *TRI) { dbgs() << printReg(Reg, TRI, SubRegIndex) << "\n"; } #endif Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/LICENSE.TXT =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/LICENSE.TXT (nonexistent) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/LICENSE.TXT (revision 345026) @@ -0,0 +1,76 @@ +============================================================================== +libunwind License +============================================================================== + +The libunwind library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. Property changes on: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/LICENSE.TXT ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/__libunwind_config.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/__libunwind_config.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/__libunwind_config.h (revision 345026) @@ -1,105 +1,148 @@ //===------------------------- __libunwind_config.h -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #ifndef ____LIBUNWIND_CONFIG_H__ #define ____LIBUNWIND_CONFIG_H__ #if defined(__arm__) && !defined(__USING_SJLJ_EXCEPTIONS__) && \ !defined(__ARM_DWARF_EH__) -#define _LIBUNWIND_ARM_EHABI 1 -#else -#define _LIBUNWIND_ARM_EHABI 0 +#define _LIBUNWIND_ARM_EHABI #endif +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86 8 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86_64 32 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC 112 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC64 116 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64 95 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM 287 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_OR1K 32 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_RISCV 95 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS 65 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC 31 + #if defined(_LIBUNWIND_IS_NATIVE_ONLY) # if defined(__i386__) -# define _LIBUNWIND_TARGET_I386 1 +# define _LIBUNWIND_TARGET_I386 # define _LIBUNWIND_CONTEXT_SIZE 8 -# define _LIBUNWIND_CURSOR_SIZE 19 -# define _LIBUNWIND_MAX_REGISTER 9 +# define _LIBUNWIND_CURSOR_SIZE 15 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86 # elif defined(__x86_64__) # define _LIBUNWIND_TARGET_X86_64 1 -# define _LIBUNWIND_CONTEXT_SIZE 21 -# define _LIBUNWIND_CURSOR_SIZE 33 -# define _LIBUNWIND_MAX_REGISTER 17 +# if defined(_WIN64) +# define _LIBUNWIND_CONTEXT_SIZE 54 +# ifdef __SEH__ +# define _LIBUNWIND_CURSOR_SIZE 204 +# else +# define _LIBUNWIND_CURSOR_SIZE 66 +# endif +# else +# define _LIBUNWIND_CONTEXT_SIZE 21 +# define _LIBUNWIND_CURSOR_SIZE 33 +# endif +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86_64 +# elif defined(__powerpc64__) +# define _LIBUNWIND_TARGET_PPC64 1 +# define _LIBUNWIND_CONTEXT_SIZE 167 +# define _LIBUNWIND_CURSOR_SIZE 179 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC64 # elif defined(__ppc__) # define _LIBUNWIND_TARGET_PPC 1 # define _LIBUNWIND_CONTEXT_SIZE 117 -# define _LIBUNWIND_CURSOR_SIZE 128 -# define _LIBUNWIND_MAX_REGISTER 113 +# define _LIBUNWIND_CURSOR_SIZE 124 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC # elif defined(__aarch64__) # define _LIBUNWIND_TARGET_AARCH64 1 # define _LIBUNWIND_CONTEXT_SIZE 66 -# define _LIBUNWIND_CURSOR_SIZE 78 -# define _LIBUNWIND_MAX_REGISTER 96 +# if defined(__SEH__) +# define _LIBUNWIND_CURSOR_SIZE 164 +# else +# define _LIBUNWIND_CURSOR_SIZE 78 +# endif +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64 # elif defined(__arm__) # define _LIBUNWIND_TARGET_ARM 1 -# define _LIBUNWIND_CONTEXT_SIZE 60 -# define _LIBUNWIND_CURSOR_SIZE 67 -# define _LIBUNWIND_MAX_REGISTER 96 +# if defined(__SEH__) +# define _LIBUNWIND_CONTEXT_SIZE 42 +# define _LIBUNWIND_CURSOR_SIZE 80 +# elif defined(__ARM_WMMX) +# define _LIBUNWIND_CONTEXT_SIZE 61 +# define _LIBUNWIND_CURSOR_SIZE 68 +# else +# define _LIBUNWIND_CONTEXT_SIZE 42 +# define _LIBUNWIND_CURSOR_SIZE 49 +# endif +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM # elif defined(__or1k__) # define _LIBUNWIND_TARGET_OR1K 1 # define _LIBUNWIND_CONTEXT_SIZE 16 -# define _LIBUNWIND_CURSOR_SIZE 28 -# define _LIBUNWIND_MAX_REGISTER 32 +# define _LIBUNWIND_CURSOR_SIZE 24 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_OR1K # elif defined(__riscv) # define _LIBUNWIND_TARGET_RISCV 1 # define _LIBUNWIND_CONTEXT_SIZE 64 # define _LIBUNWIND_CURSOR_SIZE 76 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_RISCV # define _LIBUNWIND_MAX_REGISTER 96 # elif defined(__mips__) # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define _LIBUNWIND_TARGET_MIPS_O32 1 # if defined(__mips_hard_float) # define _LIBUNWIND_CONTEXT_SIZE 50 -# define _LIBUNWIND_CURSOR_SIZE 61 +# define _LIBUNWIND_CURSOR_SIZE 57 # else # define _LIBUNWIND_CONTEXT_SIZE 18 -# define _LIBUNWIND_CURSOR_SIZE 29 +# define _LIBUNWIND_CURSOR_SIZE 24 # endif # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define _LIBUNWIND_TARGET_MIPS_NEWABI 1 # if defined(__mips_hard_float) # define _LIBUNWIND_CONTEXT_SIZE 67 -# define _LIBUNWIND_CURSOR_SIZE 78 +# define _LIBUNWIND_CURSOR_SIZE 74 # else # define _LIBUNWIND_CONTEXT_SIZE 35 -# define _LIBUNWIND_CURSOR_SIZE 46 +# define _LIBUNWIND_CURSOR_SIZE 42 # endif # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define _LIBUNWIND_TARGET_MIPS_NEWABI 1 # if defined(__mips_hard_float) # define _LIBUNWIND_CONTEXT_SIZE 67 # define _LIBUNWIND_CURSOR_SIZE 79 # else # define _LIBUNWIND_CONTEXT_SIZE 35 # define _LIBUNWIND_CURSOR_SIZE 47 # endif # else # error "Unsupported MIPS ABI and/or environment" # endif -# define _LIBUNWIND_MAX_REGISTER 66 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS +# elif defined(__sparc__) + #define _LIBUNWIND_TARGET_SPARC 1 + #define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC + #define _LIBUNWIND_CONTEXT_SIZE 16 + #define _LIBUNWIND_CURSOR_SIZE 23 # else # error "Unsupported architecture." # endif #else // !_LIBUNWIND_IS_NATIVE_ONLY -# define _LIBUNWIND_TARGET_I386 1 +# define _LIBUNWIND_TARGET_I386 # define _LIBUNWIND_TARGET_X86_64 1 # define _LIBUNWIND_TARGET_PPC 1 +# define _LIBUNWIND_TARGET_PPC64 1 # define _LIBUNWIND_TARGET_AARCH64 1 # define _LIBUNWIND_TARGET_ARM 1 # define _LIBUNWIND_TARGET_OR1K 1 # define _LIBUNWIND_TARGET_MIPS_O32 1 # define _LIBUNWIND_TARGET_MIPS_NEWABI 1 -# define _LIBUNWIND_CONTEXT_SIZE 128 -# define _LIBUNWIND_CURSOR_SIZE 140 -# define _LIBUNWIND_MAX_REGISTER 120 +# define _LIBUNWIND_TARGET_SPARC 1 +# define _LIBUNWIND_CONTEXT_SIZE 167 +# define _LIBUNWIND_CURSOR_SIZE 179 +# define _LIBUNWIND_HIGHEST_DWARF_REGISTER 287 #endif // _LIBUNWIND_IS_NATIVE_ONLY #endif // ____LIBUNWIND_CONFIG_H__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/libunwind.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/libunwind.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/libunwind.h (revision 345026) @@ -1,677 +1,933 @@ //===---------------------------- libunwind.h -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Compatible with libunwind API documented at: // http://www.nongnu.org/libunwind/man/libunwind(3).html // //===----------------------------------------------------------------------===// #ifndef __LIBUNWIND__ #define __LIBUNWIND__ #include <__libunwind_config.h> #include #include #ifdef __APPLE__ - #include - #ifdef __arm__ - #define LIBUNWIND_AVAIL __attribute__((unavailable)) + #if __clang__ + #if __has_include() + #include + #endif + #elif __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + #include + #endif + + #ifdef __arm__ + #define LIBUNWIND_AVAIL __attribute__((unavailable)) + #elif defined(__OSX_AVAILABLE_STARTING) + #define LIBUNWIND_AVAIL __OSX_AVAILABLE_STARTING(__MAC_10_6, __IPHONE_5_0) + #else + #include + #ifdef AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER + #define LIBUNWIND_AVAIL AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER #else - #define LIBUNWIND_AVAIL __OSX_AVAILABLE_STARTING(__MAC_10_6, __IPHONE_5_0) + #define LIBUNWIND_AVAIL __attribute__((unavailable)) #endif + #endif #else #define LIBUNWIND_AVAIL #endif /* error codes */ enum { UNW_ESUCCESS = 0, /* no error */ UNW_EUNSPEC = -6540, /* unspecified (general) error */ UNW_ENOMEM = -6541, /* out of memory */ UNW_EBADREG = -6542, /* bad register number */ UNW_EREADONLYREG = -6543, /* attempt to write read-only register */ UNW_ESTOPUNWIND = -6544, /* stop unwinding */ UNW_EINVALIDIP = -6545, /* invalid IP */ UNW_EBADFRAME = -6546, /* bad frame */ UNW_EINVAL = -6547, /* unsupported operation or bad value */ UNW_EBADVERSION = -6548, /* unwind info has unsupported version */ UNW_ENOINFO = -6549 /* no unwind info found */ +#if defined(_LIBUNWIND_TARGET_AARCH64) && !defined(_LIBUNWIND_IS_NATIVE_ONLY) + , UNW_ECROSSRASIGNING = -6550 /* cross unwind with return address signing */ +#endif }; struct unw_context_t { uint64_t data[_LIBUNWIND_CONTEXT_SIZE]; }; typedef struct unw_context_t unw_context_t; struct unw_cursor_t { uint64_t data[_LIBUNWIND_CURSOR_SIZE]; }; typedef struct unw_cursor_t unw_cursor_t; typedef struct unw_addr_space *unw_addr_space_t; typedef int unw_regnum_t; -#if _LIBUNWIND_ARM_EHABI -typedef uint32_t unw_word_t; +typedef uintptr_t unw_word_t; +#if defined(__arm__) typedef uint64_t unw_fpreg_t; #else -typedef uint64_t unw_word_t; typedef double unw_fpreg_t; #endif struct unw_proc_info_t { unw_word_t start_ip; /* start address of function */ unw_word_t end_ip; /* address after end of function */ unw_word_t lsda; /* address of language specific data area, */ /* or zero if not used */ unw_word_t handler; /* personality routine, or zero if not used */ unw_word_t gp; /* not used */ unw_word_t flags; /* not used */ uint32_t format; /* compact unwind encoding, or zero if none */ - uint32_t unwind_info_size; /* size of dwarf unwind info, or zero if none */ - unw_word_t unwind_info; /* address of dwarf unwind info, or zero */ + uint32_t unwind_info_size; /* size of DWARF unwind info, or zero if none */ + unw_word_t unwind_info; /* address of DWARF unwind info, or zero */ unw_word_t extra; /* mach_header of mach-o image containing func */ }; typedef struct unw_proc_info_t unw_proc_info_t; #ifdef __cplusplus extern "C" { #endif extern int unw_getcontext(unw_context_t *) LIBUNWIND_AVAIL; extern int unw_init_local(unw_cursor_t *, unw_context_t *) LIBUNWIND_AVAIL; extern int unw_step(unw_cursor_t *) LIBUNWIND_AVAIL; extern int unw_get_reg(unw_cursor_t *, unw_regnum_t, unw_word_t *) LIBUNWIND_AVAIL; extern int unw_get_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t *) LIBUNWIND_AVAIL; extern int unw_set_reg(unw_cursor_t *, unw_regnum_t, unw_word_t) LIBUNWIND_AVAIL; extern int unw_set_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t) LIBUNWIND_AVAIL; extern int unw_resume(unw_cursor_t *) LIBUNWIND_AVAIL; #ifdef __arm__ /* Save VFP registers in FSTMX format (instead of FSTMD). */ extern void unw_save_vfp_as_X(unw_cursor_t *) LIBUNWIND_AVAIL; #endif extern const char *unw_regname(unw_cursor_t *, unw_regnum_t) LIBUNWIND_AVAIL; extern int unw_get_proc_info(unw_cursor_t *, unw_proc_info_t *) LIBUNWIND_AVAIL; extern int unw_is_fpreg(unw_cursor_t *, unw_regnum_t) LIBUNWIND_AVAIL; extern int unw_is_signal_frame(unw_cursor_t *) LIBUNWIND_AVAIL; extern int unw_get_proc_name(unw_cursor_t *, char *, size_t, unw_word_t *) LIBUNWIND_AVAIL; //extern int unw_get_save_loc(unw_cursor_t*, int, unw_save_loc_t*); extern unw_addr_space_t unw_local_addr_space; #ifdef UNW_REMOTE /* * Mac OS X "remote" API for unwinding other processes on same machine * */ extern unw_addr_space_t unw_create_addr_space_for_task(task_t); extern void unw_destroy_addr_space(unw_addr_space_t); extern int unw_init_remote_thread(unw_cursor_t *, unw_addr_space_t, thread_t *); #endif /* UNW_REMOTE */ /* * traditional libunwind "remote" API * NOT IMPLEMENTED on Mac OS X * * extern int unw_init_remote(unw_cursor_t*, unw_addr_space_t, * thread_t*); * extern unw_accessors_t unw_get_accessors(unw_addr_space_t); * extern unw_addr_space_t unw_create_addr_space(unw_accessors_t, int); * extern void unw_flush_cache(unw_addr_space_t, unw_word_t, * unw_word_t); * extern int unw_set_caching_policy(unw_addr_space_t, * unw_caching_policy_t); * extern void _U_dyn_register(unw_dyn_info_t*); * extern void _U_dyn_cancel(unw_dyn_info_t*); */ #ifdef __cplusplus } #endif // architecture independent register numbers enum { UNW_REG_IP = -1, // instruction pointer UNW_REG_SP = -2, // stack pointer }; // 32-bit x86 registers enum { UNW_X86_EAX = 0, UNW_X86_ECX = 1, UNW_X86_EDX = 2, UNW_X86_EBX = 3, - UNW_X86_ESP = 4, - UNW_X86_EBP = 5, + UNW_X86_EBP = 4, + UNW_X86_ESP = 5, UNW_X86_ESI = 6, UNW_X86_EDI = 7 }; // 64-bit x86_64 registers enum { UNW_X86_64_RAX = 0, UNW_X86_64_RDX = 1, UNW_X86_64_RCX = 2, UNW_X86_64_RBX = 3, UNW_X86_64_RSI = 4, UNW_X86_64_RDI = 5, UNW_X86_64_RBP = 6, UNW_X86_64_RSP = 7, UNW_X86_64_R8 = 8, UNW_X86_64_R9 = 9, UNW_X86_64_R10 = 10, UNW_X86_64_R11 = 11, UNW_X86_64_R12 = 12, UNW_X86_64_R13 = 13, UNW_X86_64_R14 = 14, - UNW_X86_64_R15 = 15 + UNW_X86_64_R15 = 15, + UNW_X86_64_RIP = 16, + UNW_X86_64_XMM0 = 17, + UNW_X86_64_XMM1 = 18, + UNW_X86_64_XMM2 = 19, + UNW_X86_64_XMM3 = 20, + UNW_X86_64_XMM4 = 21, + UNW_X86_64_XMM5 = 22, + UNW_X86_64_XMM6 = 23, + UNW_X86_64_XMM7 = 24, + UNW_X86_64_XMM8 = 25, + UNW_X86_64_XMM9 = 26, + UNW_X86_64_XMM10 = 27, + UNW_X86_64_XMM11 = 28, + UNW_X86_64_XMM12 = 29, + UNW_X86_64_XMM13 = 30, + UNW_X86_64_XMM14 = 31, + UNW_X86_64_XMM15 = 32, }; // 32-bit ppc register numbers enum { UNW_PPC_R0 = 0, UNW_PPC_R1 = 1, UNW_PPC_R2 = 2, UNW_PPC_R3 = 3, UNW_PPC_R4 = 4, UNW_PPC_R5 = 5, UNW_PPC_R6 = 6, UNW_PPC_R7 = 7, UNW_PPC_R8 = 8, UNW_PPC_R9 = 9, UNW_PPC_R10 = 10, UNW_PPC_R11 = 11, UNW_PPC_R12 = 12, UNW_PPC_R13 = 13, UNW_PPC_R14 = 14, UNW_PPC_R15 = 15, UNW_PPC_R16 = 16, UNW_PPC_R17 = 17, UNW_PPC_R18 = 18, UNW_PPC_R19 = 19, UNW_PPC_R20 = 20, UNW_PPC_R21 = 21, UNW_PPC_R22 = 22, UNW_PPC_R23 = 23, UNW_PPC_R24 = 24, UNW_PPC_R25 = 25, UNW_PPC_R26 = 26, UNW_PPC_R27 = 27, UNW_PPC_R28 = 28, UNW_PPC_R29 = 29, UNW_PPC_R30 = 30, UNW_PPC_R31 = 31, UNW_PPC_F0 = 32, UNW_PPC_F1 = 33, UNW_PPC_F2 = 34, UNW_PPC_F3 = 35, UNW_PPC_F4 = 36, UNW_PPC_F5 = 37, UNW_PPC_F6 = 38, UNW_PPC_F7 = 39, UNW_PPC_F8 = 40, UNW_PPC_F9 = 41, UNW_PPC_F10 = 42, UNW_PPC_F11 = 43, UNW_PPC_F12 = 44, UNW_PPC_F13 = 45, UNW_PPC_F14 = 46, UNW_PPC_F15 = 47, UNW_PPC_F16 = 48, UNW_PPC_F17 = 49, UNW_PPC_F18 = 50, UNW_PPC_F19 = 51, UNW_PPC_F20 = 52, UNW_PPC_F21 = 53, UNW_PPC_F22 = 54, UNW_PPC_F23 = 55, UNW_PPC_F24 = 56, UNW_PPC_F25 = 57, UNW_PPC_F26 = 58, UNW_PPC_F27 = 59, UNW_PPC_F28 = 60, UNW_PPC_F29 = 61, UNW_PPC_F30 = 62, UNW_PPC_F31 = 63, UNW_PPC_MQ = 64, UNW_PPC_LR = 65, UNW_PPC_CTR = 66, UNW_PPC_AP = 67, UNW_PPC_CR0 = 68, UNW_PPC_CR1 = 69, UNW_PPC_CR2 = 70, UNW_PPC_CR3 = 71, UNW_PPC_CR4 = 72, UNW_PPC_CR5 = 73, UNW_PPC_CR6 = 74, UNW_PPC_CR7 = 75, UNW_PPC_XER = 76, UNW_PPC_V0 = 77, UNW_PPC_V1 = 78, UNW_PPC_V2 = 79, UNW_PPC_V3 = 80, UNW_PPC_V4 = 81, UNW_PPC_V5 = 82, UNW_PPC_V6 = 83, UNW_PPC_V7 = 84, UNW_PPC_V8 = 85, UNW_PPC_V9 = 86, UNW_PPC_V10 = 87, UNW_PPC_V11 = 88, UNW_PPC_V12 = 89, UNW_PPC_V13 = 90, UNW_PPC_V14 = 91, UNW_PPC_V15 = 92, UNW_PPC_V16 = 93, UNW_PPC_V17 = 94, UNW_PPC_V18 = 95, UNW_PPC_V19 = 96, UNW_PPC_V20 = 97, UNW_PPC_V21 = 98, UNW_PPC_V22 = 99, UNW_PPC_V23 = 100, UNW_PPC_V24 = 101, UNW_PPC_V25 = 102, UNW_PPC_V26 = 103, UNW_PPC_V27 = 104, UNW_PPC_V28 = 105, UNW_PPC_V29 = 106, UNW_PPC_V30 = 107, UNW_PPC_V31 = 108, UNW_PPC_VRSAVE = 109, UNW_PPC_VSCR = 110, UNW_PPC_SPE_ACC = 111, UNW_PPC_SPEFSCR = 112 }; +// 64-bit ppc register numbers +enum { + UNW_PPC64_R0 = 0, + UNW_PPC64_R1 = 1, + UNW_PPC64_R2 = 2, + UNW_PPC64_R3 = 3, + UNW_PPC64_R4 = 4, + UNW_PPC64_R5 = 5, + UNW_PPC64_R6 = 6, + UNW_PPC64_R7 = 7, + UNW_PPC64_R8 = 8, + UNW_PPC64_R9 = 9, + UNW_PPC64_R10 = 10, + UNW_PPC64_R11 = 11, + UNW_PPC64_R12 = 12, + UNW_PPC64_R13 = 13, + UNW_PPC64_R14 = 14, + UNW_PPC64_R15 = 15, + UNW_PPC64_R16 = 16, + UNW_PPC64_R17 = 17, + UNW_PPC64_R18 = 18, + UNW_PPC64_R19 = 19, + UNW_PPC64_R20 = 20, + UNW_PPC64_R21 = 21, + UNW_PPC64_R22 = 22, + UNW_PPC64_R23 = 23, + UNW_PPC64_R24 = 24, + UNW_PPC64_R25 = 25, + UNW_PPC64_R26 = 26, + UNW_PPC64_R27 = 27, + UNW_PPC64_R28 = 28, + UNW_PPC64_R29 = 29, + UNW_PPC64_R30 = 30, + UNW_PPC64_R31 = 31, + UNW_PPC64_F0 = 32, + UNW_PPC64_F1 = 33, + UNW_PPC64_F2 = 34, + UNW_PPC64_F3 = 35, + UNW_PPC64_F4 = 36, + UNW_PPC64_F5 = 37, + UNW_PPC64_F6 = 38, + UNW_PPC64_F7 = 39, + UNW_PPC64_F8 = 40, + UNW_PPC64_F9 = 41, + UNW_PPC64_F10 = 42, + UNW_PPC64_F11 = 43, + UNW_PPC64_F12 = 44, + UNW_PPC64_F13 = 45, + UNW_PPC64_F14 = 46, + UNW_PPC64_F15 = 47, + UNW_PPC64_F16 = 48, + UNW_PPC64_F17 = 49, + UNW_PPC64_F18 = 50, + UNW_PPC64_F19 = 51, + UNW_PPC64_F20 = 52, + UNW_PPC64_F21 = 53, + UNW_PPC64_F22 = 54, + UNW_PPC64_F23 = 55, + UNW_PPC64_F24 = 56, + UNW_PPC64_F25 = 57, + UNW_PPC64_F26 = 58, + UNW_PPC64_F27 = 59, + UNW_PPC64_F28 = 60, + UNW_PPC64_F29 = 61, + UNW_PPC64_F30 = 62, + UNW_PPC64_F31 = 63, + // 64: reserved + UNW_PPC64_LR = 65, + UNW_PPC64_CTR = 66, + // 67: reserved + UNW_PPC64_CR0 = 68, + UNW_PPC64_CR1 = 69, + UNW_PPC64_CR2 = 70, + UNW_PPC64_CR3 = 71, + UNW_PPC64_CR4 = 72, + UNW_PPC64_CR5 = 73, + UNW_PPC64_CR6 = 74, + UNW_PPC64_CR7 = 75, + UNW_PPC64_XER = 76, + UNW_PPC64_V0 = 77, + UNW_PPC64_V1 = 78, + UNW_PPC64_V2 = 79, + UNW_PPC64_V3 = 80, + UNW_PPC64_V4 = 81, + UNW_PPC64_V5 = 82, + UNW_PPC64_V6 = 83, + UNW_PPC64_V7 = 84, + UNW_PPC64_V8 = 85, + UNW_PPC64_V9 = 86, + UNW_PPC64_V10 = 87, + UNW_PPC64_V11 = 88, + UNW_PPC64_V12 = 89, + UNW_PPC64_V13 = 90, + UNW_PPC64_V14 = 91, + UNW_PPC64_V15 = 92, + UNW_PPC64_V16 = 93, + UNW_PPC64_V17 = 94, + UNW_PPC64_V18 = 95, + UNW_PPC64_V19 = 96, + UNW_PPC64_V20 = 97, + UNW_PPC64_V21 = 98, + UNW_PPC64_V22 = 99, + UNW_PPC64_V23 = 100, + UNW_PPC64_V24 = 101, + UNW_PPC64_V25 = 102, + UNW_PPC64_V26 = 103, + UNW_PPC64_V27 = 104, + UNW_PPC64_V28 = 105, + UNW_PPC64_V29 = 106, + UNW_PPC64_V30 = 107, + UNW_PPC64_V31 = 108, + // 109, 111-113: OpenPOWER ELF V2 ABI: reserved + // Borrowing VRSAVE number from PPC32. + UNW_PPC64_VRSAVE = 109, + UNW_PPC64_VSCR = 110, + UNW_PPC64_TFHAR = 114, + UNW_PPC64_TFIAR = 115, + UNW_PPC64_TEXASR = 116, + UNW_PPC64_VS0 = UNW_PPC64_F0, + UNW_PPC64_VS1 = UNW_PPC64_F1, + UNW_PPC64_VS2 = UNW_PPC64_F2, + UNW_PPC64_VS3 = UNW_PPC64_F3, + UNW_PPC64_VS4 = UNW_PPC64_F4, + UNW_PPC64_VS5 = UNW_PPC64_F5, + UNW_PPC64_VS6 = UNW_PPC64_F6, + UNW_PPC64_VS7 = UNW_PPC64_F7, + UNW_PPC64_VS8 = UNW_PPC64_F8, + UNW_PPC64_VS9 = UNW_PPC64_F9, + UNW_PPC64_VS10 = UNW_PPC64_F10, + UNW_PPC64_VS11 = UNW_PPC64_F11, + UNW_PPC64_VS12 = UNW_PPC64_F12, + UNW_PPC64_VS13 = UNW_PPC64_F13, + UNW_PPC64_VS14 = UNW_PPC64_F14, + UNW_PPC64_VS15 = UNW_PPC64_F15, + UNW_PPC64_VS16 = UNW_PPC64_F16, + UNW_PPC64_VS17 = UNW_PPC64_F17, + UNW_PPC64_VS18 = UNW_PPC64_F18, + UNW_PPC64_VS19 = UNW_PPC64_F19, + UNW_PPC64_VS20 = UNW_PPC64_F20, + UNW_PPC64_VS21 = UNW_PPC64_F21, + UNW_PPC64_VS22 = UNW_PPC64_F22, + UNW_PPC64_VS23 = UNW_PPC64_F23, + UNW_PPC64_VS24 = UNW_PPC64_F24, + UNW_PPC64_VS25 = UNW_PPC64_F25, + UNW_PPC64_VS26 = UNW_PPC64_F26, + UNW_PPC64_VS27 = UNW_PPC64_F27, + UNW_PPC64_VS28 = UNW_PPC64_F28, + UNW_PPC64_VS29 = UNW_PPC64_F29, + UNW_PPC64_VS30 = UNW_PPC64_F30, + UNW_PPC64_VS31 = UNW_PPC64_F31, + UNW_PPC64_VS32 = UNW_PPC64_V0, + UNW_PPC64_VS33 = UNW_PPC64_V1, + UNW_PPC64_VS34 = UNW_PPC64_V2, + UNW_PPC64_VS35 = UNW_PPC64_V3, + UNW_PPC64_VS36 = UNW_PPC64_V4, + UNW_PPC64_VS37 = UNW_PPC64_V5, + UNW_PPC64_VS38 = UNW_PPC64_V6, + UNW_PPC64_VS39 = UNW_PPC64_V7, + UNW_PPC64_VS40 = UNW_PPC64_V8, + UNW_PPC64_VS41 = UNW_PPC64_V9, + UNW_PPC64_VS42 = UNW_PPC64_V10, + UNW_PPC64_VS43 = UNW_PPC64_V11, + UNW_PPC64_VS44 = UNW_PPC64_V12, + UNW_PPC64_VS45 = UNW_PPC64_V13, + UNW_PPC64_VS46 = UNW_PPC64_V14, + UNW_PPC64_VS47 = UNW_PPC64_V15, + UNW_PPC64_VS48 = UNW_PPC64_V16, + UNW_PPC64_VS49 = UNW_PPC64_V17, + UNW_PPC64_VS50 = UNW_PPC64_V18, + UNW_PPC64_VS51 = UNW_PPC64_V19, + UNW_PPC64_VS52 = UNW_PPC64_V20, + UNW_PPC64_VS53 = UNW_PPC64_V21, + UNW_PPC64_VS54 = UNW_PPC64_V22, + UNW_PPC64_VS55 = UNW_PPC64_V23, + UNW_PPC64_VS56 = UNW_PPC64_V24, + UNW_PPC64_VS57 = UNW_PPC64_V25, + UNW_PPC64_VS58 = UNW_PPC64_V26, + UNW_PPC64_VS59 = UNW_PPC64_V27, + UNW_PPC64_VS60 = UNW_PPC64_V28, + UNW_PPC64_VS61 = UNW_PPC64_V29, + UNW_PPC64_VS62 = UNW_PPC64_V30, + UNW_PPC64_VS63 = UNW_PPC64_V31 +}; + // 64-bit ARM64 registers enum { UNW_ARM64_X0 = 0, UNW_ARM64_X1 = 1, UNW_ARM64_X2 = 2, UNW_ARM64_X3 = 3, UNW_ARM64_X4 = 4, UNW_ARM64_X5 = 5, UNW_ARM64_X6 = 6, UNW_ARM64_X7 = 7, UNW_ARM64_X8 = 8, UNW_ARM64_X9 = 9, UNW_ARM64_X10 = 10, UNW_ARM64_X11 = 11, UNW_ARM64_X12 = 12, UNW_ARM64_X13 = 13, UNW_ARM64_X14 = 14, UNW_ARM64_X15 = 15, UNW_ARM64_X16 = 16, UNW_ARM64_X17 = 17, UNW_ARM64_X18 = 18, UNW_ARM64_X19 = 19, UNW_ARM64_X20 = 20, UNW_ARM64_X21 = 21, UNW_ARM64_X22 = 22, UNW_ARM64_X23 = 23, UNW_ARM64_X24 = 24, UNW_ARM64_X25 = 25, UNW_ARM64_X26 = 26, UNW_ARM64_X27 = 27, UNW_ARM64_X28 = 28, UNW_ARM64_X29 = 29, UNW_ARM64_FP = 29, UNW_ARM64_X30 = 30, UNW_ARM64_LR = 30, UNW_ARM64_X31 = 31, UNW_ARM64_SP = 31, // reserved block + UNW_ARM64_RA_SIGN_STATE = 34, + // reserved block UNW_ARM64_D0 = 64, UNW_ARM64_D1 = 65, UNW_ARM64_D2 = 66, UNW_ARM64_D3 = 67, UNW_ARM64_D4 = 68, UNW_ARM64_D5 = 69, UNW_ARM64_D6 = 70, UNW_ARM64_D7 = 71, UNW_ARM64_D8 = 72, UNW_ARM64_D9 = 73, UNW_ARM64_D10 = 74, UNW_ARM64_D11 = 75, UNW_ARM64_D12 = 76, UNW_ARM64_D13 = 77, UNW_ARM64_D14 = 78, UNW_ARM64_D15 = 79, UNW_ARM64_D16 = 80, UNW_ARM64_D17 = 81, UNW_ARM64_D18 = 82, UNW_ARM64_D19 = 83, UNW_ARM64_D20 = 84, UNW_ARM64_D21 = 85, UNW_ARM64_D22 = 86, UNW_ARM64_D23 = 87, UNW_ARM64_D24 = 88, UNW_ARM64_D25 = 89, UNW_ARM64_D26 = 90, UNW_ARM64_D27 = 91, UNW_ARM64_D28 = 92, UNW_ARM64_D29 = 93, UNW_ARM64_D30 = 94, UNW_ARM64_D31 = 95, }; // 32-bit ARM registers. Numbers match DWARF for ARM spec #3.1 Table 1. // Naming scheme uses recommendations given in Note 4 for VFP-v2 and VFP-v3. // In this scheme, even though the 64-bit floating point registers D0-D31 // overlap physically with the 32-bit floating pointer registers S0-S31, // they are given a non-overlapping range of register numbers. // // Commented out ranges are not preserved during unwinding. enum { UNW_ARM_R0 = 0, UNW_ARM_R1 = 1, UNW_ARM_R2 = 2, UNW_ARM_R3 = 3, UNW_ARM_R4 = 4, UNW_ARM_R5 = 5, UNW_ARM_R6 = 6, UNW_ARM_R7 = 7, UNW_ARM_R8 = 8, UNW_ARM_R9 = 9, UNW_ARM_R10 = 10, UNW_ARM_R11 = 11, UNW_ARM_R12 = 12, UNW_ARM_SP = 13, // Logical alias for UNW_REG_SP UNW_ARM_R13 = 13, UNW_ARM_LR = 14, UNW_ARM_R14 = 14, UNW_ARM_IP = 15, // Logical alias for UNW_REG_IP UNW_ARM_R15 = 15, // 16-63 -- OBSOLETE. Used in VFP1 to represent both S0-S31 and D0-D31. UNW_ARM_S0 = 64, UNW_ARM_S1 = 65, UNW_ARM_S2 = 66, UNW_ARM_S3 = 67, UNW_ARM_S4 = 68, UNW_ARM_S5 = 69, UNW_ARM_S6 = 70, UNW_ARM_S7 = 71, UNW_ARM_S8 = 72, UNW_ARM_S9 = 73, UNW_ARM_S10 = 74, UNW_ARM_S11 = 75, UNW_ARM_S12 = 76, UNW_ARM_S13 = 77, UNW_ARM_S14 = 78, UNW_ARM_S15 = 79, UNW_ARM_S16 = 80, UNW_ARM_S17 = 81, UNW_ARM_S18 = 82, UNW_ARM_S19 = 83, UNW_ARM_S20 = 84, UNW_ARM_S21 = 85, UNW_ARM_S22 = 86, UNW_ARM_S23 = 87, UNW_ARM_S24 = 88, UNW_ARM_S25 = 89, UNW_ARM_S26 = 90, UNW_ARM_S27 = 91, UNW_ARM_S28 = 92, UNW_ARM_S29 = 93, UNW_ARM_S30 = 94, UNW_ARM_S31 = 95, // 96-103 -- OBSOLETE. F0-F7. Used by the FPA system. Superseded by VFP. // 104-111 -- wCGR0-wCGR7, ACC0-ACC7 (Intel wireless MMX) UNW_ARM_WR0 = 112, UNW_ARM_WR1 = 113, UNW_ARM_WR2 = 114, UNW_ARM_WR3 = 115, UNW_ARM_WR4 = 116, UNW_ARM_WR5 = 117, UNW_ARM_WR6 = 118, UNW_ARM_WR7 = 119, UNW_ARM_WR8 = 120, UNW_ARM_WR9 = 121, UNW_ARM_WR10 = 122, UNW_ARM_WR11 = 123, UNW_ARM_WR12 = 124, UNW_ARM_WR13 = 125, UNW_ARM_WR14 = 126, UNW_ARM_WR15 = 127, // 128-133 -- SPSR, SPSR_{FIQ|IRQ|ABT|UND|SVC} // 134-143 -- Reserved // 144-150 -- R8_USR-R14_USR // 151-157 -- R8_FIQ-R14_FIQ // 158-159 -- R13_IRQ-R14_IRQ // 160-161 -- R13_ABT-R14_ABT // 162-163 -- R13_UND-R14_UND // 164-165 -- R13_SVC-R14_SVC // 166-191 -- Reserved UNW_ARM_WC0 = 192, UNW_ARM_WC1 = 193, UNW_ARM_WC2 = 194, UNW_ARM_WC3 = 195, // 196-199 -- wC4-wC7 (Intel wireless MMX control) // 200-255 -- Reserved UNW_ARM_D0 = 256, UNW_ARM_D1 = 257, UNW_ARM_D2 = 258, UNW_ARM_D3 = 259, UNW_ARM_D4 = 260, UNW_ARM_D5 = 261, UNW_ARM_D6 = 262, UNW_ARM_D7 = 263, UNW_ARM_D8 = 264, UNW_ARM_D9 = 265, UNW_ARM_D10 = 266, UNW_ARM_D11 = 267, UNW_ARM_D12 = 268, UNW_ARM_D13 = 269, UNW_ARM_D14 = 270, UNW_ARM_D15 = 271, UNW_ARM_D16 = 272, UNW_ARM_D17 = 273, UNW_ARM_D18 = 274, UNW_ARM_D19 = 275, UNW_ARM_D20 = 276, UNW_ARM_D21 = 277, UNW_ARM_D22 = 278, UNW_ARM_D23 = 279, UNW_ARM_D24 = 280, UNW_ARM_D25 = 281, UNW_ARM_D26 = 282, UNW_ARM_D27 = 283, UNW_ARM_D28 = 284, UNW_ARM_D29 = 285, UNW_ARM_D30 = 286, UNW_ARM_D31 = 287, // 288-319 -- Reserved for VFP/Neon // 320-8191 -- Reserved // 8192-16383 -- Unspecified vendor co-processor register. }; // OpenRISC1000 register numbers enum { UNW_OR1K_R0 = 0, UNW_OR1K_R1 = 1, UNW_OR1K_R2 = 2, UNW_OR1K_R3 = 3, UNW_OR1K_R4 = 4, UNW_OR1K_R5 = 5, UNW_OR1K_R6 = 6, UNW_OR1K_R7 = 7, UNW_OR1K_R8 = 8, UNW_OR1K_R9 = 9, UNW_OR1K_R10 = 10, UNW_OR1K_R11 = 11, UNW_OR1K_R12 = 12, UNW_OR1K_R13 = 13, UNW_OR1K_R14 = 14, UNW_OR1K_R15 = 15, UNW_OR1K_R16 = 16, UNW_OR1K_R17 = 17, UNW_OR1K_R18 = 18, UNW_OR1K_R19 = 19, UNW_OR1K_R20 = 20, UNW_OR1K_R21 = 21, UNW_OR1K_R22 = 22, UNW_OR1K_R23 = 23, UNW_OR1K_R24 = 24, UNW_OR1K_R25 = 25, UNW_OR1K_R26 = 26, UNW_OR1K_R27 = 27, UNW_OR1K_R28 = 28, UNW_OR1K_R29 = 29, UNW_OR1K_R30 = 30, UNW_OR1K_R31 = 31, + UNW_OR1K_EPCR = 32, }; // 64-bit RISC-V registers enum { UNW_RISCV_X0 = 0, UNW_RISCV_X1 = 1, UNW_RISCV_RA = 1, UNW_RISCV_X2 = 2, UNW_RISCV_SP = 2, UNW_RISCV_X3 = 3, UNW_RISCV_X4 = 4, UNW_RISCV_X5 = 5, UNW_RISCV_X6 = 6, UNW_RISCV_X7 = 7, UNW_RISCV_X8 = 8, UNW_RISCV_X9 = 9, UNW_RISCV_X10 = 10, UNW_RISCV_X11 = 11, UNW_RISCV_X12 = 12, UNW_RISCV_X13 = 13, UNW_RISCV_X14 = 14, UNW_RISCV_X15 = 15, UNW_RISCV_X16 = 16, UNW_RISCV_X17 = 17, UNW_RISCV_X18 = 18, UNW_RISCV_X19 = 19, UNW_RISCV_X20 = 20, UNW_RISCV_X21 = 21, UNW_RISCV_X22 = 22, UNW_RISCV_X23 = 23, UNW_RISCV_X24 = 24, UNW_RISCV_X25 = 25, UNW_RISCV_X26 = 26, UNW_RISCV_X27 = 27, UNW_RISCV_X28 = 28, UNW_RISCV_X29 = 29, UNW_RISCV_X30 = 30, UNW_RISCV_X31 = 31, // reserved block UNW_RISCV_D0 = 64, UNW_RISCV_D1 = 65, UNW_RISCV_D2 = 66, UNW_RISCV_D3 = 67, UNW_RISCV_D4 = 68, UNW_RISCV_D5 = 69, UNW_RISCV_D6 = 70, UNW_RISCV_D7 = 71, UNW_RISCV_D8 = 72, UNW_RISCV_D9 = 73, UNW_RISCV_D10 = 74, UNW_RISCV_D11 = 75, UNW_RISCV_D12 = 76, UNW_RISCV_D13 = 77, UNW_RISCV_D14 = 78, UNW_RISCV_D15 = 79, UNW_RISCV_D16 = 80, UNW_RISCV_D17 = 81, UNW_RISCV_D18 = 82, UNW_RISCV_D19 = 83, UNW_RISCV_D20 = 84, UNW_RISCV_D21 = 85, UNW_RISCV_D22 = 86, UNW_RISCV_D23 = 87, UNW_RISCV_D24 = 88, UNW_RISCV_D25 = 89, UNW_RISCV_D26 = 90, UNW_RISCV_D27 = 91, UNW_RISCV_D28 = 92, UNW_RISCV_D29 = 93, UNW_RISCV_D30 = 94, UNW_RISCV_D31 = 95, }; // MIPS registers enum { UNW_MIPS_R0 = 0, UNW_MIPS_R1 = 1, UNW_MIPS_R2 = 2, UNW_MIPS_R3 = 3, UNW_MIPS_R4 = 4, UNW_MIPS_R5 = 5, UNW_MIPS_R6 = 6, UNW_MIPS_R7 = 7, UNW_MIPS_R8 = 8, UNW_MIPS_R9 = 9, UNW_MIPS_R10 = 10, UNW_MIPS_R11 = 11, UNW_MIPS_R12 = 12, UNW_MIPS_R13 = 13, UNW_MIPS_R14 = 14, UNW_MIPS_R15 = 15, UNW_MIPS_R16 = 16, UNW_MIPS_R17 = 17, UNW_MIPS_R18 = 18, UNW_MIPS_R19 = 19, UNW_MIPS_R20 = 20, UNW_MIPS_R21 = 21, UNW_MIPS_R22 = 22, UNW_MIPS_R23 = 23, UNW_MIPS_R24 = 24, UNW_MIPS_R25 = 25, UNW_MIPS_R26 = 26, UNW_MIPS_R27 = 27, UNW_MIPS_R28 = 28, UNW_MIPS_R29 = 29, UNW_MIPS_R30 = 30, UNW_MIPS_R31 = 31, UNW_MIPS_F0 = 32, UNW_MIPS_F1 = 33, UNW_MIPS_F2 = 34, UNW_MIPS_F3 = 35, UNW_MIPS_F4 = 36, UNW_MIPS_F5 = 37, UNW_MIPS_F6 = 38, UNW_MIPS_F7 = 39, UNW_MIPS_F8 = 40, UNW_MIPS_F9 = 41, UNW_MIPS_F10 = 42, UNW_MIPS_F11 = 43, UNW_MIPS_F12 = 44, UNW_MIPS_F13 = 45, UNW_MIPS_F14 = 46, UNW_MIPS_F15 = 47, UNW_MIPS_F16 = 48, UNW_MIPS_F17 = 49, UNW_MIPS_F18 = 50, UNW_MIPS_F19 = 51, UNW_MIPS_F20 = 52, UNW_MIPS_F21 = 53, UNW_MIPS_F22 = 54, UNW_MIPS_F23 = 55, UNW_MIPS_F24 = 56, UNW_MIPS_F25 = 57, UNW_MIPS_F26 = 58, UNW_MIPS_F27 = 59, UNW_MIPS_F28 = 60, UNW_MIPS_F29 = 61, UNW_MIPS_F30 = 62, UNW_MIPS_F31 = 63, UNW_MIPS_HI = 64, UNW_MIPS_LO = 65, +}; + +// SPARC registers +enum { + UNW_SPARC_G0 = 0, + UNW_SPARC_G1 = 1, + UNW_SPARC_G2 = 2, + UNW_SPARC_G3 = 3, + UNW_SPARC_G4 = 4, + UNW_SPARC_G5 = 5, + UNW_SPARC_G6 = 6, + UNW_SPARC_G7 = 7, + UNW_SPARC_O0 = 8, + UNW_SPARC_O1 = 9, + UNW_SPARC_O2 = 10, + UNW_SPARC_O3 = 11, + UNW_SPARC_O4 = 12, + UNW_SPARC_O5 = 13, + UNW_SPARC_O6 = 14, + UNW_SPARC_O7 = 15, + UNW_SPARC_L0 = 16, + UNW_SPARC_L1 = 17, + UNW_SPARC_L2 = 18, + UNW_SPARC_L3 = 19, + UNW_SPARC_L4 = 20, + UNW_SPARC_L5 = 21, + UNW_SPARC_L6 = 22, + UNW_SPARC_L7 = 23, + UNW_SPARC_I0 = 24, + UNW_SPARC_I1 = 25, + UNW_SPARC_I2 = 26, + UNW_SPARC_I3 = 27, + UNW_SPARC_I4 = 28, + UNW_SPARC_I5 = 29, + UNW_SPARC_I6 = 30, + UNW_SPARC_I7 = 31, }; #endif Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/mach-o/compact_unwind_encoding.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/mach-o/compact_unwind_encoding.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/mach-o/compact_unwind_encoding.h (revision 345026) @@ -1,478 +1,478 @@ //===------------------ mach-o/compact_unwind_encoding.h ------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // -// Darwin's alternative to dwarf based unwind encodings. +// Darwin's alternative to DWARF based unwind encodings. // //===----------------------------------------------------------------------===// #ifndef __COMPACT_UNWIND_ENCODING__ #define __COMPACT_UNWIND_ENCODING__ #include // -// Compilers can emit standard Dwarf FDEs in the __TEXT,__eh_frame section +// Compilers can emit standard DWARF FDEs in the __TEXT,__eh_frame section // of object files. Or compilers can emit compact unwind information in // the __LD,__compact_unwind section. // // When the linker creates a final linked image, it will create a // __TEXT,__unwind_info section. This section is a small and fast way for the // runtime to access unwind info for any given function. If the compiler // emitted compact unwind info for the function, that compact unwind info will // be encoded in the __TEXT,__unwind_info section. If the compiler emitted -// dwarf unwind info, the __TEXT,__unwind_info section will contain the offset +// DWARF unwind info, the __TEXT,__unwind_info section will contain the offset // of the FDE in the __TEXT,__eh_frame section in the final linked image. // -// Note: Previously, the linker would transform some dwarf unwind infos into +// Note: Previously, the linker would transform some DWARF unwind infos into // compact unwind info. But that is fragile and no longer done. // // The compact unwind endoding is a 32-bit value which encoded in an // architecture specific way, which registers to restore from where, and how // to unwind out of the function. // typedef uint32_t compact_unwind_encoding_t; // architecture independent bits enum { UNWIND_IS_NOT_FUNCTION_START = 0x80000000, UNWIND_HAS_LSDA = 0x40000000, UNWIND_PERSONALITY_MASK = 0x30000000, }; // // x86 // // 1-bit: start // 1-bit: has lsda // 2-bit: personality index // -// 4-bits: 0=old, 1=ebp based, 2=stack-imm, 3=stack-ind, 4=dwarf +// 4-bits: 0=old, 1=ebp based, 2=stack-imm, 3=stack-ind, 4=DWARF // ebp based: // 15-bits (5*3-bits per reg) register permutation // 8-bits for stack offset // frameless: // 8-bits stack size // 3-bits stack adjust // 3-bits register count // 10-bits register permutation // enum { UNWIND_X86_MODE_MASK = 0x0F000000, UNWIND_X86_MODE_EBP_FRAME = 0x01000000, UNWIND_X86_MODE_STACK_IMMD = 0x02000000, UNWIND_X86_MODE_STACK_IND = 0x03000000, UNWIND_X86_MODE_DWARF = 0x04000000, UNWIND_X86_EBP_FRAME_REGISTERS = 0x00007FFF, UNWIND_X86_EBP_FRAME_OFFSET = 0x00FF0000, UNWIND_X86_FRAMELESS_STACK_SIZE = 0x00FF0000, UNWIND_X86_FRAMELESS_STACK_ADJUST = 0x0000E000, UNWIND_X86_FRAMELESS_STACK_REG_COUNT = 0x00001C00, UNWIND_X86_FRAMELESS_STACK_REG_PERMUTATION = 0x000003FF, UNWIND_X86_DWARF_SECTION_OFFSET = 0x00FFFFFF, }; enum { UNWIND_X86_REG_NONE = 0, UNWIND_X86_REG_EBX = 1, UNWIND_X86_REG_ECX = 2, UNWIND_X86_REG_EDX = 3, UNWIND_X86_REG_EDI = 4, UNWIND_X86_REG_ESI = 5, UNWIND_X86_REG_EBP = 6, }; // // For x86 there are four modes for the compact unwind encoding: // UNWIND_X86_MODE_EBP_FRAME: // EBP based frame where EBP is push on stack immediately after return address, // then ESP is moved to EBP. Thus, to unwind ESP is restored with the current // EPB value, then EBP is restored by popping off the stack, and the return // is done by popping the stack once more into the pc. // All non-volatile registers that need to be restored must have been saved // in a small range in the stack that starts EBP-4 to EBP-1020. The offset/4 // is encoded in the UNWIND_X86_EBP_FRAME_OFFSET bits. The registers saved // are encoded in the UNWIND_X86_EBP_FRAME_REGISTERS bits as five 3-bit entries. // Each entry contains which register to restore. // UNWIND_X86_MODE_STACK_IMMD: // A "frameless" (EBP not used as frame pointer) function with a small // constant stack size. To return, a constant (encoded in the compact // unwind encoding) is added to the ESP. Then the return is done by // popping the stack into the pc. // All non-volatile registers that need to be restored must have been saved // on the stack immediately after the return address. The stack_size/4 is // encoded in the UNWIND_X86_FRAMELESS_STACK_SIZE (max stack size is 1024). // The number of registers saved is encoded in UNWIND_X86_FRAMELESS_STACK_REG_COUNT. // UNWIND_X86_FRAMELESS_STACK_REG_PERMUTATION constains which registers were // saved and their order. // UNWIND_X86_MODE_STACK_IND: // A "frameless" (EBP not used as frame pointer) function large constant // stack size. This case is like the previous, except the stack size is too // large to encode in the compact unwind encoding. Instead it requires that // the function contains "subl $nnnnnnnn,ESP" in its prolog. The compact // encoding contains the offset to the nnnnnnnn value in the function in // UNWIND_X86_FRAMELESS_STACK_SIZE. // UNWIND_X86_MODE_DWARF: // No compact unwind encoding is available. Instead the low 24-bits of the -// compact encoding is the offset of the dwarf FDE in the __eh_frame section. +// compact encoding is the offset of the DWARF FDE in the __eh_frame section. // This mode is never used in object files. It is only generated by the -// linker in final linked images which have only dwarf unwind info for a +// linker in final linked images which have only DWARF unwind info for a // function. // // The permutation encoding is a Lehmer code sequence encoded into a // single variable-base number so we can encode the ordering of up to // six registers in a 10-bit space. // // The following is the algorithm used to create the permutation encoding used // with frameless stacks. It is passed the number of registers to be saved and // an array of the register numbers saved. // //uint32_t permute_encode(uint32_t registerCount, const uint32_t registers[6]) //{ // uint32_t renumregs[6]; // for (int i=6-registerCount; i < 6; ++i) { // int countless = 0; // for (int j=6-registerCount; j < i; ++j) { // if ( registers[j] < registers[i] ) // ++countless; // } // renumregs[i] = registers[i] - countless -1; // } // uint32_t permutationEncoding = 0; // switch ( registerCount ) { // case 6: // permutationEncoding |= (120*renumregs[0] + 24*renumregs[1] // + 6*renumregs[2] + 2*renumregs[3] // + renumregs[4]); // break; // case 5: // permutationEncoding |= (120*renumregs[1] + 24*renumregs[2] // + 6*renumregs[3] + 2*renumregs[4] // + renumregs[5]); // break; // case 4: // permutationEncoding |= (60*renumregs[2] + 12*renumregs[3] // + 3*renumregs[4] + renumregs[5]); // break; // case 3: // permutationEncoding |= (20*renumregs[3] + 4*renumregs[4] // + renumregs[5]); // break; // case 2: // permutationEncoding |= (5*renumregs[4] + renumregs[5]); // break; // case 1: // permutationEncoding |= (renumregs[5]); // break; // } // return permutationEncoding; //} // // // x86_64 // // 1-bit: start // 1-bit: has lsda // 2-bit: personality index // -// 4-bits: 0=old, 1=rbp based, 2=stack-imm, 3=stack-ind, 4=dwarf +// 4-bits: 0=old, 1=rbp based, 2=stack-imm, 3=stack-ind, 4=DWARF // rbp based: // 15-bits (5*3-bits per reg) register permutation // 8-bits for stack offset // frameless: // 8-bits stack size // 3-bits stack adjust // 3-bits register count // 10-bits register permutation // enum { UNWIND_X86_64_MODE_MASK = 0x0F000000, UNWIND_X86_64_MODE_RBP_FRAME = 0x01000000, UNWIND_X86_64_MODE_STACK_IMMD = 0x02000000, UNWIND_X86_64_MODE_STACK_IND = 0x03000000, UNWIND_X86_64_MODE_DWARF = 0x04000000, UNWIND_X86_64_RBP_FRAME_REGISTERS = 0x00007FFF, UNWIND_X86_64_RBP_FRAME_OFFSET = 0x00FF0000, UNWIND_X86_64_FRAMELESS_STACK_SIZE = 0x00FF0000, UNWIND_X86_64_FRAMELESS_STACK_ADJUST = 0x0000E000, UNWIND_X86_64_FRAMELESS_STACK_REG_COUNT = 0x00001C00, UNWIND_X86_64_FRAMELESS_STACK_REG_PERMUTATION = 0x000003FF, UNWIND_X86_64_DWARF_SECTION_OFFSET = 0x00FFFFFF, }; enum { UNWIND_X86_64_REG_NONE = 0, UNWIND_X86_64_REG_RBX = 1, UNWIND_X86_64_REG_R12 = 2, UNWIND_X86_64_REG_R13 = 3, UNWIND_X86_64_REG_R14 = 4, UNWIND_X86_64_REG_R15 = 5, UNWIND_X86_64_REG_RBP = 6, }; // // For x86_64 there are four modes for the compact unwind encoding: // UNWIND_X86_64_MODE_RBP_FRAME: // RBP based frame where RBP is push on stack immediately after return address, // then RSP is moved to RBP. Thus, to unwind RSP is restored with the current // EPB value, then RBP is restored by popping off the stack, and the return // is done by popping the stack once more into the pc. // All non-volatile registers that need to be restored must have been saved // in a small range in the stack that starts RBP-8 to RBP-2040. The offset/8 // is encoded in the UNWIND_X86_64_RBP_FRAME_OFFSET bits. The registers saved // are encoded in the UNWIND_X86_64_RBP_FRAME_REGISTERS bits as five 3-bit entries. // Each entry contains which register to restore. // UNWIND_X86_64_MODE_STACK_IMMD: // A "frameless" (RBP not used as frame pointer) function with a small // constant stack size. To return, a constant (encoded in the compact // unwind encoding) is added to the RSP. Then the return is done by // popping the stack into the pc. // All non-volatile registers that need to be restored must have been saved // on the stack immediately after the return address. The stack_size/8 is // encoded in the UNWIND_X86_64_FRAMELESS_STACK_SIZE (max stack size is 2048). // The number of registers saved is encoded in UNWIND_X86_64_FRAMELESS_STACK_REG_COUNT. // UNWIND_X86_64_FRAMELESS_STACK_REG_PERMUTATION constains which registers were // saved and their order. // UNWIND_X86_64_MODE_STACK_IND: // A "frameless" (RBP not used as frame pointer) function large constant // stack size. This case is like the previous, except the stack size is too // large to encode in the compact unwind encoding. Instead it requires that // the function contains "subq $nnnnnnnn,RSP" in its prolog. The compact // encoding contains the offset to the nnnnnnnn value in the function in // UNWIND_X86_64_FRAMELESS_STACK_SIZE. // UNWIND_X86_64_MODE_DWARF: // No compact unwind encoding is available. Instead the low 24-bits of the -// compact encoding is the offset of the dwarf FDE in the __eh_frame section. +// compact encoding is the offset of the DWARF FDE in the __eh_frame section. // This mode is never used in object files. It is only generated by the -// linker in final linked images which have only dwarf unwind info for a +// linker in final linked images which have only DWARF unwind info for a // function. // // ARM64 // // 1-bit: start // 1-bit: has lsda // 2-bit: personality index // -// 4-bits: 4=frame-based, 3=dwarf, 2=frameless +// 4-bits: 4=frame-based, 3=DWARF, 2=frameless // frameless: // 12-bits of stack size // frame-based: // 4-bits D reg pairs saved // 5-bits X reg pairs saved -// dwarf: -// 24-bits offset of dwarf FDE in __eh_frame section +// DWARF: +// 24-bits offset of DWARF FDE in __eh_frame section // enum { UNWIND_ARM64_MODE_MASK = 0x0F000000, UNWIND_ARM64_MODE_FRAMELESS = 0x02000000, UNWIND_ARM64_MODE_DWARF = 0x03000000, UNWIND_ARM64_MODE_FRAME = 0x04000000, UNWIND_ARM64_FRAME_X19_X20_PAIR = 0x00000001, UNWIND_ARM64_FRAME_X21_X22_PAIR = 0x00000002, UNWIND_ARM64_FRAME_X23_X24_PAIR = 0x00000004, UNWIND_ARM64_FRAME_X25_X26_PAIR = 0x00000008, UNWIND_ARM64_FRAME_X27_X28_PAIR = 0x00000010, UNWIND_ARM64_FRAME_D8_D9_PAIR = 0x00000100, UNWIND_ARM64_FRAME_D10_D11_PAIR = 0x00000200, UNWIND_ARM64_FRAME_D12_D13_PAIR = 0x00000400, UNWIND_ARM64_FRAME_D14_D15_PAIR = 0x00000800, UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK = 0x00FFF000, UNWIND_ARM64_DWARF_SECTION_OFFSET = 0x00FFFFFF, }; // For arm64 there are three modes for the compact unwind encoding: // UNWIND_ARM64_MODE_FRAME: // This is a standard arm64 prolog where FP/LR are immediately pushed on the // stack, then SP is copied to FP. If there are any non-volatile registers // saved, then are copied into the stack frame in pairs in a contiguous // range right below the saved FP/LR pair. Any subset of the five X pairs // and four D pairs can be saved, but the memory layout must be in register // number order. // UNWIND_ARM64_MODE_FRAMELESS: // A "frameless" leaf function, where FP/LR are not saved. The return address // remains in LR throughout the function. If any non-volatile registers // are saved, they must be pushed onto the stack before any stack space is // allocated for local variables. The stack sized (including any saved // non-volatile registers) divided by 16 is encoded in the bits // UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK. // UNWIND_ARM64_MODE_DWARF: // No compact unwind encoding is available. Instead the low 24-bits of the -// compact encoding is the offset of the dwarf FDE in the __eh_frame section. +// compact encoding is the offset of the DWARF FDE in the __eh_frame section. // This mode is never used in object files. It is only generated by the -// linker in final linked images which have only dwarf unwind info for a +// linker in final linked images which have only DWARF unwind info for a // function. // //////////////////////////////////////////////////////////////////////////////// // // Relocatable Object Files: __LD,__compact_unwind // //////////////////////////////////////////////////////////////////////////////// // // A compiler can generated compact unwind information for a function by adding // a "row" to the __LD,__compact_unwind section. This section has the // S_ATTR_DEBUG bit set, so the section will be ignored by older linkers. // It is removed by the new linker, so never ends up in final executables. // This section is a table, initially with one row per function (that needs // unwind info). The table columns and some conceptual entries are: // // range-start pointer to start of function/range // range-length // compact-unwind-encoding 32-bit encoding // personality-function or zero if no personality function // lsda or zero if no LSDA data // // The length and encoding fields are 32-bits. The other are all pointer sized. // // In x86_64 assembly, these entry would look like: // // .section __LD,__compact_unwind,regular,debug // // #compact unwind for _foo // .quad _foo // .set L1,LfooEnd-_foo // .long L1 // .long 0x01010001 // .quad 0 // .quad 0 // // #compact unwind for _bar // .quad _bar // .set L2,LbarEnd-_bar // .long L2 // .long 0x01020011 // .quad __gxx_personality // .quad except_tab1 // // // Notes: There is no need for any labels in the the __compact_unwind section. // The use of the .set directive is to force the evaluation of the // range-length at assembly time, instead of generating relocations. // // To support future compiler optimizations where which non-volatile registers // are saved changes within a function (e.g. delay saving non-volatiles until // necessary), there can by multiple lines in the __compact_unwind table for one // function, each with a different (non-overlapping) range and each with // different compact unwind encodings that correspond to the non-volatiles // saved at that range of the function. // // If a particular function is so wacky that there is no compact unwind way -// to encode it, then the compiler can emit traditional dwarf unwind info. +// to encode it, then the compiler can emit traditional DWARF unwind info. // The runtime will use which ever is available. // // Runtime support for compact unwind encodings are only available on 10.6 // and later. So, the compiler should not generate it when targeting pre-10.6. //////////////////////////////////////////////////////////////////////////////// // // Final Linked Images: __TEXT,__unwind_info // //////////////////////////////////////////////////////////////////////////////// // // The __TEXT,__unwind_info section is laid out for an efficient two level lookup. // The header of the section contains a coarse index that maps function address // to the page (4096 byte block) containing the unwind info for that function. // #define UNWIND_SECTION_VERSION 1 struct unwind_info_section_header { uint32_t version; // UNWIND_SECTION_VERSION uint32_t commonEncodingsArraySectionOffset; uint32_t commonEncodingsArrayCount; uint32_t personalityArraySectionOffset; uint32_t personalityArrayCount; uint32_t indexSectionOffset; uint32_t indexCount; // compact_unwind_encoding_t[] // uint32_t personalities[] // unwind_info_section_header_index_entry[] // unwind_info_section_header_lsda_index_entry[] }; struct unwind_info_section_header_index_entry { uint32_t functionOffset; uint32_t secondLevelPagesSectionOffset; // section offset to start of regular or compress page uint32_t lsdaIndexArraySectionOffset; // section offset to start of lsda_index array for this range }; struct unwind_info_section_header_lsda_index_entry { uint32_t functionOffset; uint32_t lsdaOffset; }; // // There are two kinds of second level index pages: regular and compressed. // A compressed page can hold up to 1021 entries, but it cannot be used // if too many different encoding types are used. The regular page holds // 511 entries. // struct unwind_info_regular_second_level_entry { uint32_t functionOffset; compact_unwind_encoding_t encoding; }; #define UNWIND_SECOND_LEVEL_REGULAR 2 struct unwind_info_regular_second_level_page_header { uint32_t kind; // UNWIND_SECOND_LEVEL_REGULAR uint16_t entryPageOffset; uint16_t entryCount; // entry array }; #define UNWIND_SECOND_LEVEL_COMPRESSED 3 struct unwind_info_compressed_second_level_page_header { uint32_t kind; // UNWIND_SECOND_LEVEL_COMPRESSED uint16_t entryPageOffset; uint16_t entryCount; uint16_t encodingsPageOffset; uint16_t encodingsCount; // 32-bit entry array // encodings array }; #define UNWIND_INFO_COMPRESSED_ENTRY_FUNC_OFFSET(entry) (entry & 0x00FFFFFF) #define UNWIND_INFO_COMPRESSED_ENTRY_ENCODING_INDEX(entry) ((entry >> 24) & 0xFF) #endif Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/unwind.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/unwind.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/include/unwind.h (revision 345026) @@ -1,372 +1,401 @@ //===------------------------------- unwind.h -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // C++ ABI Level 1 ABI documented at: // http://mentorembedded.github.io/cxx-abi/abi-eh.html // //===----------------------------------------------------------------------===// #ifndef __UNWIND_H__ #define __UNWIND_H__ #include <__libunwind_config.h> #include #include +#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) && defined(_WIN32) +#include +#include +#endif + #if defined(__APPLE__) #define LIBUNWIND_UNAVAIL __attribute__ (( unavailable )) #else #define LIBUNWIND_UNAVAIL #endif typedef enum { _URC_NO_REASON = 0, _URC_OK = 0, _URC_FOREIGN_EXCEPTION_CAUGHT = 1, _URC_FATAL_PHASE2_ERROR = 2, _URC_FATAL_PHASE1_ERROR = 3, _URC_NORMAL_STOP = 4, _URC_END_OF_STACK = 5, _URC_HANDLER_FOUND = 6, _URC_INSTALL_CONTEXT = 7, _URC_CONTINUE_UNWIND = 8, -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) _URC_FAILURE = 9 #endif } _Unwind_Reason_Code; typedef enum { _UA_SEARCH_PHASE = 1, _UA_CLEANUP_PHASE = 2, _UA_HANDLER_FRAME = 4, _UA_FORCE_UNWIND = 8, _UA_END_OF_STACK = 16 // gcc extension to C++ ABI } _Unwind_Action; typedef struct _Unwind_Context _Unwind_Context; // opaque -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) typedef uint32_t _Unwind_State; static const _Unwind_State _US_VIRTUAL_UNWIND_FRAME = 0; static const _Unwind_State _US_UNWIND_FRAME_STARTING = 1; static const _Unwind_State _US_UNWIND_FRAME_RESUME = 2; +static const _Unwind_State _US_ACTION_MASK = 3; /* Undocumented flag for force unwinding. */ static const _Unwind_State _US_FORCE_UNWIND = 8; typedef uint32_t _Unwind_EHT_Header; struct _Unwind_Control_Block; typedef struct _Unwind_Control_Block _Unwind_Control_Block; typedef struct _Unwind_Control_Block _Unwind_Exception; /* Alias */ struct _Unwind_Control_Block { uint64_t exception_class; void (*exception_cleanup)(_Unwind_Reason_Code, _Unwind_Control_Block*); /* Unwinder cache, private fields for the unwinder's use */ struct { uint32_t reserved1; /* init reserved1 to 0, then don't touch */ uint32_t reserved2; uint32_t reserved3; uint32_t reserved4; uint32_t reserved5; } unwinder_cache; /* Propagation barrier cache (valid after phase 1): */ struct { uint32_t sp; uint32_t bitpattern[5]; } barrier_cache; /* Cleanup cache (preserved over cleanup): */ struct { uint32_t bitpattern[4]; } cleanup_cache; /* Pr cache (for pr's benefit): */ struct { uint32_t fnstart; /* function start address */ _Unwind_EHT_Header* ehtp; /* pointer to EHT entry header word */ uint32_t additional; uint32_t reserved1; } pr_cache; long long int :0; /* Enforce the 8-byte alignment */ -}; +} __attribute__((__aligned__(8))); typedef _Unwind_Reason_Code (*_Unwind_Stop_Fn) (_Unwind_State state, _Unwind_Exception* exceptionObject, struct _Unwind_Context* context); typedef _Unwind_Reason_Code (*__personality_routine) (_Unwind_State state, _Unwind_Exception* exceptionObject, struct _Unwind_Context* context); #else struct _Unwind_Context; // opaque struct _Unwind_Exception; // forward declaration typedef struct _Unwind_Exception _Unwind_Exception; struct _Unwind_Exception { uint64_t exception_class; void (*exception_cleanup)(_Unwind_Reason_Code reason, _Unwind_Exception *exc); +#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) + uintptr_t private_[6]; +#else uintptr_t private_1; // non-zero means forced unwind uintptr_t private_2; // holds sp that phase1 found for phase2 to use -#ifndef __LP64__ - // The gcc implementation of _Unwind_Exception used attribute mode on the - // above fields which had the side effect of causing this whole struct to - // round up to 32 bytes in size. To be more explicit, we add pad fields - // added for binary compatibility. +#endif +#if __SIZEOF_POINTER__ == 4 + // The implementation of _Unwind_Exception uses an attribute mode on the + // above fields which has the side effect of causing this whole struct to + // round up to 32 bytes in size (48 with SEH). To be more explicit, we add + // pad fields added for binary compatibility. uint32_t reserved[3]; #endif + // The Itanium ABI requires that _Unwind_Exception objects are "double-word + // aligned". GCC has interpreted this to mean "use the maximum useful + // alignment for the target"; so do we. } __attribute__((__aligned__)); typedef _Unwind_Reason_Code (*_Unwind_Stop_Fn) (int version, _Unwind_Action actions, uint64_t exceptionClass, _Unwind_Exception* exceptionObject, struct _Unwind_Context* context, void* stop_parameter ); typedef _Unwind_Reason_Code (*__personality_routine) (int version, _Unwind_Action actions, uint64_t exceptionClass, _Unwind_Exception* exceptionObject, struct _Unwind_Context* context); #endif #ifdef __cplusplus extern "C" { #endif // // The following are the base functions documented by the C++ ABI // #ifdef __USING_SJLJ_EXCEPTIONS__ extern _Unwind_Reason_Code _Unwind_SjLj_RaiseException(_Unwind_Exception *exception_object); extern void _Unwind_SjLj_Resume(_Unwind_Exception *exception_object); #else extern _Unwind_Reason_Code _Unwind_RaiseException(_Unwind_Exception *exception_object); extern void _Unwind_Resume(_Unwind_Exception *exception_object); #endif extern void _Unwind_DeleteException(_Unwind_Exception *exception_object); -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) typedef enum { _UVRSC_CORE = 0, /* integer register */ _UVRSC_VFP = 1, /* vfp */ _UVRSC_WMMXD = 3, /* Intel WMMX data register */ _UVRSC_WMMXC = 4 /* Intel WMMX control register */ } _Unwind_VRS_RegClass; typedef enum { _UVRSD_UINT32 = 0, _UVRSD_VFPX = 1, _UVRSD_UINT64 = 3, _UVRSD_FLOAT = 4, _UVRSD_DOUBLE = 5 } _Unwind_VRS_DataRepresentation; typedef enum { _UVRSR_OK = 0, _UVRSR_NOT_IMPLEMENTED = 1, _UVRSR_FAILED = 2 } _Unwind_VRS_Result; extern void _Unwind_Complete(_Unwind_Exception* exception_object); extern _Unwind_VRS_Result _Unwind_VRS_Get(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t regno, _Unwind_VRS_DataRepresentation representation, void *valuep); extern _Unwind_VRS_Result _Unwind_VRS_Set(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t regno, _Unwind_VRS_DataRepresentation representation, void *valuep); extern _Unwind_VRS_Result _Unwind_VRS_Pop(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t discriminator, _Unwind_VRS_DataRepresentation representation); #endif -#if !_LIBUNWIND_ARM_EHABI +#if !defined(_LIBUNWIND_ARM_EHABI) extern uintptr_t _Unwind_GetGR(struct _Unwind_Context *context, int index); extern void _Unwind_SetGR(struct _Unwind_Context *context, int index, uintptr_t new_value); extern uintptr_t _Unwind_GetIP(struct _Unwind_Context *context); extern void _Unwind_SetIP(struct _Unwind_Context *, uintptr_t new_value); -#else // _LIBUNWIND_ARM_EHABI +#else // defined(_LIBUNWIND_ARM_EHABI) #if defined(_LIBUNWIND_UNWIND_LEVEL1_EXTERNAL_LINKAGE) #define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 extern #else #define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 static __inline__ #endif // These are de facto helper functions for ARM, which delegate the function // calls to _Unwind_VRS_Get/Set(). These are not a part of ARM EHABI // specification, thus these function MUST be inlined. Please don't replace // these with the "extern" function declaration; otherwise, the program // including this header won't be ABI compatible and will result in // link error when we are linking the program with libgcc. _LIBUNWIND_EXPORT_UNWIND_LEVEL1 uintptr_t _Unwind_GetGR(struct _Unwind_Context *context, int index) { uintptr_t value = 0; _Unwind_VRS_Get(context, _UVRSC_CORE, (uint32_t)index, _UVRSD_UINT32, &value); return value; } _LIBUNWIND_EXPORT_UNWIND_LEVEL1 void _Unwind_SetGR(struct _Unwind_Context *context, int index, uintptr_t value) { _Unwind_VRS_Set(context, _UVRSC_CORE, (uint32_t)index, _UVRSD_UINT32, &value); } _LIBUNWIND_EXPORT_UNWIND_LEVEL1 uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) { // remove the thumb-bit before returning return _Unwind_GetGR(context, 15) & (~(uintptr_t)0x1); } _LIBUNWIND_EXPORT_UNWIND_LEVEL1 void _Unwind_SetIP(struct _Unwind_Context *context, uintptr_t value) { uintptr_t thumb_bit = _Unwind_GetGR(context, 15) & ((uintptr_t)0x1); _Unwind_SetGR(context, 15, value | thumb_bit); } -#endif // _LIBUNWIND_ARM_EHABI +#endif // defined(_LIBUNWIND_ARM_EHABI) extern uintptr_t _Unwind_GetRegionStart(struct _Unwind_Context *context); extern uintptr_t _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context); #ifdef __USING_SJLJ_EXCEPTIONS__ extern _Unwind_Reason_Code _Unwind_SjLj_ForcedUnwind(_Unwind_Exception *exception_object, _Unwind_Stop_Fn stop, void *stop_parameter); #else extern _Unwind_Reason_Code _Unwind_ForcedUnwind(_Unwind_Exception *exception_object, _Unwind_Stop_Fn stop, void *stop_parameter); #endif #ifdef __USING_SJLJ_EXCEPTIONS__ typedef struct _Unwind_FunctionContext *_Unwind_FunctionContext_t; extern void _Unwind_SjLj_Register(_Unwind_FunctionContext_t fc); extern void _Unwind_SjLj_Unregister(_Unwind_FunctionContext_t fc); #endif // // The following are semi-suppoted extensions to the C++ ABI // // // called by __cxa_rethrow(). // #ifdef __USING_SJLJ_EXCEPTIONS__ extern _Unwind_Reason_Code _Unwind_SjLj_Resume_or_Rethrow(_Unwind_Exception *exception_object); #else extern _Unwind_Reason_Code _Unwind_Resume_or_Rethrow(_Unwind_Exception *exception_object); #endif // _Unwind_Backtrace() is a gcc extension that walks the stack and calls the // _Unwind_Trace_Fn once per frame until it reaches the bottom of the stack // or the _Unwind_Trace_Fn function returns something other than _URC_NO_REASON. typedef _Unwind_Reason_Code (*_Unwind_Trace_Fn)(struct _Unwind_Context *, void *); extern _Unwind_Reason_Code _Unwind_Backtrace(_Unwind_Trace_Fn, void *); // _Unwind_GetCFA is a gcc extension that can be called from within a // personality handler to get the CFA (stack pointer before call) of // current frame. extern uintptr_t _Unwind_GetCFA(struct _Unwind_Context *); // _Unwind_GetIPInfo is a gcc extension that can be called from within a // personality handler. Similar to _Unwind_GetIP() but also returns in // *ipBefore a non-zero value if the instruction pointer is at or before the // instruction causing the unwind. Normally, in a function call, the IP returned // is the return address which is after the call instruction and may be past the // end of the function containing the call instruction. extern uintptr_t _Unwind_GetIPInfo(struct _Unwind_Context *context, int *ipBefore); // __register_frame() is used with dynamically generated code to register the // FDE for a generated (JIT) code. The FDE must use pc-rel addressing to point // to its function and optional LSDA. // __register_frame() has existed in all versions of Mac OS X, but in 10.4 and // 10.5 it was buggy and did not actually register the FDE with the unwinder. // In 10.6 and later it does register properly. extern void __register_frame(const void *fde); extern void __deregister_frame(const void *fde); // _Unwind_Find_FDE() will locate the FDE if the pc is in some function that has // an associated FDE. Note, Mac OS X 10.6 and later, introduces "compact unwind -// info" which the runtime uses in preference to dwarf unwind info. This +// info" which the runtime uses in preference to DWARF unwind info. This // function will only work if the target function has an FDE but no compact // unwind info. struct dwarf_eh_bases { uintptr_t tbase; uintptr_t dbase; uintptr_t func; }; extern const void *_Unwind_Find_FDE(const void *pc, struct dwarf_eh_bases *); // This function attempts to find the start (address of first instruction) of // a function given an address inside the function. It only works if the -// function has an FDE (dwarf unwind info). +// function has an FDE (DWARF unwind info). // This function is unimplemented on Mac OS X 10.6 and later. Instead, use // _Unwind_Find_FDE() and look at the dwarf_eh_bases.func result. extern void *_Unwind_FindEnclosingFunction(void *pc); // Mac OS X does not support text-rel and data-rel addressing so these functions // are unimplemented extern uintptr_t _Unwind_GetDataRelBase(struct _Unwind_Context *context) LIBUNWIND_UNAVAIL; extern uintptr_t _Unwind_GetTextRelBase(struct _Unwind_Context *context) LIBUNWIND_UNAVAIL; // Mac OS X 10.4 and 10.5 had implementations of these functions in // libgcc_s.dylib, but they never worked. /// These functions are no longer available on Mac OS X. extern void __register_frame_info_bases(const void *fde, void *ob, void *tb, void *db) LIBUNWIND_UNAVAIL; extern void __register_frame_info(const void *fde, void *ob) LIBUNWIND_UNAVAIL; extern void __register_frame_info_table_bases(const void *fde, void *ob, void *tb, void *db) LIBUNWIND_UNAVAIL; extern void __register_frame_info_table(const void *fde, void *ob) LIBUNWIND_UNAVAIL; extern void __register_frame_table(const void *fde) LIBUNWIND_UNAVAIL; extern void *__deregister_frame_info(const void *fde) LIBUNWIND_UNAVAIL; extern void *__deregister_frame_info_bases(const void *fde) LIBUNWIND_UNAVAIL; + +#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) +#ifndef _WIN32 +typedef struct _EXCEPTION_RECORD EXCEPTION_RECORD; +typedef struct _CONTEXT CONTEXT; +typedef struct _DISPATCHER_CONTEXT DISPATCHER_CONTEXT; +#elif !defined(__MINGW32__) && VER_PRODUCTBUILD < 8000 +typedef struct _DISPATCHER_CONTEXT DISPATCHER_CONTEXT; +#endif +// This is the common wrapper for GCC-style personality functions with SEH. +extern EXCEPTION_DISPOSITION _GCC_specific_handler(EXCEPTION_RECORD *exc, + void *frame, + CONTEXT *ctx, + DISPATCHER_CONTEXT *disp, + __personality_routine pers); +#endif #ifdef __cplusplus } #endif #endif // __UNWIND_H__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/unwind_ext.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/unwind_ext.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/unwind_ext.h (nonexistent) @@ -1,37 +0,0 @@ -//===-------------------------- unwind_ext.h ------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. -// -// -// Extensions to unwind API. -// -//===----------------------------------------------------------------------===// - -#ifndef __UNWIND_EXT__ -#define __UNWIND_EXT__ - -#include "unwind.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// These platform specific functions to get and set the top context are -// implemented elsewhere. - -extern struct _Unwind_FunctionContext * -__Unwind_SjLj_GetTopOfFunctionStack(); - -extern void -__Unwind_SjLj_SetTopOfFunctionStack(struct _Unwind_FunctionContext *fc); - -#ifdef __cplusplus -} -#endif - -#endif // __UNWIND_EXT__ - - Property changes on: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/unwind_ext.h ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/AddressSpace.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/AddressSpace.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/AddressSpace.hpp (revision 345026) @@ -1,614 +1,738 @@ //===------------------------- AddressSpace.hpp ---------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Abstracts accessing local vs remote address spaces. // //===----------------------------------------------------------------------===// #ifndef __ADDRESSSPACE_HPP__ #define __ADDRESSSPACE_HPP__ #include #include #include #include -#ifndef _LIBUNWIND_IS_BAREMETAL +#ifndef _LIBUNWIND_USE_DLADDR + #if !defined(_LIBUNWIND_IS_BAREMETAL) && !defined(_WIN32) + #define _LIBUNWIND_USE_DLADDR 1 + #else + #define _LIBUNWIND_USE_DLADDR 0 + #endif +#endif + +#if _LIBUNWIND_USE_DLADDR #include #endif #ifdef __APPLE__ #include namespace libunwind { bool checkKeyMgrRegisteredFDEs(uintptr_t targetAddr, void *&fde); } #endif #include "libunwind.h" #include "config.h" #include "dwarf2.h" +#include "EHHeaderParser.hpp" #include "Registers.hpp" -#if _LIBUNWIND_ARM_EHABI -#if defined(__FreeBSD__) || defined(__NetBSD__) +#ifdef __APPLE__ -#include -typedef void *_Unwind_Ptr; + struct dyld_unwind_sections + { + const struct mach_header* mh; + const void* dwarf_section; + uintptr_t dwarf_section_length; + const void* compact_unwind_section; + uintptr_t compact_unwind_section_length; + }; + #if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) \ + && (__MAC_OS_X_VERSION_MIN_REQUIRED >= 1070)) \ + || defined(__IPHONE_OS_VERSION_MIN_REQUIRED) + // In 10.7.0 or later, libSystem.dylib implements this function. + extern "C" bool _dyld_find_unwind_sections(void *, dyld_unwind_sections *); + #else + // In 10.6.x and earlier, we need to implement this functionality. Note + // that this requires a newer version of libmacho (from cctools) than is + // present in libSystem on 10.6.x (for getsectiondata). + static inline bool _dyld_find_unwind_sections(void* addr, + dyld_unwind_sections* info) { + // Find mach-o image containing address. + Dl_info dlinfo; + if (!dladdr(addr, &dlinfo)) + return false; +#if __LP64__ + const struct mach_header_64 *mh = (const struct mach_header_64 *)dlinfo.dli_fbase; +#else + const struct mach_header *mh = (const struct mach_header *)dlinfo.dli_fbase; +#endif -#elif defined(__linux__) + // Initialize the return struct + info->mh = (const struct mach_header *)mh; + info->dwarf_section = getsectiondata(mh, "__TEXT", "__eh_frame", &info->dwarf_section_length); + info->compact_unwind_section = getsectiondata(mh, "__TEXT", "__unwind_info", &info->compact_unwind_section_length); -typedef long unsigned int *_Unwind_Ptr; -extern "C" _Unwind_Ptr __gnu_Unwind_Find_exidx(_Unwind_Ptr addr, int *len); + if (!info->dwarf_section) { + info->dwarf_section_length = 0; + } -// Emulate the BSD dl_unwind_find_exidx API when on a GNU libdl system. -#define dl_unwind_find_exidx __gnu_Unwind_Find_exidx + if (!info->compact_unwind_section) { + info->compact_unwind_section_length = 0; + } -#elif !defined(_LIBUNWIND_IS_BAREMETAL) -#include -#else // !defined(_LIBUNWIND_IS_BAREMETAL) + return true; + } + #endif + +#elif defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) && defined(_LIBUNWIND_IS_BAREMETAL) + // When statically linked on bare-metal, the symbols for the EH table are looked // up without going through the dynamic loader. -struct EHTEntry { - uint32_t functionOffset; - uint32_t unwindOpcodes; -}; -extern EHTEntry __exidx_start; -extern EHTEntry __exidx_end; -#endif // !defined(_LIBUNWIND_IS_BAREMETAL) -#endif // _LIBUNWIND_ARM_EHABI -#if defined(__CloudABI__) || defined(__FreeBSD__) || defined(__linux__) || \ - defined(__NetBSD__) -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND && _LIBUNWIND_SUPPORT_DWARF_INDEX +// The following linker script may be used to produce the necessary sections and symbols. +// Unless the --eh-frame-hdr linker option is provided, the section is not generated +// and does not take space in the output file. +// +// .eh_frame : +// { +// __eh_frame_start = .; +// KEEP(*(.eh_frame)) +// __eh_frame_end = .; +// } +// +// .eh_frame_hdr : +// { +// KEEP(*(.eh_frame_hdr)) +// } +// +// __eh_frame_hdr_start = SIZEOF(.eh_frame_hdr) > 0 ? ADDR(.eh_frame_hdr) : 0; +// __eh_frame_hdr_end = SIZEOF(.eh_frame_hdr) > 0 ? . : 0; + +extern char __eh_frame_start; +extern char __eh_frame_end; + +#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) +extern char __eh_frame_hdr_start; +extern char __eh_frame_hdr_end; +#endif + +#elif defined(_LIBUNWIND_ARM_EHABI) && defined(_LIBUNWIND_IS_BAREMETAL) + +// When statically linked on bare-metal, the symbols for the EH table are looked +// up without going through the dynamic loader. +extern char __exidx_start; +extern char __exidx_end; + +#elif defined(_LIBUNWIND_ARM_EHABI) || defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) + +// ELF-based systems may use dl_iterate_phdr() to access sections +// containing unwinding information. The ElfW() macro for pointer-size +// independent ELF header traversal is not provided by on some +// systems (e.g., FreeBSD). On these systems the data structures are +// just called Elf_XXX. Define ElfW() locally. +#ifndef _WIN32 #include -// Macro for machine-independent access to the ELF program headers. This -// macro is not available on some systems (e.g., FreeBSD). On these -// systems the data structures are just called Elf_XXX. Define ElfW() -// locally. +#else +#include +#include +#endif #if !defined(ElfW) #define ElfW(type) Elf_##type #endif -#include "EHHeaderParser.hpp" + #endif -#endif namespace libunwind { /// Used by findUnwindSections() to return info about needed sections. struct UnwindInfoSections { -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND || _LIBUNWIND_SUPPORT_DWARF_INDEX || \ - _LIBUNWIND_SUPPORT_COMPACT_UNWIND - // No dso_base for ARM EHABI. +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) || defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) || \ + defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) + // No dso_base for SEH or ARM EHABI. uintptr_t dso_base; #endif -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) uintptr_t dwarf_section; uintptr_t dwarf_section_length; #endif -#if _LIBUNWIND_SUPPORT_DWARF_INDEX +#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) uintptr_t dwarf_index_section; uintptr_t dwarf_index_section_length; #endif -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) uintptr_t compact_unwind_section; uintptr_t compact_unwind_section_length; #endif -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) uintptr_t arm_section; uintptr_t arm_section_length; #endif }; /// LocalAddressSpace is used as a template parameter to UnwindCursor when /// unwinding a thread in the same process. The wrappers compile away, /// making local unwinds fast. -class __attribute__((visibility("hidden"))) LocalAddressSpace { +class _LIBUNWIND_HIDDEN LocalAddressSpace { public: -#ifdef __LP64__ - typedef uint64_t pint_t; - typedef int64_t sint_t; -#else - typedef uint32_t pint_t; - typedef int32_t sint_t; -#endif + typedef uintptr_t pint_t; + typedef intptr_t sint_t; uint8_t get8(pint_t addr) { uint8_t val; memcpy(&val, (void *)addr, sizeof(val)); return val; } uint16_t get16(pint_t addr) { uint16_t val; memcpy(&val, (void *)addr, sizeof(val)); return val; } uint32_t get32(pint_t addr) { uint32_t val; memcpy(&val, (void *)addr, sizeof(val)); return val; } uint64_t get64(pint_t addr) { uint64_t val; memcpy(&val, (void *)addr, sizeof(val)); return val; } double getDouble(pint_t addr) { double val; memcpy(&val, (void *)addr, sizeof(val)); return val; } v128 getVector(pint_t addr) { v128 val; memcpy(&val, (void *)addr, sizeof(val)); return val; } uintptr_t getP(pint_t addr); uint64_t getRegister(pint_t addr); static uint64_t getULEB128(pint_t &addr, pint_t end); static int64_t getSLEB128(pint_t &addr, pint_t end); pint_t getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, pint_t datarelBase = 0); bool findFunctionName(pint_t addr, char *buf, size_t bufLen, unw_word_t *offset); bool findUnwindSections(pint_t targetAddr, UnwindInfoSections &info); bool findOtherFDE(pint_t targetAddr, pint_t &fde); static LocalAddressSpace sThisAddressSpace; }; inline uintptr_t LocalAddressSpace::getP(pint_t addr) { -#ifdef __LP64__ +#if __SIZEOF_POINTER__ == 8 return get64(addr); #else return get32(addr); #endif } inline uint64_t LocalAddressSpace::getRegister(pint_t addr) { -#if defined(__LP64__) || defined(__mips64) +#if __SIZEOF_POINTER__ == 8 || defined(__mips64) return get64(addr); #else return get32(addr); #endif } /// Read a ULEB128 into a 64-bit word. inline uint64_t LocalAddressSpace::getULEB128(pint_t &addr, pint_t end) { const uint8_t *p = (uint8_t *)addr; const uint8_t *pend = (uint8_t *)end; uint64_t result = 0; int bit = 0; do { uint64_t b; if (p == pend) _LIBUNWIND_ABORT("truncated uleb128 expression"); b = *p & 0x7f; if (bit >= 64 || b << bit >> bit != b) { _LIBUNWIND_ABORT("malformed uleb128 expression"); } else { result |= b << bit; bit += 7; } } while (*p++ >= 0x80); addr = (pint_t) p; return result; } /// Read a SLEB128 into a 64-bit word. inline int64_t LocalAddressSpace::getSLEB128(pint_t &addr, pint_t end) { const uint8_t *p = (uint8_t *)addr; const uint8_t *pend = (uint8_t *)end; int64_t result = 0; int bit = 0; uint8_t byte; do { if (p == pend) _LIBUNWIND_ABORT("truncated sleb128 expression"); byte = *p++; result |= ((byte & 0x7f) << bit); bit += 7; } while (byte & 0x80); // sign extend negative numbers if ((byte & 0x40) != 0) - result |= (-1LL) << bit; + result |= (-1ULL) << bit; addr = (pint_t) p; return result; } inline LocalAddressSpace::pint_t LocalAddressSpace::getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, pint_t datarelBase) { pint_t startAddr = addr; const uint8_t *p = (uint8_t *)addr; pint_t result; // first get value switch (encoding & 0x0F) { case DW_EH_PE_ptr: result = getP(addr); p += sizeof(pint_t); addr = (pint_t) p; break; case DW_EH_PE_uleb128: result = (pint_t)getULEB128(addr, end); break; case DW_EH_PE_udata2: result = get16(addr); p += 2; addr = (pint_t) p; break; case DW_EH_PE_udata4: result = get32(addr); p += 4; addr = (pint_t) p; break; case DW_EH_PE_udata8: result = (pint_t)get64(addr); p += 8; addr = (pint_t) p; break; case DW_EH_PE_sleb128: result = (pint_t)getSLEB128(addr, end); break; case DW_EH_PE_sdata2: // Sign extend from signed 16-bit value. result = (pint_t)(int16_t)get16(addr); p += 2; addr = (pint_t) p; break; case DW_EH_PE_sdata4: // Sign extend from signed 32-bit value. result = (pint_t)(int32_t)get32(addr); p += 4; addr = (pint_t) p; break; case DW_EH_PE_sdata8: result = (pint_t)get64(addr); p += 8; addr = (pint_t) p; break; default: _LIBUNWIND_ABORT("unknown pointer encoding"); } // then add relative offset switch (encoding & 0x70) { case DW_EH_PE_absptr: // do nothing break; case DW_EH_PE_pcrel: result += startAddr; break; case DW_EH_PE_textrel: _LIBUNWIND_ABORT("DW_EH_PE_textrel pointer encoding not supported"); break; case DW_EH_PE_datarel: // DW_EH_PE_datarel is only valid in a few places, so the parameter has a // default value of 0, and we abort in the event that someone calls this // function with a datarelBase of 0 and DW_EH_PE_datarel encoding. if (datarelBase == 0) _LIBUNWIND_ABORT("DW_EH_PE_datarel is invalid with a datarelBase of 0"); result += datarelBase; break; case DW_EH_PE_funcrel: _LIBUNWIND_ABORT("DW_EH_PE_funcrel pointer encoding not supported"); break; case DW_EH_PE_aligned: _LIBUNWIND_ABORT("DW_EH_PE_aligned pointer encoding not supported"); break; default: _LIBUNWIND_ABORT("unknown pointer encoding"); break; } if (encoding & DW_EH_PE_indirect) result = getP(result); return result; } -#ifdef __APPLE__ - struct dyld_unwind_sections - { - const struct mach_header* mh; - const void* dwarf_section; - uintptr_t dwarf_section_length; - const void* compact_unwind_section; - uintptr_t compact_unwind_section_length; - }; - #if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) \ - && (__MAC_OS_X_VERSION_MIN_REQUIRED >= 1070)) \ - || defined(__IPHONE_OS_VERSION_MIN_REQUIRED) - // In 10.7.0 or later, libSystem.dylib implements this function. - extern "C" bool _dyld_find_unwind_sections(void *, dyld_unwind_sections *); - #else - // In 10.6.x and earlier, we need to implement this functionality. - static inline bool _dyld_find_unwind_sections(void* addr, - dyld_unwind_sections* info) { - // Find mach-o image containing address. - Dl_info dlinfo; - if (!dladdr(addr, &dlinfo)) - return false; - const mach_header *mh = (const mach_header *)dlinfo.dli_saddr; - - // Find dwarf unwind section in that image. - unsigned long size; - const uint8_t *p = getsectiondata(mh, "__TEXT", "__eh_frame", &size); - if (!p) - return false; - - // Fill in return struct. - info->mh = mh; - info->dwarf_section = p; - info->dwarf_section_length = size; - info->compact_unwind_section = 0; - info->compact_unwind_section_length = 0; - - return true; - } - #endif -#endif - inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr, UnwindInfoSections &info) { #ifdef __APPLE__ dyld_unwind_sections dyldInfo; if (_dyld_find_unwind_sections((void *)targetAddr, &dyldInfo)) { info.dso_base = (uintptr_t)dyldInfo.mh; - #if _LIBUNWIND_SUPPORT_DWARF_UNWIND + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) info.dwarf_section = (uintptr_t)dyldInfo.dwarf_section; info.dwarf_section_length = dyldInfo.dwarf_section_length; #endif info.compact_unwind_section = (uintptr_t)dyldInfo.compact_unwind_section; info.compact_unwind_section_length = dyldInfo.compact_unwind_section_length; return true; } -#elif _LIBUNWIND_ARM_EHABI - #ifdef _LIBUNWIND_IS_BAREMETAL +#elif defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) && defined(_LIBUNWIND_IS_BAREMETAL) // Bare metal is statically linked, so no need to ask the dynamic loader + info.dwarf_section_length = (uintptr_t)(&__eh_frame_end - &__eh_frame_start); + info.dwarf_section = (uintptr_t)(&__eh_frame_start); + _LIBUNWIND_TRACE_UNWINDING("findUnwindSections: section %p length %p", + (void *)info.dwarf_section, (void *)info.dwarf_section_length); +#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) + info.dwarf_index_section = (uintptr_t)(&__eh_frame_hdr_start); + info.dwarf_index_section_length = (uintptr_t)(&__eh_frame_hdr_end - &__eh_frame_hdr_start); + _LIBUNWIND_TRACE_UNWINDING("findUnwindSections: index section %p length %p", + (void *)info.dwarf_index_section, (void *)info.dwarf_index_section_length); +#endif + if (info.dwarf_section_length) + return true; +#elif defined(_LIBUNWIND_ARM_EHABI) && defined(_LIBUNWIND_IS_BAREMETAL) + // Bare metal is statically linked, so no need to ask the dynamic loader info.arm_section = (uintptr_t)(&__exidx_start); info.arm_section_length = (uintptr_t)(&__exidx_end - &__exidx_start); - #else + _LIBUNWIND_TRACE_UNWINDING("findUnwindSections: section %p length %p", + (void *)info.arm_section, (void *)info.arm_section_length); + if (info.arm_section && info.arm_section_length) + return true; +#elif defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) && defined(_WIN32) + HMODULE mods[1024]; + HANDLE process = GetCurrentProcess(); + DWORD needed; + + if (!EnumProcessModules(process, mods, sizeof(mods), &needed)) + return false; + + for (unsigned i = 0; i < (needed / sizeof(HMODULE)); i++) { + PIMAGE_DOS_HEADER pidh = (PIMAGE_DOS_HEADER)mods[i]; + PIMAGE_NT_HEADERS pinh = (PIMAGE_NT_HEADERS)((BYTE *)pidh + pidh->e_lfanew); + PIMAGE_FILE_HEADER pifh = (PIMAGE_FILE_HEADER)&pinh->FileHeader; + PIMAGE_SECTION_HEADER pish = IMAGE_FIRST_SECTION(pinh); + bool found_obj = false; + bool found_hdr = false; + + info.dso_base = (uintptr_t)mods[i]; + for (unsigned j = 0; j < pifh->NumberOfSections; j++, pish++) { + uintptr_t begin = pish->VirtualAddress + (uintptr_t)mods[i]; + uintptr_t end = begin + pish->Misc.VirtualSize; + if (!strncmp((const char *)pish->Name, ".text", + IMAGE_SIZEOF_SHORT_NAME)) { + if (targetAddr >= begin && targetAddr < end) + found_obj = true; + } else if (!strncmp((const char *)pish->Name, ".eh_frame", + IMAGE_SIZEOF_SHORT_NAME)) { + info.dwarf_section = begin; + info.dwarf_section_length = pish->Misc.VirtualSize; + found_hdr = true; + } + if (found_obj && found_hdr) + return true; + } + } + return false; +#elif defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) && defined(_WIN32) + // Don't even bother, since Windows has functions that do all this stuff + // for us. + return true; +#elif defined(_LIBUNWIND_ARM_EHABI) && defined(__BIONIC__) && \ + (__ANDROID_API__ < 21) int length = 0; - info.arm_section = (uintptr_t) dl_unwind_find_exidx( - (_Unwind_Ptr) targetAddr, &length); + info.arm_section = + (uintptr_t)dl_unwind_find_exidx((_Unwind_Ptr)targetAddr, &length); info.arm_section_length = (uintptr_t)length; - #endif - _LIBUNWIND_TRACE_UNWINDING("findUnwindSections: section %X length %x", - info.arm_section, info.arm_section_length); if (info.arm_section && info.arm_section_length) return true; -#elif _LIBUNWIND_SUPPORT_DWARF_UNWIND -#if _LIBUNWIND_SUPPORT_DWARF_INDEX +#elif defined(_LIBUNWIND_ARM_EHABI) || defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) struct dl_iterate_cb_data { LocalAddressSpace *addressSpace; UnwindInfoSections *sects; uintptr_t targetAddr; }; dl_iterate_cb_data cb_data = {this, &info, targetAddr}; int found = dl_iterate_phdr( [](struct dl_phdr_info *pinfo, size_t, void *data) -> int { auto cbdata = static_cast(data); - size_t object_length; bool found_obj = false; bool found_hdr = false; assert(cbdata); assert(cbdata->sects); if (cbdata->targetAddr < pinfo->dlpi_addr) { return false; } #if !defined(Elf_Half) typedef ElfW(Half) Elf_Half; #endif #if !defined(Elf_Phdr) typedef ElfW(Phdr) Elf_Phdr; #endif +#if !defined(Elf_Addr) && defined(__ANDROID__) + typedef ElfW(Addr) Elf_Addr; +#endif + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) + #if !defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) + #error "_LIBUNWIND_SUPPORT_DWARF_UNWIND requires _LIBUNWIND_SUPPORT_DWARF_INDEX on this platform." + #endif + size_t object_length; +#if defined(__ANDROID__) + Elf_Addr image_base = + pinfo->dlpi_phnum + ? reinterpret_cast(pinfo->dlpi_phdr) - + reinterpret_cast(pinfo->dlpi_phdr) + ->p_offset + : 0; +#endif + for (Elf_Half i = 0; i < pinfo->dlpi_phnum; i++) { const Elf_Phdr *phdr = &pinfo->dlpi_phdr[i]; if (phdr->p_type == PT_LOAD) { uintptr_t begin = pinfo->dlpi_addr + phdr->p_vaddr; +#if defined(__ANDROID__) + if (pinfo->dlpi_addr == 0 && phdr->p_vaddr < image_base) + begin = begin + image_base; +#endif uintptr_t end = begin + phdr->p_memsz; if (cbdata->targetAddr >= begin && cbdata->targetAddr < end) { cbdata->sects->dso_base = begin; object_length = phdr->p_memsz; found_obj = true; } } else if (phdr->p_type == PT_GNU_EH_FRAME) { EHHeaderParser::EHHeaderInfo hdrInfo; uintptr_t eh_frame_hdr_start = pinfo->dlpi_addr + phdr->p_vaddr; +#if defined(__ANDROID__) + if (pinfo->dlpi_addr == 0 && phdr->p_vaddr < image_base) + eh_frame_hdr_start = eh_frame_hdr_start + image_base; +#endif cbdata->sects->dwarf_index_section = eh_frame_hdr_start; cbdata->sects->dwarf_index_section_length = phdr->p_memsz; - EHHeaderParser::decodeEHHdr( + found_hdr = EHHeaderParser::decodeEHHdr( *cbdata->addressSpace, eh_frame_hdr_start, phdr->p_memsz, hdrInfo); - cbdata->sects->dwarf_section = hdrInfo.eh_frame_ptr; - found_hdr = true; + if (found_hdr) + cbdata->sects->dwarf_section = hdrInfo.eh_frame_ptr; } } if (found_obj && found_hdr) { cbdata->sects->dwarf_section_length = object_length; return true; } else { return false; } + #else // defined(_LIBUNWIND_ARM_EHABI) + for (Elf_Half i = 0; i < pinfo->dlpi_phnum; i++) { + const Elf_Phdr *phdr = &pinfo->dlpi_phdr[i]; + if (phdr->p_type == PT_LOAD) { + uintptr_t begin = pinfo->dlpi_addr + phdr->p_vaddr; + uintptr_t end = begin + phdr->p_memsz; + if (cbdata->targetAddr >= begin && cbdata->targetAddr < end) + found_obj = true; + } else if (phdr->p_type == PT_ARM_EXIDX) { + uintptr_t exidx_start = pinfo->dlpi_addr + phdr->p_vaddr; + cbdata->sects->arm_section = exidx_start; + cbdata->sects->arm_section_length = phdr->p_memsz; + found_hdr = true; + } + } + return found_obj && found_hdr; + #endif }, &cb_data); return static_cast(found); -#else -#error "_LIBUNWIND_SUPPORT_DWARF_UNWIND requires _LIBUNWIND_SUPPORT_DWARF_INDEX on this platform." #endif -#endif return false; } inline bool LocalAddressSpace::findOtherFDE(pint_t targetAddr, pint_t &fde) { #ifdef __APPLE__ return checkKeyMgrRegisteredFDEs(targetAddr, *((void**)&fde)); #else // TO DO: if OS has way to dynamically register FDEs, check that. (void)targetAddr; (void)fde; return false; #endif } inline bool LocalAddressSpace::findFunctionName(pint_t addr, char *buf, size_t bufLen, unw_word_t *offset) { -#ifndef _LIBUNWIND_IS_BAREMETAL +#if _LIBUNWIND_USE_DLADDR Dl_info dyldInfo; if (dladdr((void *)addr, &dyldInfo)) { if (dyldInfo.dli_sname != NULL) { snprintf(buf, bufLen, "%s", dyldInfo.dli_sname); *offset = (addr - (pint_t) dyldInfo.dli_saddr); return true; } } #endif return false; } #ifdef UNW_REMOTE -/// OtherAddressSpace is used as a template parameter to UnwindCursor when +/// RemoteAddressSpace is used as a template parameter to UnwindCursor when /// unwinding a thread in the another process. The other process can be a /// different endianness and a different pointer size which is handled by /// the P template parameter. template -class OtherAddressSpace { +class RemoteAddressSpace { public: - OtherAddressSpace(task_t task) : fTask(task) {} + RemoteAddressSpace(task_t task) : fTask(task) {} typedef typename P::uint_t pint_t; uint8_t get8(pint_t addr); uint16_t get16(pint_t addr); uint32_t get32(pint_t addr); uint64_t get64(pint_t addr); pint_t getP(pint_t addr); uint64_t getRegister(pint_t addr); uint64_t getULEB128(pint_t &addr, pint_t end); int64_t getSLEB128(pint_t &addr, pint_t end); pint_t getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, pint_t datarelBase = 0); bool findFunctionName(pint_t addr, char *buf, size_t bufLen, unw_word_t *offset); bool findUnwindSections(pint_t targetAddr, UnwindInfoSections &info); bool findOtherFDE(pint_t targetAddr, pint_t &fde); private: void *localCopy(pint_t addr); task_t fTask; }; -template uint8_t OtherAddressSpace

::get8(pint_t addr) { +template uint8_t RemoteAddressSpace

::get8(pint_t addr) { return *((uint8_t *)localCopy(addr)); } -template uint16_t OtherAddressSpace

::get16(pint_t addr) { +template uint16_t RemoteAddressSpace

::get16(pint_t addr) { return P::E::get16(*(uint16_t *)localCopy(addr)); } -template uint32_t OtherAddressSpace

::get32(pint_t addr) { +template uint32_t RemoteAddressSpace

::get32(pint_t addr) { return P::E::get32(*(uint32_t *)localCopy(addr)); } -template uint64_t OtherAddressSpace

::get64(pint_t addr) { +template uint64_t RemoteAddressSpace

::get64(pint_t addr) { return P::E::get64(*(uint64_t *)localCopy(addr)); } template -typename P::uint_t OtherAddressSpace

::getP(pint_t addr) { +typename P::uint_t RemoteAddressSpace

::getP(pint_t addr) { return P::getP(*(uint64_t *)localCopy(addr)); } template typename P::uint_t OtherAddressSpace

::getRegister(pint_t addr) { return P::getRegister(*(uint64_t *)localCopy(addr)); } template uint64_t OtherAddressSpace

::getULEB128(pint_t &addr, pint_t end) { uintptr_t size = (end - addr); LocalAddressSpace::pint_t laddr = (LocalAddressSpace::pint_t) localCopy(addr); LocalAddressSpace::pint_t sladdr = laddr; uint64_t result = LocalAddressSpace::getULEB128(laddr, laddr + size); addr += (laddr - sladdr); return result; } template -int64_t OtherAddressSpace

::getSLEB128(pint_t &addr, pint_t end) { +int64_t RemoteAddressSpace

::getSLEB128(pint_t &addr, pint_t end) { uintptr_t size = (end - addr); LocalAddressSpace::pint_t laddr = (LocalAddressSpace::pint_t) localCopy(addr); LocalAddressSpace::pint_t sladdr = laddr; uint64_t result = LocalAddressSpace::getSLEB128(laddr, laddr + size); addr += (laddr - sladdr); return result; } -template void *OtherAddressSpace

::localCopy(pint_t addr) { +template void *RemoteAddressSpace

::localCopy(pint_t addr) { // FIX ME } template -bool OtherAddressSpace

::findFunctionName(pint_t addr, char *buf, - size_t bufLen, unw_word_t *offset) { +bool RemoteAddressSpace

::findFunctionName(pint_t addr, char *buf, + size_t bufLen, + unw_word_t *offset) { // FIX ME } /// unw_addr_space is the base class that abstract unw_addr_space_t type in /// libunwind.h points to. struct unw_addr_space { cpu_type_t cpuType; task_t taskPort; }; /// unw_addr_space_i386 is the concrete instance that a unw_addr_space_t points /// to when examining /// a 32-bit intel process. struct unw_addr_space_i386 : public unw_addr_space { unw_addr_space_i386(task_t task) : oas(task) {} - OtherAddressSpace > oas; + RemoteAddressSpace> oas; }; /// unw_addr_space_x86_64 is the concrete instance that a unw_addr_space_t /// points to when examining /// a 64-bit intel process. struct unw_addr_space_x86_64 : public unw_addr_space { unw_addr_space_x86_64(task_t task) : oas(task) {} - OtherAddressSpace > oas; + RemoteAddressSpace> oas; }; /// unw_addr_space_ppc is the concrete instance that a unw_addr_space_t points /// to when examining /// a 32-bit PowerPC process. struct unw_addr_space_ppc : public unw_addr_space { unw_addr_space_ppc(task_t task) : oas(task) {} - OtherAddressSpace > oas; + RemoteAddressSpace> oas; +}; + +/// unw_addr_space_ppc is the concrete instance that a unw_addr_space_t points +/// to when examining a 64-bit PowerPC process. +struct unw_addr_space_ppc64 : public unw_addr_space { + unw_addr_space_ppc64(task_t task) : oas(task) {} + RemoteAddressSpace> oas; }; #endif // UNW_REMOTE } // namespace libunwind #endif // __ADDRESSSPACE_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/CompactUnwinder.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/CompactUnwinder.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/CompactUnwinder.hpp (revision 345026) @@ -1,699 +1,698 @@ //===-------------------------- CompactUnwinder.hpp -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Does runtime stack unwinding using compact unwind encodings. // //===----------------------------------------------------------------------===// #ifndef __COMPACT_UNWINDER_HPP__ #define __COMPACT_UNWINDER_HPP__ #include #include #include #include -#include "AddressSpace.hpp" #include "Registers.hpp" #define EXTRACT_BITS(value, mask) \ ((value >> __builtin_ctz(mask)) & (((1 << __builtin_popcount(mask))) - 1)) namespace libunwind { #if defined(_LIBUNWIND_TARGET_I386) /// CompactUnwinder_x86 uses a compact unwind info to virtually "step" (aka /// unwind) by modifying a Registers_x86 register set template class CompactUnwinder_x86 { public: static int stepWithCompactEncoding(compact_unwind_encoding_t info, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters); private: typename A::pint_t pint_t; static void frameUnwind(A &addressSpace, Registers_x86 ®isters); static void framelessUnwind(A &addressSpace, typename A::pint_t returnAddressLocation, Registers_x86 ®isters); static int stepWithCompactEncodingEBPFrame(compact_unwind_encoding_t compactEncoding, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters); static int stepWithCompactEncodingFrameless( compact_unwind_encoding_t compactEncoding, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters, bool indirectStackSize); }; template int CompactUnwinder_x86::stepWithCompactEncoding( compact_unwind_encoding_t compactEncoding, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters) { switch (compactEncoding & UNWIND_X86_MODE_MASK) { case UNWIND_X86_MODE_EBP_FRAME: return stepWithCompactEncodingEBPFrame(compactEncoding, functionStart, addressSpace, registers); case UNWIND_X86_MODE_STACK_IMMD: return stepWithCompactEncodingFrameless(compactEncoding, functionStart, addressSpace, registers, false); case UNWIND_X86_MODE_STACK_IND: return stepWithCompactEncodingFrameless(compactEncoding, functionStart, addressSpace, registers, true); } _LIBUNWIND_ABORT("invalid compact unwind encoding"); } template int CompactUnwinder_x86::stepWithCompactEncodingEBPFrame( compact_unwind_encoding_t compactEncoding, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters) { uint32_t savedRegistersOffset = EXTRACT_BITS(compactEncoding, UNWIND_X86_EBP_FRAME_OFFSET); uint32_t savedRegistersLocations = EXTRACT_BITS(compactEncoding, UNWIND_X86_EBP_FRAME_REGISTERS); uint32_t savedRegisters = registers.getEBP() - 4 * savedRegistersOffset; for (int i = 0; i < 5; ++i) { switch (savedRegistersLocations & 0x7) { case UNWIND_X86_REG_NONE: // no register saved in this slot break; case UNWIND_X86_REG_EBX: registers.setEBX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_ECX: registers.setECX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_EDX: registers.setEDX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_EDI: registers.setEDI(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_ESI: registers.setESI(addressSpace.get32(savedRegisters)); break; default: (void)functionStart; _LIBUNWIND_DEBUG_LOG("bad register for EBP frame, encoding=%08X for " "function starting at 0x%X", compactEncoding, functionStart); _LIBUNWIND_ABORT("invalid compact unwind encoding"); } savedRegisters += 4; savedRegistersLocations = (savedRegistersLocations >> 3); } frameUnwind(addressSpace, registers); return UNW_STEP_SUCCESS; } template int CompactUnwinder_x86::stepWithCompactEncodingFrameless( compact_unwind_encoding_t encoding, uint32_t functionStart, A &addressSpace, Registers_x86 ®isters, bool indirectStackSize) { uint32_t stackSizeEncoded = EXTRACT_BITS(encoding, UNWIND_X86_FRAMELESS_STACK_SIZE); uint32_t stackAdjust = EXTRACT_BITS(encoding, UNWIND_X86_FRAMELESS_STACK_ADJUST); uint32_t regCount = EXTRACT_BITS(encoding, UNWIND_X86_FRAMELESS_STACK_REG_COUNT); uint32_t permutation = EXTRACT_BITS(encoding, UNWIND_X86_FRAMELESS_STACK_REG_PERMUTATION); uint32_t stackSize = stackSizeEncoded * 4; if (indirectStackSize) { // stack size is encoded in subl $xxx,%esp instruction uint32_t subl = addressSpace.get32(functionStart + stackSizeEncoded); stackSize = subl + 4 * stackAdjust; } // decompress permutation uint32_t permunreg[6]; switch (regCount) { case 6: permunreg[0] = permutation / 120; permutation -= (permunreg[0] * 120); permunreg[1] = permutation / 24; permutation -= (permunreg[1] * 24); permunreg[2] = permutation / 6; permutation -= (permunreg[2] * 6); permunreg[3] = permutation / 2; permutation -= (permunreg[3] * 2); permunreg[4] = permutation; permunreg[5] = 0; break; case 5: permunreg[0] = permutation / 120; permutation -= (permunreg[0] * 120); permunreg[1] = permutation / 24; permutation -= (permunreg[1] * 24); permunreg[2] = permutation / 6; permutation -= (permunreg[2] * 6); permunreg[3] = permutation / 2; permutation -= (permunreg[3] * 2); permunreg[4] = permutation; break; case 4: permunreg[0] = permutation / 60; permutation -= (permunreg[0] * 60); permunreg[1] = permutation / 12; permutation -= (permunreg[1] * 12); permunreg[2] = permutation / 3; permutation -= (permunreg[2] * 3); permunreg[3] = permutation; break; case 3: permunreg[0] = permutation / 20; permutation -= (permunreg[0] * 20); permunreg[1] = permutation / 4; permutation -= (permunreg[1] * 4); permunreg[2] = permutation; break; case 2: permunreg[0] = permutation / 5; permutation -= (permunreg[0] * 5); permunreg[1] = permutation; break; case 1: permunreg[0] = permutation; break; } // re-number registers back to standard numbers int registersSaved[6]; bool used[7] = { false, false, false, false, false, false, false }; for (uint32_t i = 0; i < regCount; ++i) { uint32_t renum = 0; for (int u = 1; u < 7; ++u) { if (!used[u]) { if (renum == permunreg[i]) { registersSaved[i] = u; used[u] = true; break; } ++renum; } } } uint32_t savedRegisters = registers.getSP() + stackSize - 4 - 4 * regCount; for (uint32_t i = 0; i < regCount; ++i) { switch (registersSaved[i]) { case UNWIND_X86_REG_EBX: registers.setEBX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_ECX: registers.setECX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_EDX: registers.setEDX(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_EDI: registers.setEDI(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_ESI: registers.setESI(addressSpace.get32(savedRegisters)); break; case UNWIND_X86_REG_EBP: registers.setEBP(addressSpace.get32(savedRegisters)); break; default: _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for " "function starting at 0x%X", encoding, functionStart); _LIBUNWIND_ABORT("invalid compact unwind encoding"); } savedRegisters += 4; } framelessUnwind(addressSpace, savedRegisters, registers); return UNW_STEP_SUCCESS; } template void CompactUnwinder_x86::frameUnwind(A &addressSpace, Registers_x86 ®isters) { typename A::pint_t bp = registers.getEBP(); // ebp points to old ebp registers.setEBP(addressSpace.get32(bp)); // old esp is ebp less saved ebp and return address registers.setSP((uint32_t)bp + 8); // pop return address into eip registers.setIP(addressSpace.get32(bp + 4)); } template void CompactUnwinder_x86::framelessUnwind( A &addressSpace, typename A::pint_t returnAddressLocation, Registers_x86 ®isters) { // return address is on stack after last saved register registers.setIP(addressSpace.get32(returnAddressLocation)); // old esp is before return address registers.setSP((uint32_t)returnAddressLocation + 4); } #endif // _LIBUNWIND_TARGET_I386 #if defined(_LIBUNWIND_TARGET_X86_64) /// CompactUnwinder_x86_64 uses a compact unwind info to virtually "step" (aka /// unwind) by modifying a Registers_x86_64 register set template class CompactUnwinder_x86_64 { public: static int stepWithCompactEncoding(compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters); private: typename A::pint_t pint_t; static void frameUnwind(A &addressSpace, Registers_x86_64 ®isters); static void framelessUnwind(A &addressSpace, uint64_t returnAddressLocation, Registers_x86_64 ®isters); static int stepWithCompactEncodingRBPFrame(compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters); static int stepWithCompactEncodingFrameless( compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters, bool indirectStackSize); }; template int CompactUnwinder_x86_64::stepWithCompactEncoding( compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters) { switch (compactEncoding & UNWIND_X86_64_MODE_MASK) { case UNWIND_X86_64_MODE_RBP_FRAME: return stepWithCompactEncodingRBPFrame(compactEncoding, functionStart, addressSpace, registers); case UNWIND_X86_64_MODE_STACK_IMMD: return stepWithCompactEncodingFrameless(compactEncoding, functionStart, addressSpace, registers, false); case UNWIND_X86_64_MODE_STACK_IND: return stepWithCompactEncodingFrameless(compactEncoding, functionStart, addressSpace, registers, true); } _LIBUNWIND_ABORT("invalid compact unwind encoding"); } template int CompactUnwinder_x86_64::stepWithCompactEncodingRBPFrame( compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters) { uint32_t savedRegistersOffset = EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_OFFSET); uint32_t savedRegistersLocations = EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_REGISTERS); uint64_t savedRegisters = registers.getRBP() - 8 * savedRegistersOffset; for (int i = 0; i < 5; ++i) { switch (savedRegistersLocations & 0x7) { case UNWIND_X86_64_REG_NONE: // no register saved in this slot break; case UNWIND_X86_64_REG_RBX: registers.setRBX(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R12: registers.setR12(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R13: registers.setR13(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R14: registers.setR14(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R15: registers.setR15(addressSpace.get64(savedRegisters)); break; default: (void)functionStart; _LIBUNWIND_DEBUG_LOG("bad register for RBP frame, encoding=%08X for " "function starting at 0x%llX", compactEncoding, functionStart); _LIBUNWIND_ABORT("invalid compact unwind encoding"); } savedRegisters += 8; savedRegistersLocations = (savedRegistersLocations >> 3); } frameUnwind(addressSpace, registers); return UNW_STEP_SUCCESS; } template int CompactUnwinder_x86_64::stepWithCompactEncodingFrameless( compact_unwind_encoding_t encoding, uint64_t functionStart, A &addressSpace, Registers_x86_64 ®isters, bool indirectStackSize) { uint32_t stackSizeEncoded = EXTRACT_BITS(encoding, UNWIND_X86_64_FRAMELESS_STACK_SIZE); uint32_t stackAdjust = EXTRACT_BITS(encoding, UNWIND_X86_64_FRAMELESS_STACK_ADJUST); uint32_t regCount = EXTRACT_BITS(encoding, UNWIND_X86_64_FRAMELESS_STACK_REG_COUNT); uint32_t permutation = EXTRACT_BITS(encoding, UNWIND_X86_64_FRAMELESS_STACK_REG_PERMUTATION); uint32_t stackSize = stackSizeEncoded * 8; if (indirectStackSize) { // stack size is encoded in subl $xxx,%esp instruction uint32_t subl = addressSpace.get32(functionStart + stackSizeEncoded); stackSize = subl + 8 * stackAdjust; } // decompress permutation uint32_t permunreg[6]; switch (regCount) { case 6: permunreg[0] = permutation / 120; permutation -= (permunreg[0] * 120); permunreg[1] = permutation / 24; permutation -= (permunreg[1] * 24); permunreg[2] = permutation / 6; permutation -= (permunreg[2] * 6); permunreg[3] = permutation / 2; permutation -= (permunreg[3] * 2); permunreg[4] = permutation; permunreg[5] = 0; break; case 5: permunreg[0] = permutation / 120; permutation -= (permunreg[0] * 120); permunreg[1] = permutation / 24; permutation -= (permunreg[1] * 24); permunreg[2] = permutation / 6; permutation -= (permunreg[2] * 6); permunreg[3] = permutation / 2; permutation -= (permunreg[3] * 2); permunreg[4] = permutation; break; case 4: permunreg[0] = permutation / 60; permutation -= (permunreg[0] * 60); permunreg[1] = permutation / 12; permutation -= (permunreg[1] * 12); permunreg[2] = permutation / 3; permutation -= (permunreg[2] * 3); permunreg[3] = permutation; break; case 3: permunreg[0] = permutation / 20; permutation -= (permunreg[0] * 20); permunreg[1] = permutation / 4; permutation -= (permunreg[1] * 4); permunreg[2] = permutation; break; case 2: permunreg[0] = permutation / 5; permutation -= (permunreg[0] * 5); permunreg[1] = permutation; break; case 1: permunreg[0] = permutation; break; } // re-number registers back to standard numbers int registersSaved[6]; bool used[7] = { false, false, false, false, false, false, false }; for (uint32_t i = 0; i < regCount; ++i) { uint32_t renum = 0; for (int u = 1; u < 7; ++u) { if (!used[u]) { if (renum == permunreg[i]) { registersSaved[i] = u; used[u] = true; break; } ++renum; } } } uint64_t savedRegisters = registers.getSP() + stackSize - 8 - 8 * regCount; for (uint32_t i = 0; i < regCount; ++i) { switch (registersSaved[i]) { case UNWIND_X86_64_REG_RBX: registers.setRBX(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R12: registers.setR12(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R13: registers.setR13(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R14: registers.setR14(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_R15: registers.setR15(addressSpace.get64(savedRegisters)); break; case UNWIND_X86_64_REG_RBP: registers.setRBP(addressSpace.get64(savedRegisters)); break; default: _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for " "function starting at 0x%llX", encoding, functionStart); _LIBUNWIND_ABORT("invalid compact unwind encoding"); } savedRegisters += 8; } framelessUnwind(addressSpace, savedRegisters, registers); return UNW_STEP_SUCCESS; } template void CompactUnwinder_x86_64::frameUnwind(A &addressSpace, Registers_x86_64 ®isters) { uint64_t rbp = registers.getRBP(); // ebp points to old ebp registers.setRBP(addressSpace.get64(rbp)); // old esp is ebp less saved ebp and return address registers.setSP(rbp + 16); // pop return address into eip registers.setIP(addressSpace.get64(rbp + 8)); } template void CompactUnwinder_x86_64::framelessUnwind(A &addressSpace, uint64_t returnAddressLocation, Registers_x86_64 ®isters) { // return address is on stack after last saved register registers.setIP(addressSpace.get64(returnAddressLocation)); // old esp is before return address registers.setSP(returnAddressLocation + 8); } #endif // _LIBUNWIND_TARGET_X86_64 #if defined(_LIBUNWIND_TARGET_AARCH64) /// CompactUnwinder_arm64 uses a compact unwind info to virtually "step" (aka /// unwind) by modifying a Registers_arm64 register set template class CompactUnwinder_arm64 { public: static int stepWithCompactEncoding(compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_arm64 ®isters); private: typename A::pint_t pint_t; static int stepWithCompactEncodingFrame(compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_arm64 ®isters); static int stepWithCompactEncodingFrameless( compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_arm64 ®isters); }; template int CompactUnwinder_arm64::stepWithCompactEncoding( compact_unwind_encoding_t compactEncoding, uint64_t functionStart, A &addressSpace, Registers_arm64 ®isters) { switch (compactEncoding & UNWIND_ARM64_MODE_MASK) { case UNWIND_ARM64_MODE_FRAME: return stepWithCompactEncodingFrame(compactEncoding, functionStart, addressSpace, registers); case UNWIND_ARM64_MODE_FRAMELESS: return stepWithCompactEncodingFrameless(compactEncoding, functionStart, addressSpace, registers); } _LIBUNWIND_ABORT("invalid compact unwind encoding"); } template int CompactUnwinder_arm64::stepWithCompactEncodingFrameless( compact_unwind_encoding_t encoding, uint64_t, A &addressSpace, Registers_arm64 ®isters) { uint32_t stackSize = 16 * EXTRACT_BITS(encoding, UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK); uint64_t savedRegisterLoc = registers.getSP() + stackSize; if (encoding & UNWIND_ARM64_FRAME_X19_X20_PAIR) { registers.setRegister(UNW_ARM64_X19, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X20, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X21_X22_PAIR) { registers.setRegister(UNW_ARM64_X21, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X22, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X23_X24_PAIR) { registers.setRegister(UNW_ARM64_X23, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X24, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X25_X26_PAIR) { registers.setRegister(UNW_ARM64_X25, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X26, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X27_X28_PAIR) { registers.setRegister(UNW_ARM64_X27, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X28, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D8_D9_PAIR) { registers.setFloatRegister(UNW_ARM64_D8, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D9, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D10_D11_PAIR) { registers.setFloatRegister(UNW_ARM64_D10, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D11, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D12_D13_PAIR) { registers.setFloatRegister(UNW_ARM64_D12, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D13, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D14_D15_PAIR) { registers.setFloatRegister(UNW_ARM64_D14, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D15, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } // subtract stack size off of sp registers.setSP(savedRegisterLoc); // set pc to be value in lr registers.setIP(registers.getRegister(UNW_ARM64_LR)); return UNW_STEP_SUCCESS; } template int CompactUnwinder_arm64::stepWithCompactEncodingFrame( compact_unwind_encoding_t encoding, uint64_t, A &addressSpace, Registers_arm64 ®isters) { uint64_t savedRegisterLoc = registers.getFP() - 8; if (encoding & UNWIND_ARM64_FRAME_X19_X20_PAIR) { registers.setRegister(UNW_ARM64_X19, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X20, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X21_X22_PAIR) { registers.setRegister(UNW_ARM64_X21, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X22, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X23_X24_PAIR) { registers.setRegister(UNW_ARM64_X23, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X24, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X25_X26_PAIR) { registers.setRegister(UNW_ARM64_X25, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X26, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_X27_X28_PAIR) { registers.setRegister(UNW_ARM64_X27, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setRegister(UNW_ARM64_X28, addressSpace.get64(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D8_D9_PAIR) { registers.setFloatRegister(UNW_ARM64_D8, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D9, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D10_D11_PAIR) { registers.setFloatRegister(UNW_ARM64_D10, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D11, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D12_D13_PAIR) { registers.setFloatRegister(UNW_ARM64_D12, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D13, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } if (encoding & UNWIND_ARM64_FRAME_D14_D15_PAIR) { registers.setFloatRegister(UNW_ARM64_D14, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; registers.setFloatRegister(UNW_ARM64_D15, addressSpace.getDouble(savedRegisterLoc)); savedRegisterLoc -= 8; } uint64_t fp = registers.getFP(); // fp points to old fp registers.setFP(addressSpace.get64(fp)); // old sp is fp less saved fp and lr registers.setSP(fp + 16); // pop return address into pc registers.setIP(addressSpace.get64(fp + 8)); return UNW_STEP_SUCCESS; } #endif // _LIBUNWIND_TARGET_AARCH64 } // namespace libunwind #endif // __COMPACT_UNWINDER_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfInstructions.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfInstructions.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfInstructions.hpp (revision 345026) @@ -1,760 +1,795 @@ //===-------------------------- DwarfInstructions.hpp ---------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // -// Processor specific interpretation of dwarf unwind info. +// Processor specific interpretation of DWARF unwind info. // //===----------------------------------------------------------------------===// #ifndef __DWARF_INSTRUCTIONS_HPP__ #define __DWARF_INSTRUCTIONS_HPP__ #include #include #include #include "dwarf2.h" -#include "AddressSpace.hpp" #include "Registers.hpp" #include "DwarfParser.hpp" #include "config.h" namespace libunwind { -/// DwarfInstructions maps abtract dwarf unwind instructions to a particular +/// DwarfInstructions maps abtract DWARF unwind instructions to a particular /// architecture template class DwarfInstructions { public: typedef typename A::pint_t pint_t; typedef typename A::sint_t sint_t; static int stepWithDwarf(A &addressSpace, pint_t pc, pint_t fdeStart, R ®isters); private: enum { DW_X86_64_RET_ADDR = 16 }; enum { DW_X86_RET_ADDR = 8 }; typedef typename CFI_Parser::RegisterLocation RegisterLocation; typedef typename CFI_Parser::PrologInfo PrologInfo; typedef typename CFI_Parser::FDE_Info FDE_Info; typedef typename CFI_Parser::CIE_Info CIE_Info; static pint_t evaluateExpression(pint_t expression, A &addressSpace, const R ®isters, pint_t initialStackValue); static pint_t getSavedRegister(A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg); static double getSavedFloatRegister(A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg); static v128 getSavedVectorRegister(A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg); static pint_t getCFA(A &addressSpace, const PrologInfo &prolog, const R ®isters) { if (prolog.cfaRegister != 0) return (pint_t)((sint_t)registers.getRegister((int)prolog.cfaRegister) + prolog.cfaRegisterOffset); if (prolog.cfaExpression != 0) return evaluateExpression((pint_t)prolog.cfaExpression, addressSpace, registers, 0); assert(0 && "getCFA(): unknown location"); __builtin_unreachable(); } }; template typename A::pint_t DwarfInstructions::getSavedRegister( A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg) { switch (savedReg.location) { case CFI_Parser::kRegisterInCFA: return addressSpace.getRegister(cfa + (pint_t)savedReg.value); case CFI_Parser::kRegisterAtExpression: return addressSpace.getRegister( evaluateExpression((pint_t)savedReg.value, addressSpace, registers, cfa)); case CFI_Parser::kRegisterIsExpression: return evaluateExpression((pint_t)savedReg.value, addressSpace, registers, cfa); case CFI_Parser::kRegisterInRegister: return registers.getRegister((int)savedReg.value); case CFI_Parser::kRegisterUnused: case CFI_Parser::kRegisterOffsetFromCFA: // FIX ME break; } _LIBUNWIND_ABORT("unsupported restore location for register"); } template double DwarfInstructions::getSavedFloatRegister( A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg) { switch (savedReg.location) { case CFI_Parser::kRegisterInCFA: return addressSpace.getDouble(cfa + (pint_t)savedReg.value); case CFI_Parser::kRegisterAtExpression: return addressSpace.getDouble( evaluateExpression((pint_t)savedReg.value, addressSpace, registers, cfa)); case CFI_Parser::kRegisterIsExpression: case CFI_Parser::kRegisterUnused: case CFI_Parser::kRegisterOffsetFromCFA: case CFI_Parser::kRegisterInRegister: // FIX ME break; } _LIBUNWIND_ABORT("unsupported restore location for float register"); } template v128 DwarfInstructions::getSavedVectorRegister( A &addressSpace, const R ®isters, pint_t cfa, const RegisterLocation &savedReg) { switch (savedReg.location) { case CFI_Parser::kRegisterInCFA: return addressSpace.getVector(cfa + (pint_t)savedReg.value); case CFI_Parser::kRegisterAtExpression: return addressSpace.getVector( evaluateExpression((pint_t)savedReg.value, addressSpace, registers, cfa)); case CFI_Parser::kRegisterIsExpression: case CFI_Parser::kRegisterUnused: case CFI_Parser::kRegisterOffsetFromCFA: case CFI_Parser::kRegisterInRegister: // FIX ME break; } _LIBUNWIND_ABORT("unsupported restore location for vector register"); } template int DwarfInstructions::stepWithDwarf(A &addressSpace, pint_t pc, pint_t fdeStart, R ®isters) { FDE_Info fdeInfo; CIE_Info cieInfo; if (CFI_Parser::decodeFDE(addressSpace, fdeStart, &fdeInfo, &cieInfo) == NULL) { PrologInfo prolog; if (CFI_Parser::parseFDEInstructions(addressSpace, fdeInfo, cieInfo, pc, - &prolog)) { + R::getArch(), &prolog)) { // get pointer to cfa (architecture specific) pint_t cfa = getCFA(addressSpace, prolog, registers); - // restore registers that dwarf says were saved + // restore registers that DWARF says were saved R newRegisters = registers; pint_t returnAddress = 0; const int lastReg = R::lastDwarfRegNum(); - assert((int)CFI_Parser::kMaxRegisterNumber > lastReg && + assert(static_cast(CFI_Parser::kMaxRegisterNumber) >= lastReg && "register range too large"); assert(lastReg >= (int)cieInfo.returnAddressRegister && "register range does not contain return address register"); for (int i = 0; i <= lastReg; ++i) { if (prolog.savedRegisters[i].location != CFI_Parser::kRegisterUnused) { if (registers.validFloatRegister(i)) newRegisters.setFloatRegister( i, getSavedFloatRegister(addressSpace, registers, cfa, prolog.savedRegisters[i])); else if (registers.validVectorRegister(i)) newRegisters.setVectorRegister( i, getSavedVectorRegister(addressSpace, registers, cfa, prolog.savedRegisters[i])); else if (i == (int)cieInfo.returnAddressRegister) returnAddress = getSavedRegister(addressSpace, registers, cfa, prolog.savedRegisters[i]); else if (registers.validRegister(i)) newRegisters.setRegister( i, getSavedRegister(addressSpace, registers, cfa, prolog.savedRegisters[i])); else return UNW_EBADREG; } } // By definition, the CFA is the stack pointer at the call site, so // restoring SP means setting it to CFA. newRegisters.setSP(cfa); +#if defined(_LIBUNWIND_TARGET_AARCH64) + // If the target is aarch64 then the return address may have been signed + // using the v8.3 pointer authentication extensions. The original + // return address needs to be authenticated before the return address is + // restored. autia1716 is used instead of autia as autia1716 assembles + // to a NOP on pre-v8.3a architectures. + if ((R::getArch() == REGISTERS_ARM64) && + prolog.savedRegisters[UNW_ARM64_RA_SIGN_STATE].value) { +#if !defined(_LIBUNWIND_IS_NATIVE_ONLY) + return UNW_ECROSSRASIGNING; +#else + register unsigned long long x17 __asm("x17") = returnAddress; + register unsigned long long x16 __asm("x16") = cfa; + + // These are the autia1716/autib1716 instructions. The hint instructions + // are used here as gcc does not assemble autia1716/autib1716 for pre + // armv8.3a targets. + if (cieInfo.addressesSignedWithBKey) + asm("hint 0xe" : "+r"(x17) : "r"(x16)); // autib1716 + else + asm("hint 0xc" : "+r"(x17) : "r"(x16)); // autia1716 + returnAddress = x17; +#endif + } +#endif + +#if defined(_LIBUNWIND_TARGET_SPARC) + if (R::getArch() == REGISTERS_SPARC) { + // Skip call site instruction and delay slot + returnAddress += 8; + // Skip unimp instruction if function returns a struct + if ((addressSpace.get32(returnAddress) & 0xC1C00000) == 0) + returnAddress += 4; + } +#endif + // Return address is address after call site instruction, so setting IP to // that does simualates a return. newRegisters.setIP(returnAddress); // Simulate the step by replacing the register set with the new ones. registers = newRegisters; return UNW_STEP_SUCCESS; } } return UNW_EBADFRAME; } template typename A::pint_t DwarfInstructions::evaluateExpression(pint_t expression, A &addressSpace, const R ®isters, pint_t initialStackValue) { const bool log = false; pint_t p = expression; pint_t expressionEnd = expression + 20; // temp, until len read pint_t length = (pint_t)addressSpace.getULEB128(p, expressionEnd); expressionEnd = p + length; if (log) fprintf(stderr, "evaluateExpression(): length=%" PRIu64 "\n", (uint64_t)length); pint_t stack[100]; pint_t *sp = stack; *(++sp) = initialStackValue; while (p < expressionEnd) { if (log) { for (pint_t *t = sp; t > stack; --t) { fprintf(stderr, "sp[] = 0x%" PRIx64 "\n", (uint64_t)(*t)); } } uint8_t opcode = addressSpace.get8(p++); sint_t svalue, svalue2; pint_t value; uint32_t reg; switch (opcode) { case DW_OP_addr: // push immediate address sized value value = addressSpace.getP(p); p += sizeof(pint_t); *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_deref: // pop stack, dereference, push result value = *sp--; *(++sp) = addressSpace.getP(value); if (log) fprintf(stderr, "dereference 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_const1u: // push immediate 1 byte value value = addressSpace.get8(p); p += 1; *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_const1s: // push immediate 1 byte signed value svalue = (int8_t) addressSpace.get8(p); p += 1; *(++sp) = (pint_t)svalue; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)svalue); break; case DW_OP_const2u: // push immediate 2 byte value value = addressSpace.get16(p); p += 2; *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_const2s: // push immediate 2 byte signed value svalue = (int16_t) addressSpace.get16(p); p += 2; *(++sp) = (pint_t)svalue; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)svalue); break; case DW_OP_const4u: // push immediate 4 byte value value = addressSpace.get32(p); p += 4; *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_const4s: // push immediate 4 byte signed value svalue = (int32_t)addressSpace.get32(p); p += 4; *(++sp) = (pint_t)svalue; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)svalue); break; case DW_OP_const8u: // push immediate 8 byte value value = (pint_t)addressSpace.get64(p); p += 8; *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_const8s: // push immediate 8 byte signed value value = (pint_t)addressSpace.get64(p); p += 8; *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_constu: // push immediate ULEB128 value value = (pint_t)addressSpace.getULEB128(p, expressionEnd); *(++sp) = value; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_consts: // push immediate SLEB128 value svalue = (sint_t)addressSpace.getSLEB128(p, expressionEnd); *(++sp) = (pint_t)svalue; if (log) fprintf(stderr, "push 0x%" PRIx64 "\n", (uint64_t)svalue); break; case DW_OP_dup: // push top of stack value = *sp; *(++sp) = value; if (log) fprintf(stderr, "duplicate top of stack\n"); break; case DW_OP_drop: // pop --sp; if (log) fprintf(stderr, "pop top of stack\n"); break; case DW_OP_over: // dup second value = sp[-1]; *(++sp) = value; if (log) fprintf(stderr, "duplicate second in stack\n"); break; case DW_OP_pick: // pick from reg = addressSpace.get8(p); p += 1; value = sp[-reg]; *(++sp) = value; if (log) fprintf(stderr, "duplicate %d in stack\n", reg); break; case DW_OP_swap: // swap top two value = sp[0]; sp[0] = sp[-1]; sp[-1] = value; if (log) fprintf(stderr, "swap top of stack\n"); break; case DW_OP_rot: // rotate top three value = sp[0]; sp[0] = sp[-1]; sp[-1] = sp[-2]; sp[-2] = value; if (log) fprintf(stderr, "rotate top three of stack\n"); break; case DW_OP_xderef: // pop stack, dereference, push result value = *sp--; *sp = *((pint_t*)value); if (log) fprintf(stderr, "x-dereference 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_abs: svalue = (sint_t)*sp; if (svalue < 0) *sp = (pint_t)(-svalue); if (log) fprintf(stderr, "abs\n"); break; case DW_OP_and: value = *sp--; *sp &= value; if (log) fprintf(stderr, "and\n"); break; case DW_OP_div: svalue = (sint_t)(*sp--); svalue2 = (sint_t)*sp; *sp = (pint_t)(svalue2 / svalue); if (log) fprintf(stderr, "div\n"); break; case DW_OP_minus: value = *sp--; *sp = *sp - value; if (log) fprintf(stderr, "minus\n"); break; case DW_OP_mod: svalue = (sint_t)(*sp--); svalue2 = (sint_t)*sp; *sp = (pint_t)(svalue2 % svalue); if (log) fprintf(stderr, "module\n"); break; case DW_OP_mul: svalue = (sint_t)(*sp--); svalue2 = (sint_t)*sp; *sp = (pint_t)(svalue2 * svalue); if (log) fprintf(stderr, "mul\n"); break; case DW_OP_neg: *sp = 0 - *sp; if (log) fprintf(stderr, "neg\n"); break; case DW_OP_not: svalue = (sint_t)(*sp); *sp = (pint_t)(~svalue); if (log) fprintf(stderr, "not\n"); break; case DW_OP_or: value = *sp--; *sp |= value; if (log) fprintf(stderr, "or\n"); break; case DW_OP_plus: value = *sp--; *sp += value; if (log) fprintf(stderr, "plus\n"); break; case DW_OP_plus_uconst: // pop stack, add uelb128 constant, push result - *sp += addressSpace.getULEB128(p, expressionEnd); + *sp += static_cast(addressSpace.getULEB128(p, expressionEnd)); if (log) fprintf(stderr, "add constant\n"); break; case DW_OP_shl: value = *sp--; *sp = *sp << value; if (log) fprintf(stderr, "shift left\n"); break; case DW_OP_shr: value = *sp--; *sp = *sp >> value; if (log) fprintf(stderr, "shift left\n"); break; case DW_OP_shra: value = *sp--; svalue = (sint_t)*sp; *sp = (pint_t)(svalue >> value); if (log) fprintf(stderr, "shift left arithmetric\n"); break; case DW_OP_xor: value = *sp--; *sp ^= value; if (log) fprintf(stderr, "xor\n"); break; case DW_OP_skip: svalue = (int16_t) addressSpace.get16(p); p += 2; p = (pint_t)((sint_t)p + svalue); if (log) fprintf(stderr, "skip %" PRIu64 "\n", (uint64_t)svalue); break; case DW_OP_bra: svalue = (int16_t) addressSpace.get16(p); p += 2; if (*sp--) p = (pint_t)((sint_t)p + svalue); if (log) fprintf(stderr, "bra %" PRIu64 "\n", (uint64_t)svalue); break; case DW_OP_eq: value = *sp--; *sp = (*sp == value); if (log) fprintf(stderr, "eq\n"); break; case DW_OP_ge: value = *sp--; *sp = (*sp >= value); if (log) fprintf(stderr, "ge\n"); break; case DW_OP_gt: value = *sp--; *sp = (*sp > value); if (log) fprintf(stderr, "gt\n"); break; case DW_OP_le: value = *sp--; *sp = (*sp <= value); if (log) fprintf(stderr, "le\n"); break; case DW_OP_lt: value = *sp--; *sp = (*sp < value); if (log) fprintf(stderr, "lt\n"); break; case DW_OP_ne: value = *sp--; *sp = (*sp != value); if (log) fprintf(stderr, "ne\n"); break; case DW_OP_lit0: case DW_OP_lit1: case DW_OP_lit2: case DW_OP_lit3: case DW_OP_lit4: case DW_OP_lit5: case DW_OP_lit6: case DW_OP_lit7: case DW_OP_lit8: case DW_OP_lit9: case DW_OP_lit10: case DW_OP_lit11: case DW_OP_lit12: case DW_OP_lit13: case DW_OP_lit14: case DW_OP_lit15: case DW_OP_lit16: case DW_OP_lit17: case DW_OP_lit18: case DW_OP_lit19: case DW_OP_lit20: case DW_OP_lit21: case DW_OP_lit22: case DW_OP_lit23: case DW_OP_lit24: case DW_OP_lit25: case DW_OP_lit26: case DW_OP_lit27: case DW_OP_lit28: case DW_OP_lit29: case DW_OP_lit30: case DW_OP_lit31: value = static_cast(opcode - DW_OP_lit0); *(++sp) = value; if (log) fprintf(stderr, "push literal 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_reg0: case DW_OP_reg1: case DW_OP_reg2: case DW_OP_reg3: case DW_OP_reg4: case DW_OP_reg5: case DW_OP_reg6: case DW_OP_reg7: case DW_OP_reg8: case DW_OP_reg9: case DW_OP_reg10: case DW_OP_reg11: case DW_OP_reg12: case DW_OP_reg13: case DW_OP_reg14: case DW_OP_reg15: case DW_OP_reg16: case DW_OP_reg17: case DW_OP_reg18: case DW_OP_reg19: case DW_OP_reg20: case DW_OP_reg21: case DW_OP_reg22: case DW_OP_reg23: case DW_OP_reg24: case DW_OP_reg25: case DW_OP_reg26: case DW_OP_reg27: case DW_OP_reg28: case DW_OP_reg29: case DW_OP_reg30: case DW_OP_reg31: reg = static_cast(opcode - DW_OP_reg0); *(++sp) = registers.getRegister((int)reg); if (log) fprintf(stderr, "push reg %d\n", reg); break; case DW_OP_regx: reg = static_cast(addressSpace.getULEB128(p, expressionEnd)); *(++sp) = registers.getRegister((int)reg); if (log) fprintf(stderr, "push reg %d + 0x%" PRIx64 "\n", reg, (uint64_t)svalue); break; case DW_OP_breg0: case DW_OP_breg1: case DW_OP_breg2: case DW_OP_breg3: case DW_OP_breg4: case DW_OP_breg5: case DW_OP_breg6: case DW_OP_breg7: case DW_OP_breg8: case DW_OP_breg9: case DW_OP_breg10: case DW_OP_breg11: case DW_OP_breg12: case DW_OP_breg13: case DW_OP_breg14: case DW_OP_breg15: case DW_OP_breg16: case DW_OP_breg17: case DW_OP_breg18: case DW_OP_breg19: case DW_OP_breg20: case DW_OP_breg21: case DW_OP_breg22: case DW_OP_breg23: case DW_OP_breg24: case DW_OP_breg25: case DW_OP_breg26: case DW_OP_breg27: case DW_OP_breg28: case DW_OP_breg29: case DW_OP_breg30: case DW_OP_breg31: reg = static_cast(opcode - DW_OP_breg0); svalue = (sint_t)addressSpace.getSLEB128(p, expressionEnd); svalue += static_cast(registers.getRegister((int)reg)); *(++sp) = (pint_t)(svalue); if (log) fprintf(stderr, "push reg %d + 0x%" PRIx64 "\n", reg, (uint64_t)svalue); break; case DW_OP_bregx: reg = static_cast(addressSpace.getULEB128(p, expressionEnd)); svalue = (sint_t)addressSpace.getSLEB128(p, expressionEnd); svalue += static_cast(registers.getRegister((int)reg)); *(++sp) = (pint_t)(svalue); if (log) fprintf(stderr, "push reg %d + 0x%" PRIx64 "\n", reg, (uint64_t)svalue); break; case DW_OP_fbreg: _LIBUNWIND_ABORT("DW_OP_fbreg not implemented"); break; case DW_OP_piece: _LIBUNWIND_ABORT("DW_OP_piece not implemented"); break; case DW_OP_deref_size: // pop stack, dereference, push result value = *sp--; switch (addressSpace.get8(p++)) { case 1: value = addressSpace.get8(value); break; case 2: value = addressSpace.get16(value); break; case 4: value = addressSpace.get32(value); break; case 8: value = (pint_t)addressSpace.get64(value); break; default: _LIBUNWIND_ABORT("DW_OP_deref_size with bad size"); } *(++sp) = value; if (log) fprintf(stderr, "sized dereference 0x%" PRIx64 "\n", (uint64_t)value); break; case DW_OP_xderef_size: case DW_OP_nop: case DW_OP_push_object_addres: case DW_OP_call2: case DW_OP_call4: case DW_OP_call_ref: default: - _LIBUNWIND_ABORT("dwarf opcode not implemented"); + _LIBUNWIND_ABORT("DWARF opcode not implemented"); } } if (log) fprintf(stderr, "expression evaluates to 0x%" PRIx64 "\n", (uint64_t)*sp); return *sp; } } // namespace libunwind #endif // __DWARF_INSTRUCTIONS_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfParser.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfParser.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/DwarfParser.hpp (revision 345026) @@ -1,726 +1,766 @@ //===--------------------------- DwarfParser.hpp --------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Parses DWARF CFIs (FDEs and CIEs). // //===----------------------------------------------------------------------===// #ifndef __DWARF_PARSER_HPP__ #define __DWARF_PARSER_HPP__ #include #include #include #include #include "libunwind.h" #include "dwarf2.h" +#include "Registers.hpp" -#include "AddressSpace.hpp" +#include "config.h" namespace libunwind { /// CFI_Parser does basic parsing of a CFI (Call Frame Information) records. -/// See Dwarf Spec for details: +/// See DWARF Spec for details: /// http://refspecs.linuxbase.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html /// template class CFI_Parser { public: typedef typename A::pint_t pint_t; /// Information encoded in a CIE (Common Information Entry) struct CIE_Info { pint_t cieStart; pint_t cieLength; pint_t cieInstructions; uint8_t pointerEncoding; uint8_t lsdaEncoding; uint8_t personalityEncoding; uint8_t personalityOffsetInCIE; pint_t personality; uint32_t codeAlignFactor; int dataAlignFactor; bool isSignalFrame; bool fdesHaveAugmentationData; uint8_t returnAddressRegister; +#if defined(_LIBUNWIND_TARGET_AARCH64) + bool addressesSignedWithBKey; +#endif }; /// Information about an FDE (Frame Description Entry) struct FDE_Info { pint_t fdeStart; pint_t fdeLength; pint_t fdeInstructions; pint_t pcStart; pint_t pcEnd; pint_t lsda; }; enum { - kMaxRegisterNumber = _LIBUNWIND_MAX_REGISTER + kMaxRegisterNumber = _LIBUNWIND_HIGHEST_DWARF_REGISTER }; enum RegisterSavedWhere { kRegisterUnused, kRegisterInCFA, kRegisterOffsetFromCFA, kRegisterInRegister, kRegisterAtExpression, kRegisterIsExpression }; struct RegisterLocation { RegisterSavedWhere location; int64_t value; }; /// Information about a frame layout and registers saved determined - /// by "running" the dwarf FDE "instructions" + /// by "running" the DWARF FDE "instructions" struct PrologInfo { uint32_t cfaRegister; int32_t cfaRegisterOffset; // CFA = (cfaRegister)+cfaRegisterOffset int64_t cfaExpression; // CFA = expression uint32_t spExtraArgSize; uint32_t codeOffsetAtStackDecrement; bool registersInOtherRegisters; bool sameValueUsed; - RegisterLocation savedRegisters[kMaxRegisterNumber]; + RegisterLocation savedRegisters[kMaxRegisterNumber + 1]; }; struct PrologInfoStackEntry { PrologInfoStackEntry(PrologInfoStackEntry *n, const PrologInfo &i) : next(n), info(i) {} PrologInfoStackEntry *next; PrologInfo info; }; static bool findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart, uint32_t sectionLength, pint_t fdeHint, FDE_Info *fdeInfo, CIE_Info *cieInfo); static const char *decodeFDE(A &addressSpace, pint_t fdeStart, FDE_Info *fdeInfo, CIE_Info *cieInfo); static bool parseFDEInstructions(A &addressSpace, const FDE_Info &fdeInfo, const CIE_Info &cieInfo, pint_t upToPC, - PrologInfo *results); + int arch, PrologInfo *results); static const char *parseCIE(A &addressSpace, pint_t cie, CIE_Info *cieInfo); private: static bool parseInstructions(A &addressSpace, pint_t instructions, pint_t instructionsEnd, const CIE_Info &cieInfo, pint_t pcoffset, - PrologInfoStackEntry *&rememberStack, + PrologInfoStackEntry *&rememberStack, int arch, PrologInfo *results); }; /// Parse a FDE into a CIE_Info and an FDE_Info template const char *CFI_Parser::decodeFDE(A &addressSpace, pint_t fdeStart, FDE_Info *fdeInfo, CIE_Info *cieInfo) { pint_t p = fdeStart; pint_t cfiLength = (pint_t)addressSpace.get32(p); p += 4; if (cfiLength == 0xffffffff) { // 0xffffffff means length is really next 8 bytes cfiLength = (pint_t)addressSpace.get64(p); p += 8; } if (cfiLength == 0) return "FDE has zero length"; // end marker uint32_t ciePointer = addressSpace.get32(p); if (ciePointer == 0) return "FDE is really a CIE"; // this is a CIE not an FDE pint_t nextCFI = p + cfiLength; pint_t cieStart = p - ciePointer; const char *err = parseCIE(addressSpace, cieStart, cieInfo); if (err != NULL) return err; p += 4; - // parse pc begin and range + // Parse pc begin and range. pint_t pcStart = addressSpace.getEncodedP(p, nextCFI, cieInfo->pointerEncoding); pint_t pcRange = addressSpace.getEncodedP(p, nextCFI, cieInfo->pointerEncoding & 0x0F); - // parse rest of info + // Parse rest of info. fdeInfo->lsda = 0; - // check for augmentation length + // Check for augmentation length. if (cieInfo->fdesHaveAugmentationData) { pint_t augLen = (pint_t)addressSpace.getULEB128(p, nextCFI); pint_t endOfAug = p + augLen; if (cieInfo->lsdaEncoding != DW_EH_PE_omit) { - // peek at value (without indirection). Zero means no lsda + // Peek at value (without indirection). Zero means no LSDA. pint_t lsdaStart = p; if (addressSpace.getEncodedP(p, nextCFI, cieInfo->lsdaEncoding & 0x0F) != 0) { - // reset pointer and re-parse lsda address + // Reset pointer and re-parse LSDA address. p = lsdaStart; fdeInfo->lsda = addressSpace.getEncodedP(p, nextCFI, cieInfo->lsdaEncoding); } } p = endOfAug; } fdeInfo->fdeStart = fdeStart; fdeInfo->fdeLength = nextCFI - fdeStart; fdeInfo->fdeInstructions = p; fdeInfo->pcStart = pcStart; fdeInfo->pcEnd = pcStart + pcRange; return NULL; // success } /// Scan an eh_frame section to find an FDE for a pc template bool CFI_Parser::findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart, uint32_t sectionLength, pint_t fdeHint, FDE_Info *fdeInfo, CIE_Info *cieInfo) { //fprintf(stderr, "findFDE(0x%llX)\n", (long long)pc); pint_t p = (fdeHint != 0) ? fdeHint : ehSectionStart; const pint_t ehSectionEnd = p + sectionLength; while (p < ehSectionEnd) { pint_t currentCFI = p; //fprintf(stderr, "findFDE() CFI at 0x%llX\n", (long long)p); pint_t cfiLength = addressSpace.get32(p); p += 4; if (cfiLength == 0xffffffff) { // 0xffffffff means length is really next 8 bytes cfiLength = (pint_t)addressSpace.get64(p); p += 8; } if (cfiLength == 0) return false; // end marker uint32_t id = addressSpace.get32(p); if (id == 0) { - // skip over CIEs + // Skip over CIEs. p += cfiLength; } else { - // process FDE to see if it covers pc + // Process FDE to see if it covers pc. pint_t nextCFI = p + cfiLength; uint32_t ciePointer = addressSpace.get32(p); pint_t cieStart = p - ciePointer; - // validate pointer to CIE is within section + // Validate pointer to CIE is within section. if ((ehSectionStart <= cieStart) && (cieStart < ehSectionEnd)) { if (parseCIE(addressSpace, cieStart, cieInfo) == NULL) { p += 4; - // parse pc begin and range + // Parse pc begin and range. pint_t pcStart = addressSpace.getEncodedP(p, nextCFI, cieInfo->pointerEncoding); pint_t pcRange = addressSpace.getEncodedP( p, nextCFI, cieInfo->pointerEncoding & 0x0F); - // test if pc is within the function this FDE covers + // Test if pc is within the function this FDE covers. if ((pcStart < pc) && (pc <= pcStart + pcRange)) { // parse rest of info fdeInfo->lsda = 0; // check for augmentation length if (cieInfo->fdesHaveAugmentationData) { pint_t augLen = (pint_t)addressSpace.getULEB128(p, nextCFI); pint_t endOfAug = p + augLen; if (cieInfo->lsdaEncoding != DW_EH_PE_omit) { - // peek at value (without indirection). Zero means no lsda + // Peek at value (without indirection). Zero means no LSDA. pint_t lsdaStart = p; if (addressSpace.getEncodedP( p, nextCFI, cieInfo->lsdaEncoding & 0x0F) != 0) { - // reset pointer and re-parse lsda address + // Reset pointer and re-parse LSDA address. p = lsdaStart; fdeInfo->lsda = addressSpace .getEncodedP(p, nextCFI, cieInfo->lsdaEncoding); } } p = endOfAug; } fdeInfo->fdeStart = currentCFI; fdeInfo->fdeLength = nextCFI - currentCFI; fdeInfo->fdeInstructions = p; fdeInfo->pcStart = pcStart; fdeInfo->pcEnd = pcStart + pcRange; return true; } else { // pc is not in begin/range, skip this FDE } } else { - // malformed CIE, now augmentation describing pc range encoding + // Malformed CIE, now augmentation describing pc range encoding. } } else { // malformed FDE. CIE is bad } p = nextCFI; } } return false; } /// Extract info from a CIE template const char *CFI_Parser::parseCIE(A &addressSpace, pint_t cie, CIE_Info *cieInfo) { cieInfo->pointerEncoding = 0; cieInfo->lsdaEncoding = DW_EH_PE_omit; cieInfo->personalityEncoding = 0; cieInfo->personalityOffsetInCIE = 0; cieInfo->personality = 0; cieInfo->codeAlignFactor = 0; cieInfo->dataAlignFactor = 0; cieInfo->isSignalFrame = false; cieInfo->fdesHaveAugmentationData = false; +#if defined(_LIBUNWIND_TARGET_AARCH64) + cieInfo->addressesSignedWithBKey = false; +#endif cieInfo->cieStart = cie; pint_t p = cie; pint_t cieLength = (pint_t)addressSpace.get32(p); p += 4; pint_t cieContentEnd = p + cieLength; if (cieLength == 0xffffffff) { // 0xffffffff means length is really next 8 bytes cieLength = (pint_t)addressSpace.get64(p); p += 8; cieContentEnd = p + cieLength; } if (cieLength == 0) return NULL; // CIE ID is always 0 if (addressSpace.get32(p) != 0) return "CIE ID is not zero"; p += 4; // Version is always 1 or 3 uint8_t version = addressSpace.get8(p); if ((version != 1) && (version != 3)) return "CIE version is not 1 or 3"; ++p; // save start of augmentation string and find end pint_t strStart = p; while (addressSpace.get8(p) != 0) ++p; ++p; // parse code aligment factor cieInfo->codeAlignFactor = (uint32_t)addressSpace.getULEB128(p, cieContentEnd); // parse data alignment factor cieInfo->dataAlignFactor = (int)addressSpace.getSLEB128(p, cieContentEnd); // parse return address register uint64_t raReg = addressSpace.getULEB128(p, cieContentEnd); assert(raReg < 255 && "return address register too large"); cieInfo->returnAddressRegister = (uint8_t)raReg; // parse augmentation data based on augmentation string const char *result = NULL; if (addressSpace.get8(strStart) == 'z') { // parse augmentation data length addressSpace.getULEB128(p, cieContentEnd); for (pint_t s = strStart; addressSpace.get8(s) != '\0'; ++s) { switch (addressSpace.get8(s)) { case 'z': cieInfo->fdesHaveAugmentationData = true; break; case 'P': cieInfo->personalityEncoding = addressSpace.get8(p); ++p; cieInfo->personalityOffsetInCIE = (uint8_t)(p - cie); cieInfo->personality = addressSpace .getEncodedP(p, cieContentEnd, cieInfo->personalityEncoding); break; case 'L': cieInfo->lsdaEncoding = addressSpace.get8(p); ++p; break; case 'R': cieInfo->pointerEncoding = addressSpace.get8(p); ++p; break; case 'S': cieInfo->isSignalFrame = true; break; +#if defined(_LIBUNWIND_TARGET_AARCH64) + case 'B': + cieInfo->addressesSignedWithBKey = true; + break; +#endif default: // ignore unknown letters break; } } } cieInfo->cieLength = cieContentEnd - cieInfo->cieStart; cieInfo->cieInstructions = p; return result; } -/// "run" the dwarf instructions and create the abstact PrologInfo for an FDE +/// "run" the DWARF instructions and create the abstact PrologInfo for an FDE template bool CFI_Parser::parseFDEInstructions(A &addressSpace, const FDE_Info &fdeInfo, const CIE_Info &cieInfo, pint_t upToPC, - PrologInfo *results) { + int arch, PrologInfo *results) { // clear results memset(results, '\0', sizeof(PrologInfo)); PrologInfoStackEntry *rememberStack = NULL; // parse CIE then FDE instructions return parseInstructions(addressSpace, cieInfo.cieInstructions, cieInfo.cieStart + cieInfo.cieLength, cieInfo, - (pint_t)(-1), rememberStack, results) && + (pint_t)(-1), rememberStack, arch, results) && parseInstructions(addressSpace, fdeInfo.fdeInstructions, fdeInfo.fdeStart + fdeInfo.fdeLength, cieInfo, - upToPC - fdeInfo.pcStart, rememberStack, results); + upToPC - fdeInfo.pcStart, rememberStack, arch, + results); } -/// "run" the dwarf instructions +/// "run" the DWARF instructions template bool CFI_Parser::parseInstructions(A &addressSpace, pint_t instructions, pint_t instructionsEnd, const CIE_Info &cieInfo, pint_t pcoffset, PrologInfoStackEntry *&rememberStack, - PrologInfo *results) { - const bool logDwarf = false; + int arch, PrologInfo *results) { pint_t p = instructions; pint_t codeOffset = 0; PrologInfo initialState = *results; - if (logDwarf) - fprintf(stderr, "parseInstructions(instructions=0x%0" PRIx64 ")\n", - (uint64_t)instructionsEnd); - // see Dwarf Spec, section 6.4.2 for details on unwind opcodes + _LIBUNWIND_TRACE_DWARF("parseInstructions(instructions=0x%0" PRIx64 ")\n", + static_cast(instructionsEnd)); + + // see DWARF Spec, section 6.4.2 for details on unwind opcodes while ((p < instructionsEnd) && (codeOffset < pcoffset)) { uint64_t reg; uint64_t reg2; int64_t offset; uint64_t length; uint8_t opcode = addressSpace.get8(p); uint8_t operand; #if !defined(_LIBUNWIND_NO_HEAP) PrologInfoStackEntry *entry; #endif ++p; switch (opcode) { case DW_CFA_nop: - if (logDwarf) - fprintf(stderr, "DW_CFA_nop\n"); + _LIBUNWIND_TRACE_DWARF("DW_CFA_nop\n"); break; case DW_CFA_set_loc: codeOffset = addressSpace.getEncodedP(p, instructionsEnd, cieInfo.pointerEncoding); - if (logDwarf) - fprintf(stderr, "DW_CFA_set_loc\n"); + _LIBUNWIND_TRACE_DWARF("DW_CFA_set_loc\n"); break; case DW_CFA_advance_loc1: codeOffset += (addressSpace.get8(p) * cieInfo.codeAlignFactor); p += 1; - if (logDwarf) - fprintf(stderr, "DW_CFA_advance_loc1: new offset=%" PRIu64 "\n", - (uint64_t)codeOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_advance_loc1: new offset=%" PRIu64 "\n", + static_cast(codeOffset)); break; case DW_CFA_advance_loc2: codeOffset += (addressSpace.get16(p) * cieInfo.codeAlignFactor); p += 2; - if (logDwarf) - fprintf(stderr, "DW_CFA_advance_loc2: new offset=%" PRIu64 "\n", - (uint64_t)codeOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_advance_loc2: new offset=%" PRIu64 "\n", + static_cast(codeOffset)); break; case DW_CFA_advance_loc4: codeOffset += (addressSpace.get32(p) * cieInfo.codeAlignFactor); p += 4; - if (logDwarf) - fprintf(stderr, "DW_CFA_advance_loc4: new offset=%" PRIu64 "\n", - (uint64_t)codeOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_advance_loc4: new offset=%" PRIu64 "\n", + static_cast(codeOffset)); break; case DW_CFA_offset_extended: reg = addressSpace.getULEB128(p, instructionsEnd); offset = (int64_t)addressSpace.getULEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_offset_extended dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_offset_extended DWARF unwind, reg too big"); return false; } results->savedRegisters[reg].location = kRegisterInCFA; results->savedRegisters[reg].value = offset; - if (logDwarf) - fprintf(stderr, - "DW_CFA_offset_extended(reg=%" PRIu64 ", offset=%" PRId64 ")\n", - reg, offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_offset_extended(reg=%" PRIu64 ", " + "offset=%" PRId64 ")\n", + reg, offset); break; case DW_CFA_restore_extended: reg = addressSpace.getULEB128(p, instructionsEnd); - ; if (reg > kMaxRegisterNumber) { - fprintf( - stderr, - "malformed DW_CFA_restore_extended dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_restore_extended DWARF unwind, reg too big"); return false; } results->savedRegisters[reg] = initialState.savedRegisters[reg]; - if (logDwarf) - fprintf(stderr, "DW_CFA_restore_extended(reg=%" PRIu64 ")\n", reg); + _LIBUNWIND_TRACE_DWARF("DW_CFA_restore_extended(reg=%" PRIu64 ")\n", reg); break; case DW_CFA_undefined: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_undefined dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_undefined DWARF unwind, reg too big"); return false; } results->savedRegisters[reg].location = kRegisterUnused; - if (logDwarf) - fprintf(stderr, "DW_CFA_undefined(reg=%" PRIu64 ")\n", reg); + _LIBUNWIND_TRACE_DWARF("DW_CFA_undefined(reg=%" PRIu64 ")\n", reg); break; case DW_CFA_same_value: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_same_value dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_same_value DWARF unwind, reg too big"); return false; } // DW_CFA_same_value unsupported // "same value" means register was stored in frame, but its current // value has not changed, so no need to restore from frame. // We model this as if the register was never saved. results->savedRegisters[reg].location = kRegisterUnused; // set flag to disable conversion to compact unwind results->sameValueUsed = true; - if (logDwarf) - fprintf(stderr, "DW_CFA_same_value(reg=%" PRIu64 ")\n", reg); + _LIBUNWIND_TRACE_DWARF("DW_CFA_same_value(reg=%" PRIu64 ")\n", reg); break; case DW_CFA_register: reg = addressSpace.getULEB128(p, instructionsEnd); reg2 = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_register dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_register DWARF unwind, reg too big"); return false; } if (reg2 > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_register dwarf unwind, reg2 too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_register DWARF unwind, reg2 too big"); return false; } results->savedRegisters[reg].location = kRegisterInRegister; results->savedRegisters[reg].value = (int64_t)reg2; // set flag to disable conversion to compact unwind results->registersInOtherRegisters = true; - if (logDwarf) - fprintf(stderr, "DW_CFA_register(reg=%" PRIu64 ", reg2=%" PRIu64 ")\n", - reg, reg2); + _LIBUNWIND_TRACE_DWARF( + "DW_CFA_register(reg=%" PRIu64 ", reg2=%" PRIu64 ")\n", reg, reg2); break; #if !defined(_LIBUNWIND_NO_HEAP) case DW_CFA_remember_state: // avoid operator new, because that would be an upward dependency entry = (PrologInfoStackEntry *)malloc(sizeof(PrologInfoStackEntry)); if (entry != NULL) { entry->next = rememberStack; entry->info = *results; rememberStack = entry; } else { return false; } - if (logDwarf) - fprintf(stderr, "DW_CFA_remember_state\n"); + _LIBUNWIND_TRACE_DWARF("DW_CFA_remember_state\n"); break; case DW_CFA_restore_state: if (rememberStack != NULL) { PrologInfoStackEntry *top = rememberStack; *results = top->info; rememberStack = top->next; free((char *)top); } else { return false; } - if (logDwarf) - fprintf(stderr, "DW_CFA_restore_state\n"); + _LIBUNWIND_TRACE_DWARF("DW_CFA_restore_state\n"); break; #endif case DW_CFA_def_cfa: reg = addressSpace.getULEB128(p, instructionsEnd); offset = (int64_t)addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, "malformed DW_CFA_def_cfa dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0("malformed DW_CFA_def_cfa DWARF unwind, reg too big"); return false; } results->cfaRegister = (uint32_t)reg; results->cfaRegisterOffset = (int32_t)offset; - if (logDwarf) - fprintf(stderr, "DW_CFA_def_cfa(reg=%" PRIu64 ", offset=%" PRIu64 ")\n", - reg, offset); + _LIBUNWIND_TRACE_DWARF( + "DW_CFA_def_cfa(reg=%" PRIu64 ", offset=%" PRIu64 ")\n", reg, offset); break; case DW_CFA_def_cfa_register: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf( - stderr, - "malformed DW_CFA_def_cfa_register dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_def_cfa_register DWARF unwind, reg too big"); return false; } results->cfaRegister = (uint32_t)reg; - if (logDwarf) - fprintf(stderr, "DW_CFA_def_cfa_register(%" PRIu64 ")\n", reg); + _LIBUNWIND_TRACE_DWARF("DW_CFA_def_cfa_register(%" PRIu64 ")\n", reg); break; case DW_CFA_def_cfa_offset: results->cfaRegisterOffset = (int32_t) addressSpace.getULEB128(p, instructionsEnd); results->codeOffsetAtStackDecrement = (uint32_t)codeOffset; - if (logDwarf) - fprintf(stderr, "DW_CFA_def_cfa_offset(%d)\n", - results->cfaRegisterOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_def_cfa_offset(%d)\n", + results->cfaRegisterOffset); break; case DW_CFA_def_cfa_expression: results->cfaRegister = 0; results->cfaExpression = (int64_t)p; length = addressSpace.getULEB128(p, instructionsEnd); - p += length; - if (logDwarf) - fprintf(stderr, "DW_CFA_def_cfa_expression(expression=0x%" PRIx64 - ", length=%" PRIu64 ")\n", - results->cfaExpression, length); + assert(length < static_cast(~0) && "pointer overflow"); + p += static_cast(length); + _LIBUNWIND_TRACE_DWARF("DW_CFA_def_cfa_expression(expression=0x%" PRIx64 + ", length=%" PRIu64 ")\n", + results->cfaExpression, length); break; case DW_CFA_expression: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_expression dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_expression DWARF unwind, reg too big"); return false; } results->savedRegisters[reg].location = kRegisterAtExpression; results->savedRegisters[reg].value = (int64_t)p; length = addressSpace.getULEB128(p, instructionsEnd); - p += length; - if (logDwarf) - fprintf(stderr, "DW_CFA_expression(reg=%" PRIu64 - ", expression=0x%" PRIx64 ", length=%" PRIu64 ")\n", - reg, results->savedRegisters[reg].value, length); + assert(length < static_cast(~0) && "pointer overflow"); + p += static_cast(length); + _LIBUNWIND_TRACE_DWARF("DW_CFA_expression(reg=%" PRIu64 ", " + "expression=0x%" PRIx64 ", " + "length=%" PRIu64 ")\n", + reg, results->savedRegisters[reg].value, length); break; case DW_CFA_offset_extended_sf: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf( - stderr, - "malformed DW_CFA_offset_extended_sf dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_offset_extended_sf DWARF unwind, reg too big"); return false; } offset = addressSpace.getSLEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; results->savedRegisters[reg].location = kRegisterInCFA; results->savedRegisters[reg].value = offset; - if (logDwarf) - fprintf(stderr, "DW_CFA_offset_extended_sf(reg=%" PRIu64 - ", offset=%" PRId64 ")\n", - reg, offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_offset_extended_sf(reg=%" PRIu64 ", " + "offset=%" PRId64 ")\n", + reg, offset); break; case DW_CFA_def_cfa_sf: reg = addressSpace.getULEB128(p, instructionsEnd); offset = addressSpace.getSLEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_def_cfa_sf dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_def_cfa_sf DWARF unwind, reg too big"); return false; } results->cfaRegister = (uint32_t)reg; results->cfaRegisterOffset = (int32_t)offset; - if (logDwarf) - fprintf(stderr, - "DW_CFA_def_cfa_sf(reg=%" PRIu64 ", offset=%" PRId64 ")\n", reg, - offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_def_cfa_sf(reg=%" PRIu64 ", " + "offset=%" PRId64 ")\n", + reg, offset); break; case DW_CFA_def_cfa_offset_sf: results->cfaRegisterOffset = (int32_t) (addressSpace.getSLEB128(p, instructionsEnd) * cieInfo.dataAlignFactor); results->codeOffsetAtStackDecrement = (uint32_t)codeOffset; - if (logDwarf) - fprintf(stderr, "DW_CFA_def_cfa_offset_sf(%d)\n", - results->cfaRegisterOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_def_cfa_offset_sf(%d)\n", + results->cfaRegisterOffset); break; case DW_CFA_val_offset: reg = addressSpace.getULEB128(p, instructionsEnd); + if (reg > kMaxRegisterNumber) { + _LIBUNWIND_LOG( + "malformed DW_CFA_val_offset DWARF unwind, reg (%" PRIu64 + ") out of range\n", + reg); + return false; + } offset = (int64_t)addressSpace.getULEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; results->savedRegisters[reg].location = kRegisterOffsetFromCFA; results->savedRegisters[reg].value = offset; - if (logDwarf) - fprintf(stderr, - "DW_CFA_val_offset(reg=%" PRIu64 ", offset=%" PRId64 "\n", reg, - offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_val_offset(reg=%" PRIu64 ", " + "offset=%" PRId64 "\n", + reg, offset); break; case DW_CFA_val_offset_sf: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_val_offset_sf dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_val_offset_sf DWARF unwind, reg too big"); return false; } offset = addressSpace.getSLEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; results->savedRegisters[reg].location = kRegisterOffsetFromCFA; results->savedRegisters[reg].value = offset; - if (logDwarf) - fprintf(stderr, - "DW_CFA_val_offset_sf(reg=%" PRIu64 ", offset=%" PRId64 "\n", - reg, offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_val_offset_sf(reg=%" PRIu64 ", " + "offset=%" PRId64 "\n", + reg, offset); break; case DW_CFA_val_expression: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, - "malformed DW_CFA_val_expression dwarf unwind, reg too big\n"); + _LIBUNWIND_LOG0( + "malformed DW_CFA_val_expression DWARF unwind, reg too big"); return false; } results->savedRegisters[reg].location = kRegisterIsExpression; results->savedRegisters[reg].value = (int64_t)p; length = addressSpace.getULEB128(p, instructionsEnd); - p += length; - if (logDwarf) - fprintf(stderr, "DW_CFA_val_expression(reg=%" PRIu64 - ", expression=0x%" PRIx64 ", length=%" PRIu64 ")\n", - reg, results->savedRegisters[reg].value, length); + assert(length < static_cast(~0) && "pointer overflow"); + p += static_cast(length); + _LIBUNWIND_TRACE_DWARF("DW_CFA_val_expression(reg=%" PRIu64 ", " + "expression=0x%" PRIx64 ", length=%" PRIu64 ")\n", + reg, results->savedRegisters[reg].value, length); break; case DW_CFA_GNU_args_size: length = addressSpace.getULEB128(p, instructionsEnd); results->spExtraArgSize = (uint32_t)length; - if (logDwarf) - fprintf(stderr, "DW_CFA_GNU_args_size(%" PRIu64 ")\n", length); + _LIBUNWIND_TRACE_DWARF("DW_CFA_GNU_args_size(%" PRIu64 ")\n", length); break; case DW_CFA_GNU_negative_offset_extended: reg = addressSpace.getULEB128(p, instructionsEnd); if (reg > kMaxRegisterNumber) { - fprintf(stderr, "malformed DW_CFA_GNU_negative_offset_extended dwarf " - "unwind, reg too big\n"); + _LIBUNWIND_LOG0("malformed DW_CFA_GNU_negative_offset_extended DWARF " + "unwind, reg too big"); return false; } offset = (int64_t)addressSpace.getULEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; results->savedRegisters[reg].location = kRegisterInCFA; results->savedRegisters[reg].value = -offset; - if (logDwarf) - fprintf(stderr, "DW_CFA_GNU_negative_offset_extended(%" PRId64 ")\n", - offset); + _LIBUNWIND_TRACE_DWARF( + "DW_CFA_GNU_negative_offset_extended(%" PRId64 ")\n", offset); break; + +#if defined(_LIBUNWIND_TARGET_AARCH64) || defined(_LIBUNWIND_TARGET_SPARC) + // The same constant is used to represent different instructions on + // AArch64 (negate_ra_state) and SPARC (window_save). + static_assert(DW_CFA_AARCH64_negate_ra_state == DW_CFA_GNU_window_save, + "uses the same constant"); + case DW_CFA_AARCH64_negate_ra_state: + switch (arch) { +#if defined(_LIBUNWIND_TARGET_AARCH64) + case REGISTERS_ARM64: + results->savedRegisters[UNW_ARM64_RA_SIGN_STATE].value ^= 0x1; + _LIBUNWIND_TRACE_DWARF("DW_CFA_AARCH64_negate_ra_state\n"); + break; +#endif +#if defined(_LIBUNWIND_TARGET_SPARC) + // case DW_CFA_GNU_window_save: + case REGISTERS_SPARC: + _LIBUNWIND_TRACE_DWARF("DW_CFA_GNU_window_save()\n"); + for (reg = UNW_SPARC_O0; reg <= UNW_SPARC_O7; reg++) { + results->savedRegisters[reg].location = kRegisterInRegister; + results->savedRegisters[reg].value = + ((int64_t)reg - UNW_SPARC_O0) + UNW_SPARC_I0; + } + + for (reg = UNW_SPARC_L0; reg <= UNW_SPARC_I7; reg++) { + results->savedRegisters[reg].location = kRegisterInCFA; + results->savedRegisters[reg].value = + ((int64_t)reg - UNW_SPARC_L0) * 4; + } + break; +#endif + } + break; +#else + (void)arch; +#endif + default: operand = opcode & 0x3F; switch (opcode & 0xC0) { case DW_CFA_offset: reg = operand; + if (reg > kMaxRegisterNumber) { + _LIBUNWIND_LOG("malformed DW_CFA_offset DWARF unwind, reg (%" PRIu64 + ") out of range", + reg); + return false; + } offset = (int64_t)addressSpace.getULEB128(p, instructionsEnd) * cieInfo.dataAlignFactor; results->savedRegisters[reg].location = kRegisterInCFA; results->savedRegisters[reg].value = offset; - if (logDwarf) - fprintf(stderr, "DW_CFA_offset(reg=%d, offset=%" PRId64 ")\n", - operand, offset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_offset(reg=%d, offset=%" PRId64 ")\n", + operand, offset); break; case DW_CFA_advance_loc: codeOffset += operand * cieInfo.codeAlignFactor; - if (logDwarf) - fprintf(stderr, "DW_CFA_advance_loc: new offset=%" PRIu64 "\n", - (uint64_t)codeOffset); + _LIBUNWIND_TRACE_DWARF("DW_CFA_advance_loc: new offset=%" PRIu64 "\n", + static_cast(codeOffset)); break; case DW_CFA_restore: reg = operand; + if (reg > kMaxRegisterNumber) { + _LIBUNWIND_LOG("malformed DW_CFA_restore DWARF unwind, reg (%" PRIu64 + ") out of range", + reg); + return false; + } results->savedRegisters[reg] = initialState.savedRegisters[reg]; - if (logDwarf) - fprintf(stderr, "DW_CFA_restore(reg=%" PRIu64 ")\n", reg); + _LIBUNWIND_TRACE_DWARF("DW_CFA_restore(reg=%" PRIu64 ")\n", + static_cast(operand)); break; default: - if (logDwarf) - fprintf(stderr, "unknown CFA opcode 0x%02X\n", opcode); + _LIBUNWIND_TRACE_DWARF("unknown CFA opcode 0x%02X\n", opcode); return false; } } } return true; } } // namespace libunwind #endif // __DWARF_PARSER_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/EHHeaderParser.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/EHHeaderParser.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/EHHeaderParser.hpp (revision 345026) @@ -1,161 +1,168 @@ //===------------------------- EHHeaderParser.hpp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Parses ELF .eh_frame_hdr sections. // //===----------------------------------------------------------------------===// #ifndef __EHHEADERPARSER_HPP__ #define __EHHEADERPARSER_HPP__ #include "libunwind.h" -#include "AddressSpace.hpp" #include "DwarfParser.hpp" namespace libunwind { /// \brief EHHeaderParser does basic parsing of an ELF .eh_frame_hdr section. /// /// See DWARF spec for details: /// http://refspecs.linuxbase.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html /// template class EHHeaderParser { public: typedef typename A::pint_t pint_t; /// Information encoded in the EH frame header. struct EHHeaderInfo { pint_t eh_frame_ptr; size_t fde_count; pint_t table; uint8_t table_enc; }; - static void decodeEHHdr(A &addressSpace, pint_t ehHdrStart, pint_t ehHdrEnd, + static bool decodeEHHdr(A &addressSpace, pint_t ehHdrStart, pint_t ehHdrEnd, EHHeaderInfo &ehHdrInfo); static bool findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart, uint32_t sectionLength, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo); private: static bool decodeTableEntry(A &addressSpace, pint_t &tableEntry, pint_t ehHdrStart, pint_t ehHdrEnd, uint8_t tableEnc, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo); static size_t getTableEntrySize(uint8_t tableEnc); }; template -void EHHeaderParser::decodeEHHdr(A &addressSpace, pint_t ehHdrStart, +bool EHHeaderParser::decodeEHHdr(A &addressSpace, pint_t ehHdrStart, pint_t ehHdrEnd, EHHeaderInfo &ehHdrInfo) { pint_t p = ehHdrStart; uint8_t version = addressSpace.get8(p++); - if (version != 1) - _LIBUNWIND_ABORT("Unsupported .eh_frame_hdr version"); + if (version != 1) { + _LIBUNWIND_LOG0("Unsupported .eh_frame_hdr version"); + return false; + } uint8_t eh_frame_ptr_enc = addressSpace.get8(p++); uint8_t fde_count_enc = addressSpace.get8(p++); ehHdrInfo.table_enc = addressSpace.get8(p++); ehHdrInfo.eh_frame_ptr = addressSpace.getEncodedP(p, ehHdrEnd, eh_frame_ptr_enc, ehHdrStart); ehHdrInfo.fde_count = - addressSpace.getEncodedP(p, ehHdrEnd, fde_count_enc, ehHdrStart); + fde_count_enc == DW_EH_PE_omit + ? 0 + : addressSpace.getEncodedP(p, ehHdrEnd, fde_count_enc, ehHdrStart); ehHdrInfo.table = p; + + return true; } template bool EHHeaderParser::decodeTableEntry( A &addressSpace, pint_t &tableEntry, pint_t ehHdrStart, pint_t ehHdrEnd, uint8_t tableEnc, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo) { // Have to decode the whole FDE for the PC range anyway, so just throw away // the PC start. addressSpace.getEncodedP(tableEntry, ehHdrEnd, tableEnc, ehHdrStart); pint_t fde = addressSpace.getEncodedP(tableEntry, ehHdrEnd, tableEnc, ehHdrStart); const char *message = CFI_Parser::decodeFDE(addressSpace, fde, fdeInfo, cieInfo); if (message != NULL) { _LIBUNWIND_DEBUG_LOG("EHHeaderParser::decodeTableEntry: bad fde: %s", message); return false; } return true; } template bool EHHeaderParser::findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart, uint32_t sectionLength, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo) { pint_t ehHdrEnd = ehHdrStart + sectionLength; EHHeaderParser::EHHeaderInfo hdrInfo; - EHHeaderParser::decodeEHHdr(addressSpace, ehHdrStart, ehHdrEnd, hdrInfo); + if (!EHHeaderParser::decodeEHHdr(addressSpace, ehHdrStart, ehHdrEnd, + hdrInfo)) + return false; size_t tableEntrySize = getTableEntrySize(hdrInfo.table_enc); pint_t tableEntry; size_t low = 0; for (size_t len = hdrInfo.fde_count; len > 1;) { size_t mid = low + (len / 2); tableEntry = hdrInfo.table + mid * tableEntrySize; pint_t start = addressSpace.getEncodedP(tableEntry, ehHdrEnd, hdrInfo.table_enc, ehHdrStart); if (start == pc) { low = mid; break; } else if (start < pc) { low = mid; len -= (len / 2); } else { len /= 2; } } tableEntry = hdrInfo.table + low * tableEntrySize; if (decodeTableEntry(addressSpace, tableEntry, ehHdrStart, ehHdrEnd, hdrInfo.table_enc, fdeInfo, cieInfo)) { if (pc >= fdeInfo->pcStart && pc < fdeInfo->pcEnd) return true; } return false; } template size_t EHHeaderParser::getTableEntrySize(uint8_t tableEnc) { switch (tableEnc & 0x0f) { case DW_EH_PE_sdata2: case DW_EH_PE_udata2: return 4; case DW_EH_PE_sdata4: case DW_EH_PE_udata4: return 8; case DW_EH_PE_sdata8: case DW_EH_PE_udata8: return 16; case DW_EH_PE_sleb128: case DW_EH_PE_uleb128: _LIBUNWIND_ABORT("Can't binary search on variable length encoded data."); case DW_EH_PE_omit: return 0; default: _LIBUNWIND_ABORT("Unknown DWARF encoding for search table."); } } } #endif Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/RWMutex.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/RWMutex.hpp (nonexistent) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/RWMutex.hpp (revision 345026) @@ -0,0 +1,77 @@ +//===----------------------------- Registers.hpp --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +// +// Abstract interface to shared reader/writer log, hiding platform and +// configuration differences. +// +//===----------------------------------------------------------------------===// + +#ifndef __RWMUTEX_HPP__ +#define __RWMUTEX_HPP__ + +#if defined(_WIN32) +#include +#elif !defined(_LIBUNWIND_HAS_NO_THREADS) +#include +#endif + +namespace libunwind { + +#if defined(_LIBUNWIND_HAS_NO_THREADS) + +class _LIBUNWIND_HIDDEN RWMutex { +public: + bool lock_shared() { return true; } + bool unlock_shared() { return true; } + bool lock() { return true; } + bool unlock() { return true; } +}; + +#elif defined(_WIN32) + +class _LIBUNWIND_HIDDEN RWMutex { +public: + bool lock_shared() { + AcquireSRWLockShared(&_lock); + return true; + } + bool unlock_shared() { + ReleaseSRWLockShared(&_lock); + return true; + } + bool lock() { + AcquireSRWLockExclusive(&_lock); + return true; + } + bool unlock() { + ReleaseSRWLockExclusive(&_lock); + return true; + } + +private: + SRWLOCK _lock = SRWLOCK_INIT; +}; + +#else + +class _LIBUNWIND_HIDDEN RWMutex { +public: + bool lock_shared() { return pthread_rwlock_rdlock(&_lock) == 0; } + bool unlock_shared() { return pthread_rwlock_unlock(&_lock) == 0; } + bool lock() { return pthread_rwlock_wrlock(&_lock) == 0; } + bool unlock() { return pthread_rwlock_unlock(&_lock) == 0; } + +private: + pthread_rwlock_t _lock = PTHREAD_RWLOCK_INITIALIZER; +}; + +#endif + +} // namespace libunwind + +#endif // __RWMUTEX_HPP__ Property changes on: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/RWMutex.hpp ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Registers.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Registers.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Registers.hpp (revision 345026) @@ -1,2779 +1,3785 @@ //===----------------------------- Registers.hpp --------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Models register sets for supported processors. // //===----------------------------------------------------------------------===// #ifndef __REGISTERS_HPP__ #define __REGISTERS_HPP__ #include #include #include "libunwind.h" #include "config.h" namespace libunwind { // For emulating 128-bit registers struct v128 { uint32_t vec[4]; }; +enum { + REGISTERS_X86, + REGISTERS_X86_64, + REGISTERS_PPC, + REGISTERS_PPC64, + REGISTERS_ARM64, + REGISTERS_ARM, + REGISTERS_OR1K, + REGISTERS_RISCV, + REGISTERS_MIPS_O32, + REGISTERS_MIPS_NEWABI, + REGISTERS_SPARC, +}; #if defined(_LIBUNWIND_TARGET_I386) /// Registers_x86 holds the register state of a thread in a 32-bit intel /// process. class _LIBUNWIND_HIDDEN Registers_x86 { public: Registers_x86(); Registers_x86(const void *registers); bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); bool validFloatRegister(int) const { return false; } double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int) const { return false; } v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 8; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86; } + static int getArch() { return REGISTERS_X86; } uint32_t getSP() const { return _registers.__esp; } void setSP(uint32_t value) { _registers.__esp = value; } uint32_t getIP() const { return _registers.__eip; } void setIP(uint32_t value) { _registers.__eip = value; } uint32_t getEBP() const { return _registers.__ebp; } void setEBP(uint32_t value) { _registers.__ebp = value; } uint32_t getEBX() const { return _registers.__ebx; } void setEBX(uint32_t value) { _registers.__ebx = value; } uint32_t getECX() const { return _registers.__ecx; } void setECX(uint32_t value) { _registers.__ecx = value; } uint32_t getEDX() const { return _registers.__edx; } void setEDX(uint32_t value) { _registers.__edx = value; } uint32_t getESI() const { return _registers.__esi; } void setESI(uint32_t value) { _registers.__esi = value; } uint32_t getEDI() const { return _registers.__edi; } void setEDI(uint32_t value) { _registers.__edi = value; } private: struct GPRs { unsigned int __eax; unsigned int __ebx; unsigned int __ecx; unsigned int __edx; unsigned int __edi; unsigned int __esi; unsigned int __ebp; unsigned int __esp; unsigned int __ss; unsigned int __eflags; unsigned int __eip; unsigned int __cs; unsigned int __ds; unsigned int __es; unsigned int __fs; unsigned int __gs; }; GPRs _registers; }; inline Registers_x86::Registers_x86(const void *registers) { static_assert((check_fit::does_fit), "x86 registers do not fit into unw_context_t"); memcpy(&_registers, registers, sizeof(_registers)); } inline Registers_x86::Registers_x86() { memset(&_registers, 0, sizeof(_registers)); } inline bool Registers_x86::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum > 7) return false; return true; } inline uint32_t Registers_x86::getRegister(int regNum) const { switch (regNum) { case UNW_REG_IP: return _registers.__eip; case UNW_REG_SP: return _registers.__esp; case UNW_X86_EAX: return _registers.__eax; case UNW_X86_ECX: return _registers.__ecx; case UNW_X86_EDX: return _registers.__edx; case UNW_X86_EBX: return _registers.__ebx; +#if !defined(__APPLE__) + case UNW_X86_ESP: +#else case UNW_X86_EBP: +#endif return _registers.__ebp; +#if !defined(__APPLE__) + case UNW_X86_EBP: +#else case UNW_X86_ESP: +#endif return _registers.__esp; case UNW_X86_ESI: return _registers.__esi; case UNW_X86_EDI: return _registers.__edi; } _LIBUNWIND_ABORT("unsupported x86 register"); } inline void Registers_x86::setRegister(int regNum, uint32_t value) { switch (regNum) { case UNW_REG_IP: _registers.__eip = value; return; case UNW_REG_SP: _registers.__esp = value; return; case UNW_X86_EAX: _registers.__eax = value; return; case UNW_X86_ECX: _registers.__ecx = value; return; case UNW_X86_EDX: _registers.__edx = value; return; case UNW_X86_EBX: _registers.__ebx = value; return; +#if !defined(__APPLE__) + case UNW_X86_ESP: +#else case UNW_X86_EBP: +#endif _registers.__ebp = value; return; +#if !defined(__APPLE__) + case UNW_X86_EBP: +#else case UNW_X86_ESP: +#endif _registers.__esp = value; return; case UNW_X86_ESI: _registers.__esi = value; return; case UNW_X86_EDI: _registers.__edi = value; return; } _LIBUNWIND_ABORT("unsupported x86 register"); } inline const char *Registers_x86::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: return "ip"; case UNW_REG_SP: return "esp"; case UNW_X86_EAX: return "eax"; case UNW_X86_ECX: return "ecx"; case UNW_X86_EDX: return "edx"; case UNW_X86_EBX: return "ebx"; case UNW_X86_EBP: return "ebp"; case UNW_X86_ESP: return "esp"; case UNW_X86_ESI: return "esi"; case UNW_X86_EDI: return "edi"; default: return "unknown register"; } } inline double Registers_x86::getFloatRegister(int) const { _LIBUNWIND_ABORT("no x86 float registers"); } inline void Registers_x86::setFloatRegister(int, double) { _LIBUNWIND_ABORT("no x86 float registers"); } inline v128 Registers_x86::getVectorRegister(int) const { _LIBUNWIND_ABORT("no x86 vector registers"); } inline void Registers_x86::setVectorRegister(int, v128) { _LIBUNWIND_ABORT("no x86 vector registers"); } #endif // _LIBUNWIND_TARGET_I386 #if defined(_LIBUNWIND_TARGET_X86_64) /// Registers_x86_64 holds the register state of a thread in a 64-bit intel /// process. class _LIBUNWIND_HIDDEN Registers_x86_64 { public: Registers_x86_64(); Registers_x86_64(const void *registers); bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); bool validFloatRegister(int) const { return false; } double getFloatRegister(int num) const; void setFloatRegister(int num, double value); - bool validVectorRegister(int) const { return false; } + bool validVectorRegister(int) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 16; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_X86_64; } + static int getArch() { return REGISTERS_X86_64; } uint64_t getSP() const { return _registers.__rsp; } void setSP(uint64_t value) { _registers.__rsp = value; } uint64_t getIP() const { return _registers.__rip; } void setIP(uint64_t value) { _registers.__rip = value; } uint64_t getRBP() const { return _registers.__rbp; } void setRBP(uint64_t value) { _registers.__rbp = value; } uint64_t getRBX() const { return _registers.__rbx; } void setRBX(uint64_t value) { _registers.__rbx = value; } uint64_t getR12() const { return _registers.__r12; } void setR12(uint64_t value) { _registers.__r12 = value; } uint64_t getR13() const { return _registers.__r13; } void setR13(uint64_t value) { _registers.__r13 = value; } uint64_t getR14() const { return _registers.__r14; } void setR14(uint64_t value) { _registers.__r14 = value; } uint64_t getR15() const { return _registers.__r15; } void setR15(uint64_t value) { _registers.__r15 = value; } private: struct GPRs { uint64_t __rax; uint64_t __rbx; uint64_t __rcx; uint64_t __rdx; uint64_t __rdi; uint64_t __rsi; uint64_t __rbp; uint64_t __rsp; uint64_t __r8; uint64_t __r9; uint64_t __r10; uint64_t __r11; uint64_t __r12; uint64_t __r13; uint64_t __r14; uint64_t __r15; uint64_t __rip; uint64_t __rflags; uint64_t __cs; uint64_t __fs; uint64_t __gs; +#if defined(_WIN64) + uint64_t __padding; // 16-byte align +#endif }; GPRs _registers; +#if defined(_WIN64) + v128 _xmm[16]; +#endif }; inline Registers_x86_64::Registers_x86_64(const void *registers) { static_assert((check_fit::does_fit), "x86_64 registers do not fit into unw_context_t"); memcpy(&_registers, registers, sizeof(_registers)); } inline Registers_x86_64::Registers_x86_64() { memset(&_registers, 0, sizeof(_registers)); } inline bool Registers_x86_64::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum > 15) return false; return true; } inline uint64_t Registers_x86_64::getRegister(int regNum) const { switch (regNum) { case UNW_REG_IP: return _registers.__rip; case UNW_REG_SP: return _registers.__rsp; case UNW_X86_64_RAX: return _registers.__rax; case UNW_X86_64_RDX: return _registers.__rdx; case UNW_X86_64_RCX: return _registers.__rcx; case UNW_X86_64_RBX: return _registers.__rbx; case UNW_X86_64_RSI: return _registers.__rsi; case UNW_X86_64_RDI: return _registers.__rdi; case UNW_X86_64_RBP: return _registers.__rbp; case UNW_X86_64_RSP: return _registers.__rsp; case UNW_X86_64_R8: return _registers.__r8; case UNW_X86_64_R9: return _registers.__r9; case UNW_X86_64_R10: return _registers.__r10; case UNW_X86_64_R11: return _registers.__r11; case UNW_X86_64_R12: return _registers.__r12; case UNW_X86_64_R13: return _registers.__r13; case UNW_X86_64_R14: return _registers.__r14; case UNW_X86_64_R15: return _registers.__r15; } _LIBUNWIND_ABORT("unsupported x86_64 register"); } inline void Registers_x86_64::setRegister(int regNum, uint64_t value) { switch (regNum) { case UNW_REG_IP: _registers.__rip = value; return; case UNW_REG_SP: _registers.__rsp = value; return; case UNW_X86_64_RAX: _registers.__rax = value; return; case UNW_X86_64_RDX: _registers.__rdx = value; return; case UNW_X86_64_RCX: _registers.__rcx = value; return; case UNW_X86_64_RBX: _registers.__rbx = value; return; case UNW_X86_64_RSI: _registers.__rsi = value; return; case UNW_X86_64_RDI: _registers.__rdi = value; return; case UNW_X86_64_RBP: _registers.__rbp = value; return; case UNW_X86_64_RSP: _registers.__rsp = value; return; case UNW_X86_64_R8: _registers.__r8 = value; return; case UNW_X86_64_R9: _registers.__r9 = value; return; case UNW_X86_64_R10: _registers.__r10 = value; return; case UNW_X86_64_R11: _registers.__r11 = value; return; case UNW_X86_64_R12: _registers.__r12 = value; return; case UNW_X86_64_R13: _registers.__r13 = value; return; case UNW_X86_64_R14: _registers.__r14 = value; return; case UNW_X86_64_R15: _registers.__r15 = value; return; } _LIBUNWIND_ABORT("unsupported x86_64 register"); } inline const char *Registers_x86_64::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: return "rip"; case UNW_REG_SP: return "rsp"; case UNW_X86_64_RAX: return "rax"; case UNW_X86_64_RDX: return "rdx"; case UNW_X86_64_RCX: return "rcx"; case UNW_X86_64_RBX: return "rbx"; case UNW_X86_64_RSI: return "rsi"; case UNW_X86_64_RDI: return "rdi"; case UNW_X86_64_RBP: return "rbp"; case UNW_X86_64_RSP: return "rsp"; case UNW_X86_64_R8: return "r8"; case UNW_X86_64_R9: return "r9"; case UNW_X86_64_R10: return "r10"; case UNW_X86_64_R11: return "r11"; case UNW_X86_64_R12: return "r12"; case UNW_X86_64_R13: return "r13"; case UNW_X86_64_R14: return "r14"; case UNW_X86_64_R15: return "r15"; + case UNW_X86_64_XMM0: + return "xmm0"; + case UNW_X86_64_XMM1: + return "xmm1"; + case UNW_X86_64_XMM2: + return "xmm2"; + case UNW_X86_64_XMM3: + return "xmm3"; + case UNW_X86_64_XMM4: + return "xmm4"; + case UNW_X86_64_XMM5: + return "xmm5"; + case UNW_X86_64_XMM6: + return "xmm6"; + case UNW_X86_64_XMM7: + return "xmm7"; + case UNW_X86_64_XMM8: + return "xmm8"; + case UNW_X86_64_XMM9: + return "xmm9"; + case UNW_X86_64_XMM10: + return "xmm10"; + case UNW_X86_64_XMM11: + return "xmm11"; + case UNW_X86_64_XMM12: + return "xmm12"; + case UNW_X86_64_XMM13: + return "xmm13"; + case UNW_X86_64_XMM14: + return "xmm14"; + case UNW_X86_64_XMM15: + return "xmm15"; default: return "unknown register"; } } inline double Registers_x86_64::getFloatRegister(int) const { _LIBUNWIND_ABORT("no x86_64 float registers"); } inline void Registers_x86_64::setFloatRegister(int, double) { _LIBUNWIND_ABORT("no x86_64 float registers"); } -inline v128 Registers_x86_64::getVectorRegister(int) const { +inline bool Registers_x86_64::validVectorRegister(int regNum) const { +#if defined(_WIN64) + if (regNum < UNW_X86_64_XMM0) + return false; + if (regNum > UNW_X86_64_XMM15) + return false; + return true; +#else + (void)regNum; // suppress unused parameter warning + return false; +#endif +} + +inline v128 Registers_x86_64::getVectorRegister(int regNum) const { +#if defined(_WIN64) + assert(validVectorRegister(regNum)); + return _xmm[regNum - UNW_X86_64_XMM0]; +#else + (void)regNum; // suppress unused parameter warning _LIBUNWIND_ABORT("no x86_64 vector registers"); +#endif } -inline void Registers_x86_64::setVectorRegister(int, v128) { +inline void Registers_x86_64::setVectorRegister(int regNum, v128 value) { +#if defined(_WIN64) + assert(validVectorRegister(regNum)); + _xmm[regNum - UNW_X86_64_XMM0] = value; +#else + (void)regNum; (void)value; // suppress unused parameter warnings _LIBUNWIND_ABORT("no x86_64 vector registers"); +#endif } #endif // _LIBUNWIND_TARGET_X86_64 #if defined(_LIBUNWIND_TARGET_PPC) /// Registers_ppc holds the register state of a thread in a 32-bit PowerPC /// process. class _LIBUNWIND_HIDDEN Registers_ppc { public: Registers_ppc(); Registers_ppc(const void *registers); bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 112; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC; } + static int getArch() { return REGISTERS_PPC; } uint64_t getSP() const { return _registers.__r1; } void setSP(uint32_t value) { _registers.__r1 = value; } uint64_t getIP() const { return _registers.__srr0; } void setIP(uint32_t value) { _registers.__srr0 = value; } private: struct ppc_thread_state_t { unsigned int __srr0; /* Instruction address register (PC) */ unsigned int __srr1; /* Machine state register (supervisor) */ unsigned int __r0; unsigned int __r1; unsigned int __r2; unsigned int __r3; unsigned int __r4; unsigned int __r5; unsigned int __r6; unsigned int __r7; unsigned int __r8; unsigned int __r9; unsigned int __r10; unsigned int __r11; unsigned int __r12; unsigned int __r13; unsigned int __r14; unsigned int __r15; unsigned int __r16; unsigned int __r17; unsigned int __r18; unsigned int __r19; unsigned int __r20; unsigned int __r21; unsigned int __r22; unsigned int __r23; unsigned int __r24; unsigned int __r25; unsigned int __r26; unsigned int __r27; unsigned int __r28; unsigned int __r29; unsigned int __r30; unsigned int __r31; unsigned int __cr; /* Condition register */ unsigned int __xer; /* User's integer exception register */ unsigned int __lr; /* Link register */ unsigned int __ctr; /* Count register */ unsigned int __mq; /* MQ register (601 only) */ unsigned int __vrsave; /* Vector Save Register */ }; struct ppc_float_state_t { double __fpregs[32]; unsigned int __fpscr_pad; /* fpscr is 64 bits, 32 bits of rubbish */ unsigned int __fpscr; /* floating point status register */ }; ppc_thread_state_t _registers; ppc_float_state_t _floatRegisters; v128 _vectorRegisters[32]; // offset 424 }; inline Registers_ppc::Registers_ppc(const void *registers) { static_assert((check_fit::does_fit), "ppc registers do not fit into unw_context_t"); memcpy(&_registers, static_cast(registers), sizeof(_registers)); static_assert(sizeof(ppc_thread_state_t) == 160, "expected float register offset to be 160"); memcpy(&_floatRegisters, static_cast(registers) + sizeof(ppc_thread_state_t), sizeof(_floatRegisters)); static_assert(sizeof(ppc_thread_state_t) + sizeof(ppc_float_state_t) == 424, "expected vector register offset to be 424 bytes"); memcpy(_vectorRegisters, static_cast(registers) + sizeof(ppc_thread_state_t) + sizeof(ppc_float_state_t), sizeof(_vectorRegisters)); } inline Registers_ppc::Registers_ppc() { memset(&_registers, 0, sizeof(_registers)); memset(&_floatRegisters, 0, sizeof(_floatRegisters)); memset(&_vectorRegisters, 0, sizeof(_vectorRegisters)); } inline bool Registers_ppc::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum == UNW_PPC_VRSAVE) return true; if (regNum < 0) return false; if (regNum <= UNW_PPC_R31) return true; if (regNum == UNW_PPC_MQ) return true; if (regNum == UNW_PPC_LR) return true; if (regNum == UNW_PPC_CTR) return true; if ((UNW_PPC_CR0 <= regNum) && (regNum <= UNW_PPC_CR7)) return true; return false; } inline uint32_t Registers_ppc::getRegister(int regNum) const { switch (regNum) { case UNW_REG_IP: return _registers.__srr0; case UNW_REG_SP: return _registers.__r1; case UNW_PPC_R0: return _registers.__r0; case UNW_PPC_R1: return _registers.__r1; case UNW_PPC_R2: return _registers.__r2; case UNW_PPC_R3: return _registers.__r3; case UNW_PPC_R4: return _registers.__r4; case UNW_PPC_R5: return _registers.__r5; case UNW_PPC_R6: return _registers.__r6; case UNW_PPC_R7: return _registers.__r7; case UNW_PPC_R8: return _registers.__r8; case UNW_PPC_R9: return _registers.__r9; case UNW_PPC_R10: return _registers.__r10; case UNW_PPC_R11: return _registers.__r11; case UNW_PPC_R12: return _registers.__r12; case UNW_PPC_R13: return _registers.__r13; case UNW_PPC_R14: return _registers.__r14; case UNW_PPC_R15: return _registers.__r15; case UNW_PPC_R16: return _registers.__r16; case UNW_PPC_R17: return _registers.__r17; case UNW_PPC_R18: return _registers.__r18; case UNW_PPC_R19: return _registers.__r19; case UNW_PPC_R20: return _registers.__r20; case UNW_PPC_R21: return _registers.__r21; case UNW_PPC_R22: return _registers.__r22; case UNW_PPC_R23: return _registers.__r23; case UNW_PPC_R24: return _registers.__r24; case UNW_PPC_R25: return _registers.__r25; case UNW_PPC_R26: return _registers.__r26; case UNW_PPC_R27: return _registers.__r27; case UNW_PPC_R28: return _registers.__r28; case UNW_PPC_R29: return _registers.__r29; case UNW_PPC_R30: return _registers.__r30; case UNW_PPC_R31: return _registers.__r31; case UNW_PPC_LR: return _registers.__lr; case UNW_PPC_CR0: return (_registers.__cr & 0xF0000000); case UNW_PPC_CR1: return (_registers.__cr & 0x0F000000); case UNW_PPC_CR2: return (_registers.__cr & 0x00F00000); case UNW_PPC_CR3: return (_registers.__cr & 0x000F0000); case UNW_PPC_CR4: return (_registers.__cr & 0x0000F000); case UNW_PPC_CR5: return (_registers.__cr & 0x00000F00); case UNW_PPC_CR6: return (_registers.__cr & 0x000000F0); case UNW_PPC_CR7: return (_registers.__cr & 0x0000000F); case UNW_PPC_VRSAVE: return _registers.__vrsave; } _LIBUNWIND_ABORT("unsupported ppc register"); } inline void Registers_ppc::setRegister(int regNum, uint32_t value) { //fprintf(stderr, "Registers_ppc::setRegister(%d, 0x%08X)\n", regNum, value); switch (regNum) { case UNW_REG_IP: _registers.__srr0 = value; return; case UNW_REG_SP: _registers.__r1 = value; return; case UNW_PPC_R0: _registers.__r0 = value; return; case UNW_PPC_R1: _registers.__r1 = value; return; case UNW_PPC_R2: _registers.__r2 = value; return; case UNW_PPC_R3: _registers.__r3 = value; return; case UNW_PPC_R4: _registers.__r4 = value; return; case UNW_PPC_R5: _registers.__r5 = value; return; case UNW_PPC_R6: _registers.__r6 = value; return; case UNW_PPC_R7: _registers.__r7 = value; return; case UNW_PPC_R8: _registers.__r8 = value; return; case UNW_PPC_R9: _registers.__r9 = value; return; case UNW_PPC_R10: _registers.__r10 = value; return; case UNW_PPC_R11: _registers.__r11 = value; return; case UNW_PPC_R12: _registers.__r12 = value; return; case UNW_PPC_R13: _registers.__r13 = value; return; case UNW_PPC_R14: _registers.__r14 = value; return; case UNW_PPC_R15: _registers.__r15 = value; return; case UNW_PPC_R16: _registers.__r16 = value; return; case UNW_PPC_R17: _registers.__r17 = value; return; case UNW_PPC_R18: _registers.__r18 = value; return; case UNW_PPC_R19: _registers.__r19 = value; return; case UNW_PPC_R20: _registers.__r20 = value; return; case UNW_PPC_R21: _registers.__r21 = value; return; case UNW_PPC_R22: _registers.__r22 = value; return; case UNW_PPC_R23: _registers.__r23 = value; return; case UNW_PPC_R24: _registers.__r24 = value; return; case UNW_PPC_R25: _registers.__r25 = value; return; case UNW_PPC_R26: _registers.__r26 = value; return; case UNW_PPC_R27: _registers.__r27 = value; return; case UNW_PPC_R28: _registers.__r28 = value; return; case UNW_PPC_R29: _registers.__r29 = value; return; case UNW_PPC_R30: _registers.__r30 = value; return; case UNW_PPC_R31: _registers.__r31 = value; return; case UNW_PPC_MQ: _registers.__mq = value; return; case UNW_PPC_LR: _registers.__lr = value; return; case UNW_PPC_CTR: _registers.__ctr = value; return; case UNW_PPC_CR0: _registers.__cr &= 0x0FFFFFFF; _registers.__cr |= (value & 0xF0000000); return; case UNW_PPC_CR1: _registers.__cr &= 0xF0FFFFFF; _registers.__cr |= (value & 0x0F000000); return; case UNW_PPC_CR2: _registers.__cr &= 0xFF0FFFFF; _registers.__cr |= (value & 0x00F00000); return; case UNW_PPC_CR3: _registers.__cr &= 0xFFF0FFFF; _registers.__cr |= (value & 0x000F0000); return; case UNW_PPC_CR4: _registers.__cr &= 0xFFFF0FFF; _registers.__cr |= (value & 0x0000F000); return; case UNW_PPC_CR5: _registers.__cr &= 0xFFFFF0FF; _registers.__cr |= (value & 0x00000F00); return; case UNW_PPC_CR6: _registers.__cr &= 0xFFFFFF0F; _registers.__cr |= (value & 0x000000F0); return; case UNW_PPC_CR7: _registers.__cr &= 0xFFFFFFF0; _registers.__cr |= (value & 0x0000000F); return; case UNW_PPC_VRSAVE: _registers.__vrsave = value; return; // not saved return; case UNW_PPC_XER: _registers.__xer = value; return; case UNW_PPC_AP: case UNW_PPC_VSCR: case UNW_PPC_SPEFSCR: // not saved return; } _LIBUNWIND_ABORT("unsupported ppc register"); } inline bool Registers_ppc::validFloatRegister(int regNum) const { if (regNum < UNW_PPC_F0) return false; if (regNum > UNW_PPC_F31) return false; return true; } inline double Registers_ppc::getFloatRegister(int regNum) const { assert(validFloatRegister(regNum)); return _floatRegisters.__fpregs[regNum - UNW_PPC_F0]; } inline void Registers_ppc::setFloatRegister(int regNum, double value) { assert(validFloatRegister(regNum)); _floatRegisters.__fpregs[regNum - UNW_PPC_F0] = value; } inline bool Registers_ppc::validVectorRegister(int regNum) const { if (regNum < UNW_PPC_V0) return false; if (regNum > UNW_PPC_V31) return false; return true; } inline v128 Registers_ppc::getVectorRegister(int regNum) const { assert(validVectorRegister(regNum)); v128 result = _vectorRegisters[regNum - UNW_PPC_V0]; return result; } inline void Registers_ppc::setVectorRegister(int regNum, v128 value) { assert(validVectorRegister(regNum)); _vectorRegisters[regNum - UNW_PPC_V0] = value; } inline const char *Registers_ppc::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: return "ip"; case UNW_REG_SP: return "sp"; case UNW_PPC_R0: return "r0"; case UNW_PPC_R1: return "r1"; case UNW_PPC_R2: return "r2"; case UNW_PPC_R3: return "r3"; case UNW_PPC_R4: return "r4"; case UNW_PPC_R5: return "r5"; case UNW_PPC_R6: return "r6"; case UNW_PPC_R7: return "r7"; case UNW_PPC_R8: return "r8"; case UNW_PPC_R9: return "r9"; case UNW_PPC_R10: return "r10"; case UNW_PPC_R11: return "r11"; case UNW_PPC_R12: return "r12"; case UNW_PPC_R13: return "r13"; case UNW_PPC_R14: return "r14"; case UNW_PPC_R15: return "r15"; case UNW_PPC_R16: return "r16"; case UNW_PPC_R17: return "r17"; case UNW_PPC_R18: return "r18"; case UNW_PPC_R19: return "r19"; case UNW_PPC_R20: return "r20"; case UNW_PPC_R21: return "r21"; case UNW_PPC_R22: return "r22"; case UNW_PPC_R23: return "r23"; case UNW_PPC_R24: return "r24"; case UNW_PPC_R25: return "r25"; case UNW_PPC_R26: return "r26"; case UNW_PPC_R27: return "r27"; case UNW_PPC_R28: return "r28"; case UNW_PPC_R29: return "r29"; case UNW_PPC_R30: return "r30"; case UNW_PPC_R31: return "r31"; case UNW_PPC_F0: return "fp0"; case UNW_PPC_F1: return "fp1"; case UNW_PPC_F2: return "fp2"; case UNW_PPC_F3: return "fp3"; case UNW_PPC_F4: return "fp4"; case UNW_PPC_F5: return "fp5"; case UNW_PPC_F6: return "fp6"; case UNW_PPC_F7: return "fp7"; case UNW_PPC_F8: return "fp8"; case UNW_PPC_F9: return "fp9"; case UNW_PPC_F10: return "fp10"; case UNW_PPC_F11: return "fp11"; case UNW_PPC_F12: return "fp12"; case UNW_PPC_F13: return "fp13"; case UNW_PPC_F14: return "fp14"; case UNW_PPC_F15: return "fp15"; case UNW_PPC_F16: return "fp16"; case UNW_PPC_F17: return "fp17"; case UNW_PPC_F18: return "fp18"; case UNW_PPC_F19: return "fp19"; case UNW_PPC_F20: return "fp20"; case UNW_PPC_F21: return "fp21"; case UNW_PPC_F22: return "fp22"; case UNW_PPC_F23: return "fp23"; case UNW_PPC_F24: return "fp24"; case UNW_PPC_F25: return "fp25"; case UNW_PPC_F26: return "fp26"; case UNW_PPC_F27: return "fp27"; case UNW_PPC_F28: return "fp28"; case UNW_PPC_F29: return "fp29"; case UNW_PPC_F30: return "fp30"; case UNW_PPC_F31: return "fp31"; case UNW_PPC_LR: return "lr"; default: return "unknown register"; } } #endif // _LIBUNWIND_TARGET_PPC +#if defined(_LIBUNWIND_TARGET_PPC64) +/// Registers_ppc64 holds the register state of a thread in a 64-bit PowerPC +/// process. +class _LIBUNWIND_HIDDEN Registers_ppc64 { +public: + Registers_ppc64(); + Registers_ppc64(const void *registers); + bool validRegister(int num) const; + uint64_t getRegister(int num) const; + void setRegister(int num, uint64_t value); + bool validFloatRegister(int num) const; + double getFloatRegister(int num) const; + void setFloatRegister(int num, double value); + bool validVectorRegister(int num) const; + v128 getVectorRegister(int num) const; + void setVectorRegister(int num, v128 value); + static const char *getRegisterName(int num); + void jumpto(); + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC64; } + static int getArch() { return REGISTERS_PPC64; } + + uint64_t getSP() const { return _registers.__r1; } + void setSP(uint64_t value) { _registers.__r1 = value; } + uint64_t getIP() const { return _registers.__srr0; } + void setIP(uint64_t value) { _registers.__srr0 = value; } + +private: + struct ppc64_thread_state_t { + uint64_t __srr0; // Instruction address register (PC) + uint64_t __srr1; // Machine state register (supervisor) + uint64_t __r0; + uint64_t __r1; + uint64_t __r2; + uint64_t __r3; + uint64_t __r4; + uint64_t __r5; + uint64_t __r6; + uint64_t __r7; + uint64_t __r8; + uint64_t __r9; + uint64_t __r10; + uint64_t __r11; + uint64_t __r12; + uint64_t __r13; + uint64_t __r14; + uint64_t __r15; + uint64_t __r16; + uint64_t __r17; + uint64_t __r18; + uint64_t __r19; + uint64_t __r20; + uint64_t __r21; + uint64_t __r22; + uint64_t __r23; + uint64_t __r24; + uint64_t __r25; + uint64_t __r26; + uint64_t __r27; + uint64_t __r28; + uint64_t __r29; + uint64_t __r30; + uint64_t __r31; + uint64_t __cr; // Condition register + uint64_t __xer; // User's integer exception register + uint64_t __lr; // Link register + uint64_t __ctr; // Count register + uint64_t __vrsave; // Vector Save Register + }; + + union ppc64_vsr_t { + struct asfloat_s { + double f; + uint64_t v2; + } asfloat; + v128 v; + }; + + ppc64_thread_state_t _registers; + ppc64_vsr_t _vectorScalarRegisters[64]; + + static int getVectorRegNum(int num); +}; + +inline Registers_ppc64::Registers_ppc64(const void *registers) { + static_assert((check_fit::does_fit), + "ppc64 registers do not fit into unw_context_t"); + memcpy(&_registers, static_cast(registers), + sizeof(_registers)); + static_assert(sizeof(_registers) == 312, + "expected vector scalar register offset to be 312"); + memcpy(&_vectorScalarRegisters, + static_cast(registers) + sizeof(_registers), + sizeof(_vectorScalarRegisters)); + static_assert(sizeof(_registers) + + sizeof(_vectorScalarRegisters) == 1336, + "expected vector register offset to be 1336 bytes"); +} + +inline Registers_ppc64::Registers_ppc64() { + memset(&_registers, 0, sizeof(_registers)); + memset(&_vectorScalarRegisters, 0, sizeof(_vectorScalarRegisters)); +} + +inline bool Registers_ppc64::validRegister(int regNum) const { + switch (regNum) { + case UNW_REG_IP: + case UNW_REG_SP: + case UNW_PPC64_XER: + case UNW_PPC64_LR: + case UNW_PPC64_CTR: + case UNW_PPC64_VRSAVE: + return true; + } + + if (regNum >= UNW_PPC64_R0 && regNum <= UNW_PPC64_R31) + return true; + if (regNum >= UNW_PPC64_CR0 && regNum <= UNW_PPC64_CR7) + return true; + + return false; +} + +inline uint64_t Registers_ppc64::getRegister(int regNum) const { + switch (regNum) { + case UNW_REG_IP: + return _registers.__srr0; + case UNW_PPC64_R0: + return _registers.__r0; + case UNW_PPC64_R1: + case UNW_REG_SP: + return _registers.__r1; + case UNW_PPC64_R2: + return _registers.__r2; + case UNW_PPC64_R3: + return _registers.__r3; + case UNW_PPC64_R4: + return _registers.__r4; + case UNW_PPC64_R5: + return _registers.__r5; + case UNW_PPC64_R6: + return _registers.__r6; + case UNW_PPC64_R7: + return _registers.__r7; + case UNW_PPC64_R8: + return _registers.__r8; + case UNW_PPC64_R9: + return _registers.__r9; + case UNW_PPC64_R10: + return _registers.__r10; + case UNW_PPC64_R11: + return _registers.__r11; + case UNW_PPC64_R12: + return _registers.__r12; + case UNW_PPC64_R13: + return _registers.__r13; + case UNW_PPC64_R14: + return _registers.__r14; + case UNW_PPC64_R15: + return _registers.__r15; + case UNW_PPC64_R16: + return _registers.__r16; + case UNW_PPC64_R17: + return _registers.__r17; + case UNW_PPC64_R18: + return _registers.__r18; + case UNW_PPC64_R19: + return _registers.__r19; + case UNW_PPC64_R20: + return _registers.__r20; + case UNW_PPC64_R21: + return _registers.__r21; + case UNW_PPC64_R22: + return _registers.__r22; + case UNW_PPC64_R23: + return _registers.__r23; + case UNW_PPC64_R24: + return _registers.__r24; + case UNW_PPC64_R25: + return _registers.__r25; + case UNW_PPC64_R26: + return _registers.__r26; + case UNW_PPC64_R27: + return _registers.__r27; + case UNW_PPC64_R28: + return _registers.__r28; + case UNW_PPC64_R29: + return _registers.__r29; + case UNW_PPC64_R30: + return _registers.__r30; + case UNW_PPC64_R31: + return _registers.__r31; + case UNW_PPC64_CR0: + return (_registers.__cr & 0xF0000000); + case UNW_PPC64_CR1: + return (_registers.__cr & 0x0F000000); + case UNW_PPC64_CR2: + return (_registers.__cr & 0x00F00000); + case UNW_PPC64_CR3: + return (_registers.__cr & 0x000F0000); + case UNW_PPC64_CR4: + return (_registers.__cr & 0x0000F000); + case UNW_PPC64_CR5: + return (_registers.__cr & 0x00000F00); + case UNW_PPC64_CR6: + return (_registers.__cr & 0x000000F0); + case UNW_PPC64_CR7: + return (_registers.__cr & 0x0000000F); + case UNW_PPC64_XER: + return _registers.__xer; + case UNW_PPC64_LR: + return _registers.__lr; + case UNW_PPC64_CTR: + return _registers.__ctr; + case UNW_PPC64_VRSAVE: + return _registers.__vrsave; + } + _LIBUNWIND_ABORT("unsupported ppc64 register"); +} + +inline void Registers_ppc64::setRegister(int regNum, uint64_t value) { + switch (regNum) { + case UNW_REG_IP: + _registers.__srr0 = value; + return; + case UNW_PPC64_R0: + _registers.__r0 = value; + return; + case UNW_PPC64_R1: + case UNW_REG_SP: + _registers.__r1 = value; + return; + case UNW_PPC64_R2: + _registers.__r2 = value; + return; + case UNW_PPC64_R3: + _registers.__r3 = value; + return; + case UNW_PPC64_R4: + _registers.__r4 = value; + return; + case UNW_PPC64_R5: + _registers.__r5 = value; + return; + case UNW_PPC64_R6: + _registers.__r6 = value; + return; + case UNW_PPC64_R7: + _registers.__r7 = value; + return; + case UNW_PPC64_R8: + _registers.__r8 = value; + return; + case UNW_PPC64_R9: + _registers.__r9 = value; + return; + case UNW_PPC64_R10: + _registers.__r10 = value; + return; + case UNW_PPC64_R11: + _registers.__r11 = value; + return; + case UNW_PPC64_R12: + _registers.__r12 = value; + return; + case UNW_PPC64_R13: + _registers.__r13 = value; + return; + case UNW_PPC64_R14: + _registers.__r14 = value; + return; + case UNW_PPC64_R15: + _registers.__r15 = value; + return; + case UNW_PPC64_R16: + _registers.__r16 = value; + return; + case UNW_PPC64_R17: + _registers.__r17 = value; + return; + case UNW_PPC64_R18: + _registers.__r18 = value; + return; + case UNW_PPC64_R19: + _registers.__r19 = value; + return; + case UNW_PPC64_R20: + _registers.__r20 = value; + return; + case UNW_PPC64_R21: + _registers.__r21 = value; + return; + case UNW_PPC64_R22: + _registers.__r22 = value; + return; + case UNW_PPC64_R23: + _registers.__r23 = value; + return; + case UNW_PPC64_R24: + _registers.__r24 = value; + return; + case UNW_PPC64_R25: + _registers.__r25 = value; + return; + case UNW_PPC64_R26: + _registers.__r26 = value; + return; + case UNW_PPC64_R27: + _registers.__r27 = value; + return; + case UNW_PPC64_R28: + _registers.__r28 = value; + return; + case UNW_PPC64_R29: + _registers.__r29 = value; + return; + case UNW_PPC64_R30: + _registers.__r30 = value; + return; + case UNW_PPC64_R31: + _registers.__r31 = value; + return; + case UNW_PPC64_CR0: + _registers.__cr &= 0x0FFFFFFF; + _registers.__cr |= (value & 0xF0000000); + return; + case UNW_PPC64_CR1: + _registers.__cr &= 0xF0FFFFFF; + _registers.__cr |= (value & 0x0F000000); + return; + case UNW_PPC64_CR2: + _registers.__cr &= 0xFF0FFFFF; + _registers.__cr |= (value & 0x00F00000); + return; + case UNW_PPC64_CR3: + _registers.__cr &= 0xFFF0FFFF; + _registers.__cr |= (value & 0x000F0000); + return; + case UNW_PPC64_CR4: + _registers.__cr &= 0xFFFF0FFF; + _registers.__cr |= (value & 0x0000F000); + return; + case UNW_PPC64_CR5: + _registers.__cr &= 0xFFFFF0FF; + _registers.__cr |= (value & 0x00000F00); + return; + case UNW_PPC64_CR6: + _registers.__cr &= 0xFFFFFF0F; + _registers.__cr |= (value & 0x000000F0); + return; + case UNW_PPC64_CR7: + _registers.__cr &= 0xFFFFFFF0; + _registers.__cr |= (value & 0x0000000F); + return; + case UNW_PPC64_XER: + _registers.__xer = value; + return; + case UNW_PPC64_LR: + _registers.__lr = value; + return; + case UNW_PPC64_CTR: + _registers.__ctr = value; + return; + case UNW_PPC64_VRSAVE: + _registers.__vrsave = value; + return; + } + _LIBUNWIND_ABORT("unsupported ppc64 register"); +} + +inline bool Registers_ppc64::validFloatRegister(int regNum) const { + return regNum >= UNW_PPC64_F0 && regNum <= UNW_PPC64_F31; +} + +inline double Registers_ppc64::getFloatRegister(int regNum) const { + assert(validFloatRegister(regNum)); + return _vectorScalarRegisters[regNum - UNW_PPC64_F0].asfloat.f; +} + +inline void Registers_ppc64::setFloatRegister(int regNum, double value) { + assert(validFloatRegister(regNum)); + _vectorScalarRegisters[regNum - UNW_PPC64_F0].asfloat.f = value; +} + +inline bool Registers_ppc64::validVectorRegister(int regNum) const { +#ifdef PPC64_HAS_VMX + if (regNum >= UNW_PPC64_VS0 && regNum <= UNW_PPC64_VS31) + return true; + if (regNum >= UNW_PPC64_VS32 && regNum <= UNW_PPC64_VS63) + return true; +#else + if (regNum >= UNW_PPC64_V0 && regNum <= UNW_PPC64_V31) + return true; +#endif + return false; +} + +inline int Registers_ppc64::getVectorRegNum(int num) +{ + if (num >= UNW_PPC64_VS0 && num <= UNW_PPC64_VS31) + return num - UNW_PPC64_VS0; + else + return num - UNW_PPC64_VS32 + 32; +} + +inline v128 Registers_ppc64::getVectorRegister(int regNum) const { + assert(validVectorRegister(regNum)); + return _vectorScalarRegisters[getVectorRegNum(regNum)].v; +} + +inline void Registers_ppc64::setVectorRegister(int regNum, v128 value) { + assert(validVectorRegister(regNum)); + _vectorScalarRegisters[getVectorRegNum(regNum)].v = value; +} + +inline const char *Registers_ppc64::getRegisterName(int regNum) { + switch (regNum) { + case UNW_REG_IP: + return "ip"; + case UNW_REG_SP: + return "sp"; + case UNW_PPC64_R0: + return "r0"; + case UNW_PPC64_R1: + return "r1"; + case UNW_PPC64_R2: + return "r2"; + case UNW_PPC64_R3: + return "r3"; + case UNW_PPC64_R4: + return "r4"; + case UNW_PPC64_R5: + return "r5"; + case UNW_PPC64_R6: + return "r6"; + case UNW_PPC64_R7: + return "r7"; + case UNW_PPC64_R8: + return "r8"; + case UNW_PPC64_R9: + return "r9"; + case UNW_PPC64_R10: + return "r10"; + case UNW_PPC64_R11: + return "r11"; + case UNW_PPC64_R12: + return "r12"; + case UNW_PPC64_R13: + return "r13"; + case UNW_PPC64_R14: + return "r14"; + case UNW_PPC64_R15: + return "r15"; + case UNW_PPC64_R16: + return "r16"; + case UNW_PPC64_R17: + return "r17"; + case UNW_PPC64_R18: + return "r18"; + case UNW_PPC64_R19: + return "r19"; + case UNW_PPC64_R20: + return "r20"; + case UNW_PPC64_R21: + return "r21"; + case UNW_PPC64_R22: + return "r22"; + case UNW_PPC64_R23: + return "r23"; + case UNW_PPC64_R24: + return "r24"; + case UNW_PPC64_R25: + return "r25"; + case UNW_PPC64_R26: + return "r26"; + case UNW_PPC64_R27: + return "r27"; + case UNW_PPC64_R28: + return "r28"; + case UNW_PPC64_R29: + return "r29"; + case UNW_PPC64_R30: + return "r30"; + case UNW_PPC64_R31: + return "r31"; + case UNW_PPC64_CR0: + return "cr0"; + case UNW_PPC64_CR1: + return "cr1"; + case UNW_PPC64_CR2: + return "cr2"; + case UNW_PPC64_CR3: + return "cr3"; + case UNW_PPC64_CR4: + return "cr4"; + case UNW_PPC64_CR5: + return "cr5"; + case UNW_PPC64_CR6: + return "cr6"; + case UNW_PPC64_CR7: + return "cr7"; + case UNW_PPC64_XER: + return "xer"; + case UNW_PPC64_LR: + return "lr"; + case UNW_PPC64_CTR: + return "ctr"; + case UNW_PPC64_VRSAVE: + return "vrsave"; + case UNW_PPC64_F0: + return "fp0"; + case UNW_PPC64_F1: + return "fp1"; + case UNW_PPC64_F2: + return "fp2"; + case UNW_PPC64_F3: + return "fp3"; + case UNW_PPC64_F4: + return "fp4"; + case UNW_PPC64_F5: + return "fp5"; + case UNW_PPC64_F6: + return "fp6"; + case UNW_PPC64_F7: + return "fp7"; + case UNW_PPC64_F8: + return "fp8"; + case UNW_PPC64_F9: + return "fp9"; + case UNW_PPC64_F10: + return "fp10"; + case UNW_PPC64_F11: + return "fp11"; + case UNW_PPC64_F12: + return "fp12"; + case UNW_PPC64_F13: + return "fp13"; + case UNW_PPC64_F14: + return "fp14"; + case UNW_PPC64_F15: + return "fp15"; + case UNW_PPC64_F16: + return "fp16"; + case UNW_PPC64_F17: + return "fp17"; + case UNW_PPC64_F18: + return "fp18"; + case UNW_PPC64_F19: + return "fp19"; + case UNW_PPC64_F20: + return "fp20"; + case UNW_PPC64_F21: + return "fp21"; + case UNW_PPC64_F22: + return "fp22"; + case UNW_PPC64_F23: + return "fp23"; + case UNW_PPC64_F24: + return "fp24"; + case UNW_PPC64_F25: + return "fp25"; + case UNW_PPC64_F26: + return "fp26"; + case UNW_PPC64_F27: + return "fp27"; + case UNW_PPC64_F28: + return "fp28"; + case UNW_PPC64_F29: + return "fp29"; + case UNW_PPC64_F30: + return "fp30"; + case UNW_PPC64_F31: + return "fp31"; + case UNW_PPC64_V0: + return "v0"; + case UNW_PPC64_V1: + return "v1"; + case UNW_PPC64_V2: + return "v2"; + case UNW_PPC64_V3: + return "v3"; + case UNW_PPC64_V4: + return "v4"; + case UNW_PPC64_V5: + return "v5"; + case UNW_PPC64_V6: + return "v6"; + case UNW_PPC64_V7: + return "v7"; + case UNW_PPC64_V8: + return "v8"; + case UNW_PPC64_V9: + return "v9"; + case UNW_PPC64_V10: + return "v10"; + case UNW_PPC64_V11: + return "v11"; + case UNW_PPC64_V12: + return "v12"; + case UNW_PPC64_V13: + return "v13"; + case UNW_PPC64_V14: + return "v14"; + case UNW_PPC64_V15: + return "v15"; + case UNW_PPC64_V16: + return "v16"; + case UNW_PPC64_V17: + return "v17"; + case UNW_PPC64_V18: + return "v18"; + case UNW_PPC64_V19: + return "v19"; + case UNW_PPC64_V20: + return "v20"; + case UNW_PPC64_V21: + return "v21"; + case UNW_PPC64_V22: + return "v22"; + case UNW_PPC64_V23: + return "v23"; + case UNW_PPC64_V24: + return "v24"; + case UNW_PPC64_V25: + return "v25"; + case UNW_PPC64_V26: + return "v26"; + case UNW_PPC64_V27: + return "v27"; + case UNW_PPC64_V28: + return "v28"; + case UNW_PPC64_V29: + return "v29"; + case UNW_PPC64_V30: + return "v30"; + case UNW_PPC64_V31: + return "v31"; + } + return "unknown register"; +} +#endif // _LIBUNWIND_TARGET_PPC64 + + #if defined(_LIBUNWIND_TARGET_AARCH64) /// Registers_arm64 holds the register state of a thread in a 64-bit arm /// process. class _LIBUNWIND_HIDDEN Registers_arm64 { public: Registers_arm64(); Registers_arm64(const void *registers); bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 95; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64; } + static int getArch() { return REGISTERS_ARM64; } uint64_t getSP() const { return _registers.__sp; } void setSP(uint64_t value) { _registers.__sp = value; } uint64_t getIP() const { return _registers.__pc; } void setIP(uint64_t value) { _registers.__pc = value; } uint64_t getFP() const { return _registers.__fp; } void setFP(uint64_t value) { _registers.__fp = value; } private: struct GPRs { uint64_t __x[29]; // x0-x28 uint64_t __fp; // Frame pointer x29 uint64_t __lr; // Link register x30 uint64_t __sp; // Stack pointer x31 uint64_t __pc; // Program counter - uint64_t padding; // 16-byte align + uint64_t __ra_sign_state; // RA sign state register }; GPRs _registers; double _vectorHalfRegisters[32]; // Currently only the lower double in 128-bit vectore registers // is perserved during unwinding. We could define new register // numbers (> 96) which mean whole vector registers, then this // struct would need to change to contain whole vector registers. }; inline Registers_arm64::Registers_arm64(const void *registers) { static_assert((check_fit::does_fit), "arm64 registers do not fit into unw_context_t"); memcpy(&_registers, registers, sizeof(_registers)); static_assert(sizeof(GPRs) == 0x110, "expected VFP registers to be at offset 272"); memcpy(_vectorHalfRegisters, static_cast(registers) + sizeof(GPRs), sizeof(_vectorHalfRegisters)); } inline Registers_arm64::Registers_arm64() { memset(&_registers, 0, sizeof(_registers)); memset(&_vectorHalfRegisters, 0, sizeof(_vectorHalfRegisters)); } inline bool Registers_arm64::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum > 95) return false; + if (regNum == UNW_ARM64_RA_SIGN_STATE) + return true; if ((regNum > 31) && (regNum < 64)) return false; return true; } inline uint64_t Registers_arm64::getRegister(int regNum) const { if (regNum == UNW_REG_IP) return _registers.__pc; if (regNum == UNW_REG_SP) return _registers.__sp; + if (regNum == UNW_ARM64_RA_SIGN_STATE) + return _registers.__ra_sign_state; if ((regNum >= 0) && (regNum < 32)) return _registers.__x[regNum]; _LIBUNWIND_ABORT("unsupported arm64 register"); } inline void Registers_arm64::setRegister(int regNum, uint64_t value) { if (regNum == UNW_REG_IP) _registers.__pc = value; else if (regNum == UNW_REG_SP) _registers.__sp = value; + else if (regNum == UNW_ARM64_RA_SIGN_STATE) + _registers.__ra_sign_state = value; else if ((regNum >= 0) && (regNum < 32)) _registers.__x[regNum] = value; else _LIBUNWIND_ABORT("unsupported arm64 register"); } inline const char *Registers_arm64::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: return "pc"; case UNW_REG_SP: return "sp"; case UNW_ARM64_X0: return "x0"; case UNW_ARM64_X1: return "x1"; case UNW_ARM64_X2: return "x2"; case UNW_ARM64_X3: return "x3"; case UNW_ARM64_X4: return "x4"; case UNW_ARM64_X5: return "x5"; case UNW_ARM64_X6: return "x6"; case UNW_ARM64_X7: return "x7"; case UNW_ARM64_X8: return "x8"; case UNW_ARM64_X9: return "x9"; case UNW_ARM64_X10: return "x10"; case UNW_ARM64_X11: return "x11"; case UNW_ARM64_X12: return "x12"; case UNW_ARM64_X13: return "x13"; case UNW_ARM64_X14: return "x14"; case UNW_ARM64_X15: return "x15"; case UNW_ARM64_X16: return "x16"; case UNW_ARM64_X17: return "x17"; case UNW_ARM64_X18: return "x18"; case UNW_ARM64_X19: return "x19"; case UNW_ARM64_X20: return "x20"; case UNW_ARM64_X21: return "x21"; case UNW_ARM64_X22: return "x22"; case UNW_ARM64_X23: return "x23"; case UNW_ARM64_X24: return "x24"; case UNW_ARM64_X25: return "x25"; case UNW_ARM64_X26: return "x26"; case UNW_ARM64_X27: return "x27"; case UNW_ARM64_X28: return "x28"; case UNW_ARM64_X29: return "fp"; case UNW_ARM64_X30: return "lr"; case UNW_ARM64_X31: return "sp"; case UNW_ARM64_D0: return "d0"; case UNW_ARM64_D1: return "d1"; case UNW_ARM64_D2: return "d2"; case UNW_ARM64_D3: return "d3"; case UNW_ARM64_D4: return "d4"; case UNW_ARM64_D5: return "d5"; case UNW_ARM64_D6: return "d6"; case UNW_ARM64_D7: return "d7"; case UNW_ARM64_D8: return "d8"; case UNW_ARM64_D9: return "d9"; case UNW_ARM64_D10: return "d10"; case UNW_ARM64_D11: return "d11"; case UNW_ARM64_D12: return "d12"; case UNW_ARM64_D13: return "d13"; case UNW_ARM64_D14: return "d14"; case UNW_ARM64_D15: return "d15"; case UNW_ARM64_D16: return "d16"; case UNW_ARM64_D17: return "d17"; case UNW_ARM64_D18: return "d18"; case UNW_ARM64_D19: return "d19"; case UNW_ARM64_D20: return "d20"; case UNW_ARM64_D21: return "d21"; case UNW_ARM64_D22: return "d22"; case UNW_ARM64_D23: return "d23"; case UNW_ARM64_D24: return "d24"; case UNW_ARM64_D25: return "d25"; case UNW_ARM64_D26: return "d26"; case UNW_ARM64_D27: return "d27"; case UNW_ARM64_D28: return "d28"; case UNW_ARM64_D29: return "d29"; case UNW_ARM64_D30: return "d30"; case UNW_ARM64_D31: return "d31"; default: return "unknown register"; } } inline bool Registers_arm64::validFloatRegister(int regNum) const { if (regNum < UNW_ARM64_D0) return false; if (regNum > UNW_ARM64_D31) return false; return true; } inline double Registers_arm64::getFloatRegister(int regNum) const { assert(validFloatRegister(regNum)); return _vectorHalfRegisters[regNum - UNW_ARM64_D0]; } inline void Registers_arm64::setFloatRegister(int regNum, double value) { assert(validFloatRegister(regNum)); _vectorHalfRegisters[regNum - UNW_ARM64_D0] = value; } inline bool Registers_arm64::validVectorRegister(int) const { return false; } inline v128 Registers_arm64::getVectorRegister(int) const { _LIBUNWIND_ABORT("no arm64 vector register support yet"); } inline void Registers_arm64::setVectorRegister(int, v128) { _LIBUNWIND_ABORT("no arm64 vector register support yet"); } #endif // _LIBUNWIND_TARGET_AARCH64 #if defined(_LIBUNWIND_TARGET_ARM) /// Registers_arm holds the register state of a thread in a 32-bit arm /// process. /// /// NOTE: Assumes VFPv3. On ARM processors without a floating point unit, /// this uses more memory than required. class _LIBUNWIND_HIDDEN Registers_arm { public: Registers_arm(); Registers_arm(const void *registers); bool validRegister(int num) const; - uint32_t getRegister(int num); + uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); bool validFloatRegister(int num) const; unw_fpreg_t getFloatRegister(int num); void setFloatRegister(int num, unw_fpreg_t value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto() { restoreSavedFloatRegisters(); restoreCoreAndJumpTo(); } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM; } + static int getArch() { return REGISTERS_ARM; } uint32_t getSP() const { return _registers.__sp; } void setSP(uint32_t value) { _registers.__sp = value; } uint32_t getIP() const { return _registers.__pc; } void setIP(uint32_t value) { _registers.__pc = value; } void saveVFPAsX() { assert(_use_X_for_vfp_save || !_saved_vfp_d0_d15); _use_X_for_vfp_save = true; } void restoreSavedFloatRegisters() { if (_saved_vfp_d0_d15) { if (_use_X_for_vfp_save) restoreVFPWithFLDMX(_vfp_d0_d15_pad); else restoreVFPWithFLDMD(_vfp_d0_d15_pad); } if (_saved_vfp_d16_d31) restoreVFPv3(_vfp_d16_d31); +#if defined(__ARM_WMMX) if (_saved_iwmmx) restoreiWMMX(_iwmmx); if (_saved_iwmmx_control) restoreiWMMXControl(_iwmmx_control); +#endif } private: struct GPRs { uint32_t __r[13]; // r0-r12 uint32_t __sp; // Stack pointer r13 uint32_t __lr; // Link register r14 uint32_t __pc; // Program counter r15 }; static void saveVFPWithFSTMD(unw_fpreg_t*); static void saveVFPWithFSTMX(unw_fpreg_t*); static void saveVFPv3(unw_fpreg_t*); - static void saveiWMMX(unw_fpreg_t*); - static void saveiWMMXControl(uint32_t*); static void restoreVFPWithFLDMD(unw_fpreg_t*); static void restoreVFPWithFLDMX(unw_fpreg_t*); static void restoreVFPv3(unw_fpreg_t*); +#if defined(__ARM_WMMX) + static void saveiWMMX(unw_fpreg_t*); + static void saveiWMMXControl(uint32_t*); static void restoreiWMMX(unw_fpreg_t*); static void restoreiWMMXControl(uint32_t*); +#endif void restoreCoreAndJumpTo(); // ARM registers GPRs _registers; // We save floating point registers lazily because we can't know ahead of // time which ones are used. See EHABI #4.7. // Whether D0-D15 are saved in the FTSMX instead of FSTMD format. // // See EHABI #7.5 that explains how matching instruction sequences for load // and store need to be used to correctly restore the exact register bits. bool _use_X_for_vfp_save; // Whether VFP D0-D15 are saved. bool _saved_vfp_d0_d15; // Whether VFPv3 D16-D31 are saved. bool _saved_vfp_d16_d31; - // Whether iWMMX data registers are saved. - bool _saved_iwmmx; - // Whether iWMMX control registers are saved. - bool _saved_iwmmx_control; // VFP registers D0-D15, + padding if saved using FSTMX unw_fpreg_t _vfp_d0_d15_pad[17]; // VFPv3 registers D16-D31, always saved using FSTMD unw_fpreg_t _vfp_d16_d31[16]; +#if defined(__ARM_WMMX) + // Whether iWMMX data registers are saved. + bool _saved_iwmmx; + // Whether iWMMX control registers are saved. + mutable bool _saved_iwmmx_control; // iWMMX registers unw_fpreg_t _iwmmx[16]; // iWMMX control registers - uint32_t _iwmmx_control[4]; + mutable uint32_t _iwmmx_control[4]; +#endif }; inline Registers_arm::Registers_arm(const void *registers) : _use_X_for_vfp_save(false), _saved_vfp_d0_d15(false), - _saved_vfp_d16_d31(false), - _saved_iwmmx(false), - _saved_iwmmx_control(false) { + _saved_vfp_d16_d31(false) { static_assert((check_fit::does_fit), "arm registers do not fit into unw_context_t"); // See unw_getcontext() note about data. memcpy(&_registers, registers, sizeof(_registers)); memset(&_vfp_d0_d15_pad, 0, sizeof(_vfp_d0_d15_pad)); memset(&_vfp_d16_d31, 0, sizeof(_vfp_d16_d31)); +#if defined(__ARM_WMMX) + _saved_iwmmx = false; + _saved_iwmmx_control = false; memset(&_iwmmx, 0, sizeof(_iwmmx)); memset(&_iwmmx_control, 0, sizeof(_iwmmx_control)); +#endif } inline Registers_arm::Registers_arm() : _use_X_for_vfp_save(false), _saved_vfp_d0_d15(false), - _saved_vfp_d16_d31(false), - _saved_iwmmx(false), - _saved_iwmmx_control(false) { + _saved_vfp_d16_d31(false) { memset(&_registers, 0, sizeof(_registers)); memset(&_vfp_d0_d15_pad, 0, sizeof(_vfp_d0_d15_pad)); memset(&_vfp_d16_d31, 0, sizeof(_vfp_d16_d31)); +#if defined(__ARM_WMMX) + _saved_iwmmx = false; + _saved_iwmmx_control = false; memset(&_iwmmx, 0, sizeof(_iwmmx)); memset(&_iwmmx_control, 0, sizeof(_iwmmx_control)); +#endif } inline bool Registers_arm::validRegister(int regNum) const { // Returns true for all non-VFP registers supported by the EHABI // virtual register set (VRS). if (regNum == UNW_REG_IP) return true; + if (regNum == UNW_REG_SP) return true; + if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R15) return true; + +#if defined(__ARM_WMMX) if (regNum >= UNW_ARM_WC0 && regNum <= UNW_ARM_WC3) return true; +#endif + return false; } -inline uint32_t Registers_arm::getRegister(int regNum) { +inline uint32_t Registers_arm::getRegister(int regNum) const { if (regNum == UNW_REG_SP || regNum == UNW_ARM_SP) return _registers.__sp; + if (regNum == UNW_ARM_LR) return _registers.__lr; + if (regNum == UNW_REG_IP || regNum == UNW_ARM_IP) return _registers.__pc; + if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R12) return _registers.__r[regNum]; + +#if defined(__ARM_WMMX) if (regNum >= UNW_ARM_WC0 && regNum <= UNW_ARM_WC3) { if (!_saved_iwmmx_control) { _saved_iwmmx_control = true; saveiWMMXControl(_iwmmx_control); } return _iwmmx_control[regNum - UNW_ARM_WC0]; } +#endif + _LIBUNWIND_ABORT("unsupported arm register"); } inline void Registers_arm::setRegister(int regNum, uint32_t value) { - if (regNum == UNW_REG_SP || regNum == UNW_ARM_SP) + if (regNum == UNW_REG_SP || regNum == UNW_ARM_SP) { _registers.__sp = value; - else if (regNum == UNW_ARM_LR) + return; + } + + if (regNum == UNW_ARM_LR) { _registers.__lr = value; - else if (regNum == UNW_REG_IP || regNum == UNW_ARM_IP) + return; + } + + if (regNum == UNW_REG_IP || regNum == UNW_ARM_IP) { _registers.__pc = value; - else if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R12) + return; + } + + if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R12) { _registers.__r[regNum] = value; - else if (regNum >= UNW_ARM_WC0 && regNum <= UNW_ARM_WC3) { + return; + } + +#if defined(__ARM_WMMX) + if (regNum >= UNW_ARM_WC0 && regNum <= UNW_ARM_WC3) { if (!_saved_iwmmx_control) { _saved_iwmmx_control = true; saveiWMMXControl(_iwmmx_control); } _iwmmx_control[regNum - UNW_ARM_WC0] = value; - } else - _LIBUNWIND_ABORT("unsupported arm register"); + return; + } +#endif + + _LIBUNWIND_ABORT("unsupported arm register"); } inline const char *Registers_arm::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: case UNW_ARM_IP: // UNW_ARM_R15 is alias return "pc"; case UNW_ARM_LR: // UNW_ARM_R14 is alias return "lr"; case UNW_REG_SP: case UNW_ARM_SP: // UNW_ARM_R13 is alias return "sp"; case UNW_ARM_R0: return "r0"; case UNW_ARM_R1: return "r1"; case UNW_ARM_R2: return "r2"; case UNW_ARM_R3: return "r3"; case UNW_ARM_R4: return "r4"; case UNW_ARM_R5: return "r5"; case UNW_ARM_R6: return "r6"; case UNW_ARM_R7: return "r7"; case UNW_ARM_R8: return "r8"; case UNW_ARM_R9: return "r9"; case UNW_ARM_R10: return "r10"; case UNW_ARM_R11: return "r11"; case UNW_ARM_R12: return "r12"; case UNW_ARM_S0: return "s0"; case UNW_ARM_S1: return "s1"; case UNW_ARM_S2: return "s2"; case UNW_ARM_S3: return "s3"; case UNW_ARM_S4: return "s4"; case UNW_ARM_S5: return "s5"; case UNW_ARM_S6: return "s6"; case UNW_ARM_S7: return "s7"; case UNW_ARM_S8: return "s8"; case UNW_ARM_S9: return "s9"; case UNW_ARM_S10: return "s10"; case UNW_ARM_S11: return "s11"; case UNW_ARM_S12: return "s12"; case UNW_ARM_S13: return "s13"; case UNW_ARM_S14: return "s14"; case UNW_ARM_S15: return "s15"; case UNW_ARM_S16: return "s16"; case UNW_ARM_S17: return "s17"; case UNW_ARM_S18: return "s18"; case UNW_ARM_S19: return "s19"; case UNW_ARM_S20: return "s20"; case UNW_ARM_S21: return "s21"; case UNW_ARM_S22: return "s22"; case UNW_ARM_S23: return "s23"; case UNW_ARM_S24: return "s24"; case UNW_ARM_S25: return "s25"; case UNW_ARM_S26: return "s26"; case UNW_ARM_S27: return "s27"; case UNW_ARM_S28: return "s28"; case UNW_ARM_S29: return "s29"; case UNW_ARM_S30: return "s30"; case UNW_ARM_S31: return "s31"; case UNW_ARM_D0: return "d0"; case UNW_ARM_D1: return "d1"; case UNW_ARM_D2: return "d2"; case UNW_ARM_D3: return "d3"; case UNW_ARM_D4: return "d4"; case UNW_ARM_D5: return "d5"; case UNW_ARM_D6: return "d6"; case UNW_ARM_D7: return "d7"; case UNW_ARM_D8: return "d8"; case UNW_ARM_D9: return "d9"; case UNW_ARM_D10: return "d10"; case UNW_ARM_D11: return "d11"; case UNW_ARM_D12: return "d12"; case UNW_ARM_D13: return "d13"; case UNW_ARM_D14: return "d14"; case UNW_ARM_D15: return "d15"; case UNW_ARM_D16: return "d16"; case UNW_ARM_D17: return "d17"; case UNW_ARM_D18: return "d18"; case UNW_ARM_D19: return "d19"; case UNW_ARM_D20: return "d20"; case UNW_ARM_D21: return "d21"; case UNW_ARM_D22: return "d22"; case UNW_ARM_D23: return "d23"; case UNW_ARM_D24: return "d24"; case UNW_ARM_D25: return "d25"; case UNW_ARM_D26: return "d26"; case UNW_ARM_D27: return "d27"; case UNW_ARM_D28: return "d28"; case UNW_ARM_D29: return "d29"; case UNW_ARM_D30: return "d30"; case UNW_ARM_D31: return "d31"; default: return "unknown register"; } } inline bool Registers_arm::validFloatRegister(int regNum) const { // NOTE: Consider the intel MMX registers floating points so the // unw_get_fpreg can be used to transmit the 64-bit data back. return ((regNum >= UNW_ARM_D0) && (regNum <= UNW_ARM_D31)) - || ((regNum >= UNW_ARM_WR0) && (regNum <= UNW_ARM_WR15)); +#if defined(__ARM_WMMX) + || ((regNum >= UNW_ARM_WR0) && (regNum <= UNW_ARM_WR15)) +#endif + ; } inline unw_fpreg_t Registers_arm::getFloatRegister(int regNum) { if (regNum >= UNW_ARM_D0 && regNum <= UNW_ARM_D15) { if (!_saved_vfp_d0_d15) { _saved_vfp_d0_d15 = true; if (_use_X_for_vfp_save) saveVFPWithFSTMX(_vfp_d0_d15_pad); else saveVFPWithFSTMD(_vfp_d0_d15_pad); } return _vfp_d0_d15_pad[regNum - UNW_ARM_D0]; - } else if (regNum >= UNW_ARM_D16 && regNum <= UNW_ARM_D31) { + } + + if (regNum >= UNW_ARM_D16 && regNum <= UNW_ARM_D31) { if (!_saved_vfp_d16_d31) { _saved_vfp_d16_d31 = true; saveVFPv3(_vfp_d16_d31); } return _vfp_d16_d31[regNum - UNW_ARM_D16]; - } else if (regNum >= UNW_ARM_WR0 && regNum <= UNW_ARM_WR15) { + } + +#if defined(__ARM_WMMX) + if (regNum >= UNW_ARM_WR0 && regNum <= UNW_ARM_WR15) { if (!_saved_iwmmx) { _saved_iwmmx = true; saveiWMMX(_iwmmx); } return _iwmmx[regNum - UNW_ARM_WR0]; - } else { - _LIBUNWIND_ABORT("Unknown ARM float register"); } +#endif + + _LIBUNWIND_ABORT("Unknown ARM float register"); } inline void Registers_arm::setFloatRegister(int regNum, unw_fpreg_t value) { if (regNum >= UNW_ARM_D0 && regNum <= UNW_ARM_D15) { if (!_saved_vfp_d0_d15) { _saved_vfp_d0_d15 = true; if (_use_X_for_vfp_save) saveVFPWithFSTMX(_vfp_d0_d15_pad); else saveVFPWithFSTMD(_vfp_d0_d15_pad); } _vfp_d0_d15_pad[regNum - UNW_ARM_D0] = value; - } else if (regNum >= UNW_ARM_D16 && regNum <= UNW_ARM_D31) { + return; + } + + if (regNum >= UNW_ARM_D16 && regNum <= UNW_ARM_D31) { if (!_saved_vfp_d16_d31) { _saved_vfp_d16_d31 = true; saveVFPv3(_vfp_d16_d31); } _vfp_d16_d31[regNum - UNW_ARM_D16] = value; - } else if (regNum >= UNW_ARM_WR0 && regNum <= UNW_ARM_WR15) { + return; + } + +#if defined(__ARM_WMMX) + if (regNum >= UNW_ARM_WR0 && regNum <= UNW_ARM_WR15) { if (!_saved_iwmmx) { _saved_iwmmx = true; saveiWMMX(_iwmmx); } _iwmmx[regNum - UNW_ARM_WR0] = value; - } else { - _LIBUNWIND_ABORT("Unknown ARM float register"); + return; } +#endif + + _LIBUNWIND_ABORT("Unknown ARM float register"); } inline bool Registers_arm::validVectorRegister(int) const { return false; } inline v128 Registers_arm::getVectorRegister(int) const { _LIBUNWIND_ABORT("ARM vector support not implemented"); } inline void Registers_arm::setVectorRegister(int, v128) { _LIBUNWIND_ABORT("ARM vector support not implemented"); } #endif // _LIBUNWIND_TARGET_ARM #if defined(_LIBUNWIND_TARGET_OR1K) /// Registers_or1k holds the register state of a thread in an OpenRISC1000 /// process. class _LIBUNWIND_HIDDEN Registers_or1k { public: Registers_or1k(); Registers_or1k(const void *registers); bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 31; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_OR1K; } + static int getArch() { return REGISTERS_OR1K; } uint64_t getSP() const { return _registers.__r[1]; } void setSP(uint32_t value) { _registers.__r[1] = value; } - uint64_t getIP() const { return _registers.__r[9]; } - void setIP(uint32_t value) { _registers.__r[9] = value; } + uint64_t getIP() const { return _registers.__pc; } + void setIP(uint32_t value) { _registers.__pc = value; } private: struct or1k_thread_state_t { - unsigned int __r[32]; + unsigned int __r[32]; // r0-r31 + unsigned int __pc; // Program counter + unsigned int __epcr; // Program counter at exception }; or1k_thread_state_t _registers; }; inline Registers_or1k::Registers_or1k(const void *registers) { static_assert((check_fit::does_fit), "or1k registers do not fit into unw_context_t"); memcpy(&_registers, static_cast(registers), sizeof(_registers)); } inline Registers_or1k::Registers_or1k() { memset(&_registers, 0, sizeof(_registers)); } inline bool Registers_or1k::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum <= UNW_OR1K_R31) return true; + if (regNum == UNW_OR1K_EPCR) + return true; return false; } inline uint32_t Registers_or1k::getRegister(int regNum) const { if (regNum >= UNW_OR1K_R0 && regNum <= UNW_OR1K_R31) return _registers.__r[regNum - UNW_OR1K_R0]; switch (regNum) { case UNW_REG_IP: - return _registers.__r[9]; + return _registers.__pc; case UNW_REG_SP: return _registers.__r[1]; + case UNW_OR1K_EPCR: + return _registers.__epcr; } _LIBUNWIND_ABORT("unsupported or1k register"); } inline void Registers_or1k::setRegister(int regNum, uint32_t value) { if (regNum >= UNW_OR1K_R0 && regNum <= UNW_OR1K_R31) { _registers.__r[regNum - UNW_OR1K_R0] = value; return; } switch (regNum) { case UNW_REG_IP: - _registers.__r[9] = value; + _registers.__pc = value; return; case UNW_REG_SP: _registers.__r[1] = value; return; + case UNW_OR1K_EPCR: + _registers.__epcr = value; + return; } _LIBUNWIND_ABORT("unsupported or1k register"); } inline bool Registers_or1k::validFloatRegister(int /* regNum */) const { return false; } inline double Registers_or1k::getFloatRegister(int /* regNum */) const { _LIBUNWIND_ABORT("or1k float support not implemented"); } inline void Registers_or1k::setFloatRegister(int /* regNum */, double /* value */) { _LIBUNWIND_ABORT("or1k float support not implemented"); } inline bool Registers_or1k::validVectorRegister(int /* regNum */) const { return false; } inline v128 Registers_or1k::getVectorRegister(int /* regNum */) const { _LIBUNWIND_ABORT("or1k vector support not implemented"); } inline void Registers_or1k::setVectorRegister(int /* regNum */, v128 /* value */) { _LIBUNWIND_ABORT("or1k vector support not implemented"); } inline const char *Registers_or1k::getRegisterName(int regNum) { switch (regNum) { case UNW_OR1K_R0: return "r0"; case UNW_OR1K_R1: return "r1"; case UNW_OR1K_R2: return "r2"; case UNW_OR1K_R3: return "r3"; case UNW_OR1K_R4: return "r4"; case UNW_OR1K_R5: return "r5"; case UNW_OR1K_R6: return "r6"; case UNW_OR1K_R7: return "r7"; case UNW_OR1K_R8: return "r8"; case UNW_OR1K_R9: return "r9"; case UNW_OR1K_R10: return "r10"; case UNW_OR1K_R11: return "r11"; case UNW_OR1K_R12: return "r12"; case UNW_OR1K_R13: return "r13"; case UNW_OR1K_R14: return "r14"; case UNW_OR1K_R15: return "r15"; case UNW_OR1K_R16: return "r16"; case UNW_OR1K_R17: return "r17"; case UNW_OR1K_R18: return "r18"; case UNW_OR1K_R19: return "r19"; case UNW_OR1K_R20: return "r20"; case UNW_OR1K_R21: return "r21"; case UNW_OR1K_R22: return "r22"; case UNW_OR1K_R23: return "r23"; case UNW_OR1K_R24: return "r24"; case UNW_OR1K_R25: return "r25"; case UNW_OR1K_R26: return "r26"; case UNW_OR1K_R27: return "r27"; case UNW_OR1K_R28: return "r28"; case UNW_OR1K_R29: return "r29"; case UNW_OR1K_R30: return "r30"; case UNW_OR1K_R31: return "r31"; + case UNW_OR1K_EPCR: + return "EPCR"; default: return "unknown register"; } } #endif // _LIBUNWIND_TARGET_OR1K - #if defined(_LIBUNWIND_TARGET_RISCV) /// Registers_riscv holds the register state of a thread in a 64-bit RISC-V /// process. class _LIBUNWIND_HIDDEN Registers_riscv { public: Registers_riscv(); Registers_riscv(const void *registers); bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 95; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_RISCV; } + static int getArch() { return REGISTERS_RISCV; } uint64_t getSP() const { return _registers.__x[2]; } void setSP(uint64_t value) { _registers.__x[2] = value; } uint64_t getIP() const { return _registers.__x[1]; } void setIP(uint64_t value) { _registers.__x[1] = value; } private: struct GPRs { uint64_t __x[32]; // x0-x31 }; GPRs _registers; double _vectorHalfRegisters[32]; // Currently only the lower double in 128-bit vectore registers // is perserved during unwinding. We could define new register // numbers (> 96) which mean whole vector registers, then this // struct would need to change to contain whole vector registers. }; inline Registers_riscv::Registers_riscv(const void *registers) { static_assert((check_fit::does_fit), "riscv registers do not fit into unw_context_t"); memcpy(&_registers, registers, sizeof(_registers)); static_assert(sizeof(GPRs) == 0x100, "expected VFP registers to be at offset 256"); memcpy(_vectorHalfRegisters, static_cast(registers) + sizeof(GPRs), sizeof(_vectorHalfRegisters)); } inline Registers_riscv::Registers_riscv() { memset(&_registers, 0, sizeof(_registers)); memset(&_vectorHalfRegisters, 0, sizeof(_vectorHalfRegisters)); } inline bool Registers_riscv::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum > 95) return false; if ((regNum > 31) && (regNum < 64)) return false; return true; } inline uint64_t Registers_riscv::getRegister(int regNum) const { if (regNum == UNW_REG_IP) return _registers.__x[1]; if (regNum == UNW_REG_SP) return _registers.__x[2]; if ((regNum >= 0) && (regNum < 32)) return _registers.__x[regNum]; _LIBUNWIND_ABORT("unsupported riscv register"); } inline void Registers_riscv::setRegister(int regNum, uint64_t value) { if (regNum == UNW_REG_IP) _registers.__x[1] = value; else if (regNum == UNW_REG_SP) _registers.__x[2] = value; else if ((regNum >= 0) && (regNum < 32)) _registers.__x[regNum] = value; else _LIBUNWIND_ABORT("unsupported riscv register"); } inline const char *Registers_riscv::getRegisterName(int regNum) { switch (regNum) { case UNW_REG_IP: return "ra"; case UNW_REG_SP: return "sp"; case UNW_RISCV_X0: return "x0"; case UNW_RISCV_X1: return "ra"; case UNW_RISCV_X2: return "sp"; case UNW_RISCV_X3: return "x3"; case UNW_RISCV_X4: return "x4"; case UNW_RISCV_X5: return "x5"; case UNW_RISCV_X6: return "x6"; case UNW_RISCV_X7: return "x7"; case UNW_RISCV_X8: return "x8"; case UNW_RISCV_X9: return "x9"; case UNW_RISCV_X10: return "x10"; case UNW_RISCV_X11: return "x11"; case UNW_RISCV_X12: return "x12"; case UNW_RISCV_X13: return "x13"; case UNW_RISCV_X14: return "x14"; case UNW_RISCV_X15: return "x15"; case UNW_RISCV_X16: return "x16"; case UNW_RISCV_X17: return "x17"; case UNW_RISCV_X18: return "x18"; case UNW_RISCV_X19: return "x19"; case UNW_RISCV_X20: return "x20"; case UNW_RISCV_X21: return "x21"; case UNW_RISCV_X22: return "x22"; case UNW_RISCV_X23: return "x23"; case UNW_RISCV_X24: return "x24"; case UNW_RISCV_X25: return "x25"; case UNW_RISCV_X26: return "x26"; case UNW_RISCV_X27: return "x27"; case UNW_RISCV_X28: return "x28"; case UNW_RISCV_X29: return "x29"; case UNW_RISCV_X30: return "x30"; case UNW_RISCV_X31: return "x31"; case UNW_RISCV_D0: return "d0"; case UNW_RISCV_D1: return "d1"; case UNW_RISCV_D2: return "d2"; case UNW_RISCV_D3: return "d3"; case UNW_RISCV_D4: return "d4"; case UNW_RISCV_D5: return "d5"; case UNW_RISCV_D6: return "d6"; case UNW_RISCV_D7: return "d7"; case UNW_RISCV_D8: return "d8"; case UNW_RISCV_D9: return "d9"; case UNW_RISCV_D10: return "d10"; case UNW_RISCV_D11: return "d11"; case UNW_RISCV_D12: return "d12"; case UNW_RISCV_D13: return "d13"; case UNW_RISCV_D14: return "d14"; case UNW_RISCV_D15: return "d15"; case UNW_RISCV_D16: return "d16"; case UNW_RISCV_D17: return "d17"; case UNW_RISCV_D18: return "d18"; case UNW_RISCV_D19: return "d19"; case UNW_RISCV_D20: return "d20"; case UNW_RISCV_D21: return "d21"; case UNW_RISCV_D22: return "d22"; case UNW_RISCV_D23: return "d23"; case UNW_RISCV_D24: return "d24"; case UNW_RISCV_D25: return "d25"; case UNW_RISCV_D26: return "d26"; case UNW_RISCV_D27: return "d27"; case UNW_RISCV_D28: return "d28"; case UNW_RISCV_D29: return "d29"; case UNW_RISCV_D30: return "d30"; case UNW_RISCV_D31: return "d31"; default: return "unknown register"; } } inline bool Registers_riscv::validFloatRegister(int regNum) const { if (regNum < UNW_RISCV_D0) return false; if (regNum > UNW_RISCV_D31) return false; return true; } inline double Registers_riscv::getFloatRegister(int regNum) const { assert(validFloatRegister(regNum)); return _vectorHalfRegisters[regNum - UNW_RISCV_D0]; } inline void Registers_riscv::setFloatRegister(int regNum, double value) { assert(validFloatRegister(regNum)); _vectorHalfRegisters[regNum - UNW_RISCV_D0] = value; } inline bool Registers_riscv::validVectorRegister(int) const { return false; } inline v128 Registers_riscv::getVectorRegister(int) const { _LIBUNWIND_ABORT("no riscv vector register support yet"); } inline void Registers_riscv::setVectorRegister(int, v128) { _LIBUNWIND_ABORT("no riscv vector register support yet"); } #endif // _LIBUNWIND_TARGET_RISCV #if defined(_LIBUNWIND_TARGET_MIPS_O32) /// Registers_mips_o32 holds the register state of a thread in a 32-bit MIPS /// process. class _LIBUNWIND_HIDDEN Registers_mips_o32 { public: Registers_mips_o32(); Registers_mips_o32(const void *registers); bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 65; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS; } + static int getArch() { return REGISTERS_MIPS_O32; } uint32_t getSP() const { return _registers.__r[29]; } void setSP(uint32_t value) { _registers.__r[29] = value; } uint32_t getIP() const { return _registers.__pc; } void setIP(uint32_t value) { _registers.__pc = value; } private: struct mips_o32_thread_state_t { uint32_t __r[32]; uint32_t __pc; uint32_t __hi; uint32_t __lo; }; mips_o32_thread_state_t _registers; #ifdef __mips_hard_float /// O32 with 32-bit floating point registers only uses half of this /// space. However, using the same layout for 32-bit vs 64-bit /// floating point registers results in a single context size for /// O32 with hard float. uint32_t _padding; double _floats[32]; #endif }; inline Registers_mips_o32::Registers_mips_o32(const void *registers) { static_assert((check_fit::does_fit), "mips_o32 registers do not fit into unw_context_t"); memcpy(&_registers, static_cast(registers), sizeof(_registers)); } inline Registers_mips_o32::Registers_mips_o32() { memset(&_registers, 0, sizeof(_registers)); } inline bool Registers_mips_o32::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum <= UNW_MIPS_R31) return true; +#if __mips_isa_rev != 6 if (regNum == UNW_MIPS_HI) return true; if (regNum == UNW_MIPS_LO) return true; +#endif #if defined(__mips_hard_float) && __mips_fpr == 32 if (regNum >= UNW_MIPS_F0 && regNum <= UNW_MIPS_F31) return true; #endif // FIXME: DSP accumulator registers, MSA registers return false; } inline uint32_t Registers_mips_o32::getRegister(int regNum) const { if (regNum >= UNW_MIPS_R0 && regNum <= UNW_MIPS_R31) return _registers.__r[regNum - UNW_MIPS_R0]; #if defined(__mips_hard_float) && __mips_fpr == 32 if (regNum >= UNW_MIPS_F0 && regNum <= UNW_MIPS_F31) { uint32_t *p; if (regNum % 2 == 0) p = (uint32_t *)&_floats[regNum - UNW_MIPS_F0]; else p = (uint32_t *)&_floats[(regNum - 1) - UNW_MIPS_F0] + 1; return *p; } #endif switch (regNum) { case UNW_REG_IP: return _registers.__pc; case UNW_REG_SP: return _registers.__r[29]; case UNW_MIPS_HI: return _registers.__hi; case UNW_MIPS_LO: return _registers.__lo; } _LIBUNWIND_ABORT("unsupported mips_o32 register"); } inline void Registers_mips_o32::setRegister(int regNum, uint32_t value) { if (regNum >= UNW_MIPS_R0 && regNum <= UNW_MIPS_R31) { _registers.__r[regNum - UNW_MIPS_R0] = value; return; } #if defined(__mips_hard_float) && __mips_fpr == 32 if (regNum >= UNW_MIPS_F0 && regNum <= UNW_MIPS_F31) { uint32_t *p; if (regNum % 2 == 0) p = (uint32_t *)&_floats[regNum - UNW_MIPS_F0]; else p = (uint32_t *)&_floats[(regNum - 1) - UNW_MIPS_F0] + 1; *p = value; return; } #endif switch (regNum) { case UNW_REG_IP: _registers.__pc = value; return; case UNW_REG_SP: _registers.__r[29] = value; return; case UNW_MIPS_HI: _registers.__hi = value; return; case UNW_MIPS_LO: _registers.__lo = value; return; } _LIBUNWIND_ABORT("unsupported mips_o32 register"); } inline bool Registers_mips_o32::validFloatRegister(int regNum) const { #if defined(__mips_hard_float) && __mips_fpr == 64 if (regNum >= UNW_MIPS_F0 && regNum <= UNW_MIPS_F31) return true; #endif return false; } inline double Registers_mips_o32::getFloatRegister(int regNum) const { #if defined(__mips_hard_float) && __mips_fpr == 64 assert(validFloatRegister(regNum)); return _floats[regNum - UNW_MIPS_F0]; #else _LIBUNWIND_ABORT("mips_o32 float support not implemented"); #endif } inline void Registers_mips_o32::setFloatRegister(int regNum, double value) { #if defined(__mips_hard_float) && __mips_fpr == 64 assert(validFloatRegister(regNum)); _floats[regNum - UNW_MIPS_F0] = value; #else _LIBUNWIND_ABORT("mips_o32 float support not implemented"); #endif } inline bool Registers_mips_o32::validVectorRegister(int /* regNum */) const { return false; } inline v128 Registers_mips_o32::getVectorRegister(int /* regNum */) const { _LIBUNWIND_ABORT("mips_o32 vector support not implemented"); } inline void Registers_mips_o32::setVectorRegister(int /* regNum */, v128 /* value */) { _LIBUNWIND_ABORT("mips_o32 vector support not implemented"); } inline const char *Registers_mips_o32::getRegisterName(int regNum) { switch (regNum) { case UNW_MIPS_R0: return "$0"; case UNW_MIPS_R1: return "$1"; case UNW_MIPS_R2: return "$2"; case UNW_MIPS_R3: return "$3"; case UNW_MIPS_R4: return "$4"; case UNW_MIPS_R5: return "$5"; case UNW_MIPS_R6: return "$6"; case UNW_MIPS_R7: return "$7"; case UNW_MIPS_R8: return "$8"; case UNW_MIPS_R9: return "$9"; case UNW_MIPS_R10: return "$10"; case UNW_MIPS_R11: return "$11"; case UNW_MIPS_R12: return "$12"; case UNW_MIPS_R13: return "$13"; case UNW_MIPS_R14: return "$14"; case UNW_MIPS_R15: return "$15"; case UNW_MIPS_R16: return "$16"; case UNW_MIPS_R17: return "$17"; case UNW_MIPS_R18: return "$18"; case UNW_MIPS_R19: return "$19"; case UNW_MIPS_R20: return "$20"; case UNW_MIPS_R21: return "$21"; case UNW_MIPS_R22: return "$22"; case UNW_MIPS_R23: return "$23"; case UNW_MIPS_R24: return "$24"; case UNW_MIPS_R25: return "$25"; case UNW_MIPS_R26: return "$26"; case UNW_MIPS_R27: return "$27"; case UNW_MIPS_R28: return "$28"; case UNW_MIPS_R29: return "$29"; case UNW_MIPS_R30: return "$30"; case UNW_MIPS_R31: return "$31"; case UNW_MIPS_F0: return "$f0"; case UNW_MIPS_F1: return "$f1"; case UNW_MIPS_F2: return "$f2"; case UNW_MIPS_F3: return "$f3"; case UNW_MIPS_F4: return "$f4"; case UNW_MIPS_F5: return "$f5"; case UNW_MIPS_F6: return "$f6"; case UNW_MIPS_F7: return "$f7"; case UNW_MIPS_F8: return "$f8"; case UNW_MIPS_F9: return "$f9"; case UNW_MIPS_F10: return "$f10"; case UNW_MIPS_F11: return "$f11"; case UNW_MIPS_F12: return "$f12"; case UNW_MIPS_F13: return "$f13"; case UNW_MIPS_F14: return "$f14"; case UNW_MIPS_F15: return "$f15"; case UNW_MIPS_F16: return "$f16"; case UNW_MIPS_F17: return "$f17"; case UNW_MIPS_F18: return "$f18"; case UNW_MIPS_F19: return "$f19"; case UNW_MIPS_F20: return "$f20"; case UNW_MIPS_F21: return "$f21"; case UNW_MIPS_F22: return "$f22"; case UNW_MIPS_F23: return "$f23"; case UNW_MIPS_F24: return "$f24"; case UNW_MIPS_F25: return "$f25"; case UNW_MIPS_F26: return "$f26"; case UNW_MIPS_F27: return "$f27"; case UNW_MIPS_F28: return "$f28"; case UNW_MIPS_F29: return "$f29"; case UNW_MIPS_F30: return "$f30"; case UNW_MIPS_F31: return "$f31"; case UNW_MIPS_HI: return "$hi"; case UNW_MIPS_LO: return "$lo"; default: return "unknown register"; } } #endif // _LIBUNWIND_TARGET_MIPS_O32 #if defined(_LIBUNWIND_TARGET_MIPS_NEWABI) /// Registers_mips_newabi holds the register state of a thread in a /// MIPS process using NEWABI (the N32 or N64 ABIs). class _LIBUNWIND_HIDDEN Registers_mips_newabi { public: Registers_mips_newabi(); Registers_mips_newabi(const void *registers); bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); bool validFloatRegister(int num) const; double getFloatRegister(int num) const; void setFloatRegister(int num, double value); bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); - const char *getRegisterName(int num); + static const char *getRegisterName(int num); void jumpto(); - static int lastDwarfRegNum() { return 65; } + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS; } + static int getArch() { return REGISTERS_MIPS_NEWABI; } uint64_t getSP() const { return _registers.__r[29]; } void setSP(uint64_t value) { _registers.__r[29] = value; } uint64_t getIP() const { return _registers.__pc; } void setIP(uint64_t value) { _registers.__pc = value; } private: struct mips_newabi_thread_state_t { uint64_t __r[32]; uint64_t __pc; uint64_t __hi; uint64_t __lo; }; mips_newabi_thread_state_t _registers; #ifdef __mips_hard_float double _floats[32]; #endif }; inline Registers_mips_newabi::Registers_mips_newabi(const void *registers) { static_assert((check_fit::does_fit), "mips_newabi registers do not fit into unw_context_t"); memcpy(&_registers, static_cast(registers), sizeof(_registers)); } inline Registers_mips_newabi::Registers_mips_newabi() { memset(&_registers, 0, sizeof(_registers)); } inline bool Registers_mips_newabi::validRegister(int regNum) const { if (regNum == UNW_REG_IP) return true; if (regNum == UNW_REG_SP) return true; if (regNum < 0) return false; if (regNum <= UNW_MIPS_R31) return true; +#if __mips_isa_rev != 6 if (regNum == UNW_MIPS_HI) return true; if (regNum == UNW_MIPS_LO) return true; +#endif // FIXME: Hard float, DSP accumulator registers, MSA registers return false; } inline uint64_t Registers_mips_newabi::getRegister(int regNum) const { if (regNum >= UNW_MIPS_R0 && regNum <= UNW_MIPS_R31) return _registers.__r[regNum - UNW_MIPS_R0]; switch (regNum) { case UNW_REG_IP: return _registers.__pc; case UNW_REG_SP: return _registers.__r[29]; case UNW_MIPS_HI: return _registers.__hi; case UNW_MIPS_LO: return _registers.__lo; } _LIBUNWIND_ABORT("unsupported mips_newabi register"); } inline void Registers_mips_newabi::setRegister(int regNum, uint64_t value) { if (regNum >= UNW_MIPS_R0 && regNum <= UNW_MIPS_R31) { _registers.__r[regNum - UNW_MIPS_R0] = value; return; } switch (regNum) { case UNW_REG_IP: _registers.__pc = value; return; case UNW_REG_SP: _registers.__r[29] = value; return; case UNW_MIPS_HI: _registers.__hi = value; return; case UNW_MIPS_LO: _registers.__lo = value; return; } _LIBUNWIND_ABORT("unsupported mips_newabi register"); } inline bool Registers_mips_newabi::validFloatRegister(int regNum) const { #ifdef __mips_hard_float if (regNum >= UNW_MIPS_F0 && regNum <= UNW_MIPS_F31) return true; #endif return false; } inline double Registers_mips_newabi::getFloatRegister(int regNum) const { #ifdef __mips_hard_float assert(validFloatRegister(regNum)); return _floats[regNum - UNW_MIPS_F0]; #else _LIBUNWIND_ABORT("mips_newabi float support not implemented"); #endif } inline void Registers_mips_newabi::setFloatRegister(int regNum, double value) { #ifdef __mips_hard_float assert(validFloatRegister(regNum)); _floats[regNum - UNW_MIPS_F0] = value; #else _LIBUNWIND_ABORT("mips_newabi float support not implemented"); #endif } inline bool Registers_mips_newabi::validVectorRegister(int /* regNum */) const { return false; } inline v128 Registers_mips_newabi::getVectorRegister(int /* regNum */) const { _LIBUNWIND_ABORT("mips_newabi vector support not implemented"); } inline void Registers_mips_newabi::setVectorRegister(int /* regNum */, v128 /* value */) { _LIBUNWIND_ABORT("mips_newabi vector support not implemented"); } inline const char *Registers_mips_newabi::getRegisterName(int regNum) { switch (regNum) { case UNW_MIPS_R0: return "$0"; case UNW_MIPS_R1: return "$1"; case UNW_MIPS_R2: return "$2"; case UNW_MIPS_R3: return "$3"; case UNW_MIPS_R4: return "$4"; case UNW_MIPS_R5: return "$5"; case UNW_MIPS_R6: return "$6"; case UNW_MIPS_R7: return "$7"; case UNW_MIPS_R8: return "$8"; case UNW_MIPS_R9: return "$9"; case UNW_MIPS_R10: return "$10"; case UNW_MIPS_R11: return "$11"; case UNW_MIPS_R12: return "$12"; case UNW_MIPS_R13: return "$13"; case UNW_MIPS_R14: return "$14"; case UNW_MIPS_R15: return "$15"; case UNW_MIPS_R16: return "$16"; case UNW_MIPS_R17: return "$17"; case UNW_MIPS_R18: return "$18"; case UNW_MIPS_R19: return "$19"; case UNW_MIPS_R20: return "$20"; case UNW_MIPS_R21: return "$21"; case UNW_MIPS_R22: return "$22"; case UNW_MIPS_R23: return "$23"; case UNW_MIPS_R24: return "$24"; case UNW_MIPS_R25: return "$25"; case UNW_MIPS_R26: return "$26"; case UNW_MIPS_R27: return "$27"; case UNW_MIPS_R28: return "$28"; case UNW_MIPS_R29: return "$29"; case UNW_MIPS_R30: return "$30"; case UNW_MIPS_R31: return "$31"; case UNW_MIPS_F0: return "$f0"; case UNW_MIPS_F1: return "$f1"; case UNW_MIPS_F2: return "$f2"; case UNW_MIPS_F3: return "$f3"; case UNW_MIPS_F4: return "$f4"; case UNW_MIPS_F5: return "$f5"; case UNW_MIPS_F6: return "$f6"; case UNW_MIPS_F7: return "$f7"; case UNW_MIPS_F8: return "$f8"; case UNW_MIPS_F9: return "$f9"; case UNW_MIPS_F10: return "$f10"; case UNW_MIPS_F11: return "$f11"; case UNW_MIPS_F12: return "$f12"; case UNW_MIPS_F13: return "$f13"; case UNW_MIPS_F14: return "$f14"; case UNW_MIPS_F15: return "$f15"; case UNW_MIPS_F16: return "$f16"; case UNW_MIPS_F17: return "$f17"; case UNW_MIPS_F18: return "$f18"; case UNW_MIPS_F19: return "$f19"; case UNW_MIPS_F20: return "$f20"; case UNW_MIPS_F21: return "$f21"; case UNW_MIPS_F22: return "$f22"; case UNW_MIPS_F23: return "$f23"; case UNW_MIPS_F24: return "$f24"; case UNW_MIPS_F25: return "$f25"; case UNW_MIPS_F26: return "$f26"; case UNW_MIPS_F27: return "$f27"; case UNW_MIPS_F28: return "$f28"; case UNW_MIPS_F29: return "$f29"; case UNW_MIPS_F30: return "$f30"; case UNW_MIPS_F31: return "$f31"; case UNW_MIPS_HI: return "$hi"; case UNW_MIPS_LO: return "$lo"; default: return "unknown register"; } } #endif // _LIBUNWIND_TARGET_MIPS_NEWABI + +#if defined(_LIBUNWIND_TARGET_SPARC) +/// Registers_sparc holds the register state of a thread in a 32-bit Sparc +/// process. +class _LIBUNWIND_HIDDEN Registers_sparc { +public: + Registers_sparc(); + Registers_sparc(const void *registers); + + bool validRegister(int num) const; + uint32_t getRegister(int num) const; + void setRegister(int num, uint32_t value); + bool validFloatRegister(int num) const; + double getFloatRegister(int num) const; + void setFloatRegister(int num, double value); + bool validVectorRegister(int num) const; + v128 getVectorRegister(int num) const; + void setVectorRegister(int num, v128 value); + static const char *getRegisterName(int num); + void jumpto(); + static int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC; } + static int getArch() { return REGISTERS_SPARC; } + + uint64_t getSP() const { return _registers.__regs[UNW_SPARC_O6]; } + void setSP(uint32_t value) { _registers.__regs[UNW_SPARC_O6] = value; } + uint64_t getIP() const { return _registers.__regs[UNW_SPARC_O7]; } + void setIP(uint32_t value) { _registers.__regs[UNW_SPARC_O7] = value; } + +private: + struct sparc_thread_state_t { + unsigned int __regs[32]; + }; + + sparc_thread_state_t _registers; +}; + +inline Registers_sparc::Registers_sparc(const void *registers) { + static_assert((check_fit::does_fit), + "sparc registers do not fit into unw_context_t"); + memcpy(&_registers, static_cast(registers), + sizeof(_registers)); +} + +inline Registers_sparc::Registers_sparc() { + memset(&_registers, 0, sizeof(_registers)); +} + +inline bool Registers_sparc::validRegister(int regNum) const { + if (regNum == UNW_REG_IP) + return true; + if (regNum == UNW_REG_SP) + return true; + if (regNum < 0) + return false; + if (regNum <= UNW_SPARC_I7) + return true; + return false; +} + +inline uint32_t Registers_sparc::getRegister(int regNum) const { + if ((UNW_SPARC_G0 <= regNum) && (regNum <= UNW_SPARC_I7)) { + return _registers.__regs[regNum]; + } + + switch (regNum) { + case UNW_REG_IP: + return _registers.__regs[UNW_SPARC_O7]; + case UNW_REG_SP: + return _registers.__regs[UNW_SPARC_O6]; + } + _LIBUNWIND_ABORT("unsupported sparc register"); +} + +inline void Registers_sparc::setRegister(int regNum, uint32_t value) { + if ((UNW_SPARC_G0 <= regNum) && (regNum <= UNW_SPARC_I7)) { + _registers.__regs[regNum] = value; + return; + } + + switch (regNum) { + case UNW_REG_IP: + _registers.__regs[UNW_SPARC_O7] = value; + return; + case UNW_REG_SP: + _registers.__regs[UNW_SPARC_O6] = value; + return; + } + _LIBUNWIND_ABORT("unsupported sparc register"); +} + +inline bool Registers_sparc::validFloatRegister(int) const { return false; } + +inline double Registers_sparc::getFloatRegister(int) const { + _LIBUNWIND_ABORT("no Sparc float registers"); +} + +inline void Registers_sparc::setFloatRegister(int, double) { + _LIBUNWIND_ABORT("no Sparc float registers"); +} + +inline bool Registers_sparc::validVectorRegister(int) const { return false; } + +inline v128 Registers_sparc::getVectorRegister(int) const { + _LIBUNWIND_ABORT("no Sparc vector registers"); +} + +inline void Registers_sparc::setVectorRegister(int, v128) { + _LIBUNWIND_ABORT("no Sparc vector registers"); +} + +inline const char *Registers_sparc::getRegisterName(int regNum) { + switch (regNum) { + case UNW_REG_IP: + return "pc"; + case UNW_SPARC_G0: + return "g0"; + case UNW_SPARC_G1: + return "g1"; + case UNW_SPARC_G2: + return "g2"; + case UNW_SPARC_G3: + return "g3"; + case UNW_SPARC_G4: + return "g4"; + case UNW_SPARC_G5: + return "g5"; + case UNW_SPARC_G6: + return "g6"; + case UNW_SPARC_G7: + return "g7"; + case UNW_SPARC_O0: + return "o0"; + case UNW_SPARC_O1: + return "o1"; + case UNW_SPARC_O2: + return "o2"; + case UNW_SPARC_O3: + return "o3"; + case UNW_SPARC_O4: + return "o4"; + case UNW_SPARC_O5: + return "o5"; + case UNW_REG_SP: + case UNW_SPARC_O6: + return "sp"; + case UNW_SPARC_O7: + return "o7"; + case UNW_SPARC_L0: + return "l0"; + case UNW_SPARC_L1: + return "l1"; + case UNW_SPARC_L2: + return "l2"; + case UNW_SPARC_L3: + return "l3"; + case UNW_SPARC_L4: + return "l4"; + case UNW_SPARC_L5: + return "l5"; + case UNW_SPARC_L6: + return "l6"; + case UNW_SPARC_L7: + return "l7"; + case UNW_SPARC_I0: + return "i0"; + case UNW_SPARC_I1: + return "i1"; + case UNW_SPARC_I2: + return "i2"; + case UNW_SPARC_I3: + return "i3"; + case UNW_SPARC_I4: + return "i4"; + case UNW_SPARC_I5: + return "i5"; + case UNW_SPARC_I6: + return "fp"; + case UNW_SPARC_I7: + return "i7"; + default: + return "unknown register"; + } +} +#endif // _LIBUNWIND_TARGET_SPARC + } // namespace libunwind #endif // __REGISTERS_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.cpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.cpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.cpp (revision 345026) @@ -1,977 +1,991 @@ //===--------------------------- Unwind-EHABI.cpp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Implements ARM zero-cost C++ exceptions // //===----------------------------------------------------------------------===// #include "Unwind-EHABI.h" -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) +#include #include #include #include #include #include #include #include "config.h" #include "libunwind.h" #include "libunwind_ext.h" #include "unwind.h" namespace { // Strange order: take words in order, but inside word, take from most to least // signinficant byte. uint8_t getByte(const uint32_t* data, size_t offset) { const uint8_t* byteData = reinterpret_cast(data); return byteData[(offset & ~(size_t)0x03) + (3 - (offset & (size_t)0x03))]; } const char* getNextWord(const char* data, uint32_t* out) { *out = *reinterpret_cast(data); return data + 4; } const char* getNextNibble(const char* data, uint32_t* out) { *out = *reinterpret_cast(data); return data + 2; } struct Descriptor { // See # 9.2 typedef enum { SU16 = 0, // Short descriptor, 16-bit entries LU16 = 1, // Long descriptor, 16-bit entries LU32 = 3, // Long descriptor, 32-bit entries RESERVED0 = 4, RESERVED1 = 5, RESERVED2 = 6, RESERVED3 = 7, RESERVED4 = 8, RESERVED5 = 9, RESERVED6 = 10, RESERVED7 = 11, RESERVED8 = 12, RESERVED9 = 13, RESERVED10 = 14, RESERVED11 = 15 } Format; // See # 9.2 typedef enum { CLEANUP = 0x0, FUNC = 0x1, CATCH = 0x2, INVALID = 0x4 } Kind; }; _Unwind_Reason_Code ProcessDescriptors( _Unwind_State state, _Unwind_Control_Block* ucbp, struct _Unwind_Context* context, Descriptor::Format format, const char* descriptorStart, uint32_t flags) { // EHT is inlined in the index using compact form. No descriptors. #5 if (flags & 0x1) return _URC_CONTINUE_UNWIND; // TODO: We should check the state here, and determine whether we need to // perform phase1 or phase2 unwinding. (void)state; const char* descriptor = descriptorStart; uint32_t descriptorWord; getNextWord(descriptor, &descriptorWord); while (descriptorWord) { // Read descriptor based on # 9.2. uint32_t length; uint32_t offset; switch (format) { case Descriptor::LU32: descriptor = getNextWord(descriptor, &length); descriptor = getNextWord(descriptor, &offset); case Descriptor::LU16: descriptor = getNextNibble(descriptor, &length); descriptor = getNextNibble(descriptor, &offset); default: assert(false); return _URC_FAILURE; } // See # 9.2 table for decoding the kind of descriptor. It's a 2-bit value. Descriptor::Kind kind = static_cast((length & 0x1) | ((offset & 0x1) << 1)); // Clear off flag from last bit. length &= ~1u; offset &= ~1u; uintptr_t scopeStart = ucbp->pr_cache.fnstart + offset; uintptr_t scopeEnd = scopeStart + length; uintptr_t pc = _Unwind_GetIP(context); bool isInScope = (scopeStart <= pc) && (pc < scopeEnd); switch (kind) { case Descriptor::CLEANUP: { // TODO(ajwong): Handle cleanup descriptors. break; } case Descriptor::FUNC: { // TODO(ajwong): Handle function descriptors. break; } case Descriptor::CATCH: { // Catch descriptors require gobbling one more word. uint32_t landing_pad; descriptor = getNextWord(descriptor, &landing_pad); if (isInScope) { // TODO(ajwong): This is only phase1 compatible logic. Implement // phase2. landing_pad = signExtendPrel31(landing_pad & ~0x80000000); if (landing_pad == 0xffffffff) { return _URC_HANDLER_FOUND; } else if (landing_pad == 0xfffffffe) { return _URC_FAILURE; } else { /* bool is_reference_type = landing_pad & 0x80000000; void* matched_object; if (__cxxabiv1::__cxa_type_match( ucbp, reinterpret_cast(landing_pad), is_reference_type, &matched_object) != __cxxabiv1::ctm_failed) return _URC_HANDLER_FOUND; */ _LIBUNWIND_ABORT("Type matching not implemented"); } } break; } default: _LIBUNWIND_ABORT("Invalid descriptor kind found."); } getNextWord(descriptor, &descriptorWord); } return _URC_CONTINUE_UNWIND; } static _Unwind_Reason_Code unwindOneFrame(_Unwind_State state, _Unwind_Control_Block* ucbp, struct _Unwind_Context* context) { // Read the compact model EHT entry's header # 6.3 const uint32_t* unwindingData = ucbp->pr_cache.ehtp; assert((*unwindingData & 0xf0000000) == 0x80000000 && "Must be a compact entry"); Descriptor::Format format = static_cast((*unwindingData & 0x0f000000) >> 24); const char *lsda = reinterpret_cast(_Unwind_GetLanguageSpecificData(context)); // Handle descriptors before unwinding so they are processed in the context // of the correct stack frame. _Unwind_Reason_Code result = ProcessDescriptors(state, ucbp, context, format, lsda, ucbp->pr_cache.additional); if (result != _URC_CONTINUE_UNWIND) return result; if (unw_step(reinterpret_cast(context)) != UNW_STEP_SUCCESS) return _URC_FAILURE; return _URC_CONTINUE_UNWIND; } // Generates mask discriminator for _Unwind_VRS_Pop, e.g. for _UVRSC_CORE / // _UVRSD_UINT32. uint32_t RegisterMask(uint8_t start, uint8_t count_minus_one) { return ((1U << (count_minus_one + 1)) - 1) << start; } // Generates mask discriminator for _Unwind_VRS_Pop, e.g. for _UVRSC_VFP / // _UVRSD_DOUBLE. uint32_t RegisterRange(uint8_t start, uint8_t count_minus_one) { return ((uint32_t)start << 16) | ((uint32_t)count_minus_one + 1); } } // end anonymous namespace /** * Decodes an EHT entry. * * @param data Pointer to EHT. * @param[out] off Offset from return value (in bytes) to begin interpretation. * @param[out] len Number of bytes in unwind code. * @return Pointer to beginning of unwind code. */ extern "C" const uint32_t* decode_eht_entry(const uint32_t* data, size_t* off, size_t* len) { if ((*data & 0x80000000) == 0) { // 6.2: Generic Model // // EHT entry is a prel31 pointing to the PR, followed by data understood // only by the personality routine. Fortunately, all existing assembler // implementations, including GNU assembler, LLVM integrated assembler, // and ARM assembler, assume that the unwind opcodes come after the // personality rountine address. *off = 1; // First byte is size data. *len = (((data[1] >> 24) & 0xff) + 1) * 4; data++; // Skip the first word, which is the prel31 offset. } else { // 6.3: ARM Compact Model // // EHT entries here correspond to the __aeabi_unwind_cpp_pr[012] PRs indeded // by format: Descriptor::Format format = static_cast((*data & 0x0f000000) >> 24); switch (format) { case Descriptor::SU16: *len = 4; *off = 1; break; case Descriptor::LU16: case Descriptor::LU32: *len = 4 + 4 * ((*data & 0x00ff0000) >> 16); *off = 2; break; default: return nullptr; } } return data; } -_Unwind_Reason_Code _Unwind_VRS_Interpret( - _Unwind_Context* context, - const uint32_t* data, - size_t offset, - size_t len) { +_LIBUNWIND_EXPORT _Unwind_Reason_Code +_Unwind_VRS_Interpret(_Unwind_Context *context, const uint32_t *data, + size_t offset, size_t len) { bool wrotePC = false; bool finish = false; while (offset < len && !finish) { uint8_t byte = getByte(data, offset++); if ((byte & 0x80) == 0) { uint32_t sp; _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); if (byte & 0x40) sp -= (((uint32_t)byte & 0x3f) << 2) + 4; else sp += ((uint32_t)byte << 2) + 4; _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); } else { switch (byte & 0xf0) { case 0x80: { if (offset >= len) return _URC_FAILURE; uint32_t registers = (((uint32_t)byte & 0x0f) << 12) | (((uint32_t)getByte(data, offset++)) << 4); if (!registers) return _URC_FAILURE; if (registers & (1 << 15)) wrotePC = true; _Unwind_VRS_Pop(context, _UVRSC_CORE, registers, _UVRSD_UINT32); break; } case 0x90: { uint8_t reg = byte & 0x0f; if (reg == 13 || reg == 15) return _URC_FAILURE; uint32_t sp; _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_R0 + reg, _UVRSD_UINT32, &sp); _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); break; } case 0xa0: { uint32_t registers = RegisterMask(4, byte & 0x07); if (byte & 0x08) registers |= 1 << 14; _Unwind_VRS_Pop(context, _UVRSC_CORE, registers, _UVRSD_UINT32); break; } case 0xb0: { switch (byte) { case 0xb0: finish = true; break; case 0xb1: { if (offset >= len) return _URC_FAILURE; uint8_t registers = getByte(data, offset++); if (registers & 0xf0 || !registers) return _URC_FAILURE; _Unwind_VRS_Pop(context, _UVRSC_CORE, registers, _UVRSD_UINT32); break; } case 0xb2: { uint32_t addend = 0; uint32_t shift = 0; // This decodes a uleb128 value. while (true) { if (offset >= len) return _URC_FAILURE; uint32_t v = getByte(data, offset++); addend |= (v & 0x7f) << shift; if ((v & 0x80) == 0) break; shift += 7; } uint32_t sp; _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); sp += 0x204 + (addend << 2); _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); break; } case 0xb3: { uint8_t v = getByte(data, offset++); _Unwind_VRS_Pop(context, _UVRSC_VFP, RegisterRange(static_cast(v >> 4), v & 0x0f), _UVRSD_VFPX); break; } case 0xb4: case 0xb5: case 0xb6: case 0xb7: return _URC_FAILURE; default: _Unwind_VRS_Pop(context, _UVRSC_VFP, RegisterRange(8, byte & 0x07), _UVRSD_VFPX); break; } break; } case 0xc0: { switch (byte) { +#if defined(__ARM_WMMX) case 0xc0: case 0xc1: case 0xc2: case 0xc3: case 0xc4: case 0xc5: _Unwind_VRS_Pop(context, _UVRSC_WMMXD, RegisterRange(10, byte & 0x7), _UVRSD_DOUBLE); break; case 0xc6: { uint8_t v = getByte(data, offset++); uint8_t start = static_cast(v >> 4); uint8_t count_minus_one = v & 0xf; if (start + count_minus_one >= 16) return _URC_FAILURE; _Unwind_VRS_Pop(context, _UVRSC_WMMXD, RegisterRange(start, count_minus_one), _UVRSD_DOUBLE); break; } case 0xc7: { uint8_t v = getByte(data, offset++); if (!v || v & 0xf0) return _URC_FAILURE; _Unwind_VRS_Pop(context, _UVRSC_WMMXC, v, _UVRSD_DOUBLE); break; } +#endif case 0xc8: case 0xc9: { uint8_t v = getByte(data, offset++); uint8_t start = static_cast(((byte == 0xc8) ? 16 : 0) + (v >> 4)); uint8_t count_minus_one = v & 0xf; if (start + count_minus_one >= 32) return _URC_FAILURE; _Unwind_VRS_Pop(context, _UVRSC_VFP, RegisterRange(start, count_minus_one), _UVRSD_DOUBLE); break; } default: return _URC_FAILURE; } break; } case 0xd0: { if (byte & 0x08) return _URC_FAILURE; _Unwind_VRS_Pop(context, _UVRSC_VFP, RegisterRange(8, byte & 0x7), _UVRSD_DOUBLE); break; } default: return _URC_FAILURE; } } } if (!wrotePC) { uint32_t lr; _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_LR, _UVRSD_UINT32, &lr); _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_IP, _UVRSD_UINT32, &lr); } return _URC_CONTINUE_UNWIND; } -extern "C" _Unwind_Reason_Code __aeabi_unwind_cpp_pr0( - _Unwind_State state, - _Unwind_Control_Block *ucbp, - _Unwind_Context *context) { +extern "C" _LIBUNWIND_EXPORT _Unwind_Reason_Code +__aeabi_unwind_cpp_pr0(_Unwind_State state, _Unwind_Control_Block *ucbp, + _Unwind_Context *context) { return unwindOneFrame(state, ucbp, context); } -extern "C" _Unwind_Reason_Code __aeabi_unwind_cpp_pr1( - _Unwind_State state, - _Unwind_Control_Block *ucbp, - _Unwind_Context *context) { +extern "C" _LIBUNWIND_EXPORT _Unwind_Reason_Code +__aeabi_unwind_cpp_pr1(_Unwind_State state, _Unwind_Control_Block *ucbp, + _Unwind_Context *context) { return unwindOneFrame(state, ucbp, context); } -extern "C" _Unwind_Reason_Code __aeabi_unwind_cpp_pr2( - _Unwind_State state, - _Unwind_Control_Block *ucbp, - _Unwind_Context *context) { +extern "C" _LIBUNWIND_EXPORT _Unwind_Reason_Code +__aeabi_unwind_cpp_pr2(_Unwind_State state, _Unwind_Control_Block *ucbp, + _Unwind_Context *context) { return unwindOneFrame(state, ucbp, context); } static _Unwind_Reason_Code unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object) { // EHABI #7.3 discusses preserving the VRS in a "temporary VRS" during // phase 1 and then restoring it to the "primary VRS" for phase 2. The // effect is phase 2 doesn't see any of the VRS manipulations from phase 1. // In this implementation, the phases don't share the VRS backing store. // Instead, they are passed the original |uc| and they create a new VRS // from scratch thus achieving the same effect. unw_init_local(cursor, uc); // Walk each frame looking for a place to stop. for (bool handlerNotFound = true; handlerNotFound;) { // See if frame has code to run (has personality routine). unw_proc_info_t frameInfo; if (unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): unw_get_proc_info " "failed => _URC_FATAL_PHASE1_ERROR", static_cast(exception_object)); return _URC_FATAL_PHASE1_ERROR; } // When tracing, print state information. if (_LIBUNWIND_TRACING_UNWINDING) { char functionBuf[512]; const char *functionName = functionBuf; unw_word_t offset; if ((unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf), &offset) != UNW_ESUCCESS) || (frameInfo.start_ip + offset > frameInfo.end_ip)) functionName = ".anonymous."; unw_word_t pc; unw_get_reg(cursor, UNW_REG_IP, &pc); _LIBUNWIND_TRACE_UNWINDING( - "unwind_phase1(ex_ojb=%p): pc=0x%llX, start_ip=0x%llX, func=%s, " - "lsda=0x%llX, personality=0x%llX", - static_cast(exception_object), (long long)pc, - (long long)frameInfo.start_ip, functionName, - (long long)frameInfo.lsda, (long long)frameInfo.handler); + "unwind_phase1(ex_ojb=%p): pc=0x%" PRIxPTR ", start_ip=0x%" PRIxPTR ", func=%s, " + "lsda=0x%" PRIxPTR ", personality=0x%" PRIxPTR, + static_cast(exception_object), pc, + frameInfo.start_ip, functionName, + frameInfo.lsda, frameInfo.handler); } // If there is a personality routine, ask it if it will want to stop at // this frame. if (frameInfo.handler != 0) { __personality_routine p = (__personality_routine)(long)(frameInfo.handler); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): calling personality function %p", static_cast(exception_object), reinterpret_cast(reinterpret_cast(p))); struct _Unwind_Context *context = (struct _Unwind_Context *)(cursor); exception_object->pr_cache.fnstart = frameInfo.start_ip; exception_object->pr_cache.ehtp = (_Unwind_EHT_Header *)frameInfo.unwind_info; exception_object->pr_cache.additional = frameInfo.flags; _Unwind_Reason_Code personalityResult = (*p)(_US_VIRTUAL_UNWIND_FRAME, exception_object, context); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): personality result %d start_ip %x ehtp %p " "additional %x", static_cast(exception_object), personalityResult, exception_object->pr_cache.fnstart, static_cast(exception_object->pr_cache.ehtp), exception_object->pr_cache.additional); switch (personalityResult) { case _URC_HANDLER_FOUND: // found a catch clause or locals that need destructing in this frame // stop search and remember stack pointer at the frame handlerNotFound = false; // p should have initialized barrier_cache. EHABI #7.3.5 _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_HANDLER_FOUND", static_cast(exception_object)); return _URC_NO_REASON; case _URC_CONTINUE_UNWIND: _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_CONTINUE_UNWIND", static_cast(exception_object)); // continue unwinding break; // EHABI #7.3.3 case _URC_FAILURE: return _URC_FAILURE; default: // something went wrong _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_FATAL_PHASE1_ERROR", static_cast(exception_object)); return _URC_FATAL_PHASE1_ERROR; } } } return _URC_NO_REASON; } static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object, bool resume) { // See comment at the start of unwind_phase1 regarding VRS integrity. unw_init_local(cursor, uc); _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p)", static_cast(exception_object)); int frame_count = 0; // Walk each frame until we reach where search phase said to stop. while (true) { // Ask libunwind to get next frame (skip over first which is // _Unwind_RaiseException or _Unwind_Resume). // // Resume only ever makes sense for 1 frame. _Unwind_State state = resume ? _US_UNWIND_FRAME_RESUME : _US_UNWIND_FRAME_STARTING; if (resume && frame_count == 1) { // On a resume, first unwind the _Unwind_Resume() frame. The next frame // is now the landing pad for the cleanup from a previous execution of // phase2. To continue unwindingly correctly, replace VRS[15] with the // IP of the frame that the previous run of phase2 installed the context // for. After this, continue unwinding as if normal. // // See #7.4.6 for details. unw_set_reg(cursor, UNW_REG_IP, exception_object->unwinder_cache.reserved2); resume = false; } // Get info about this frame. unw_word_t sp; unw_proc_info_t frameInfo; unw_get_reg(cursor, UNW_REG_SP, &sp); if (unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_get_proc_info " "failed => _URC_FATAL_PHASE2_ERROR", static_cast(exception_object)); return _URC_FATAL_PHASE2_ERROR; } // When tracing, print state information. if (_LIBUNWIND_TRACING_UNWINDING) { char functionBuf[512]; const char *functionName = functionBuf; unw_word_t offset; if ((unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf), &offset) != UNW_ESUCCESS) || (frameInfo.start_ip + offset > frameInfo.end_ip)) functionName = ".anonymous."; _LIBUNWIND_TRACE_UNWINDING( - "unwind_phase2(ex_ojb=%p): start_ip=0x%llX, func=%s, sp=0x%llX, " - "lsda=0x%llX, personality=0x%llX", - static_cast(exception_object), (long long)frameInfo.start_ip, - functionName, (long long)sp, (long long)frameInfo.lsda, - (long long)frameInfo.handler); + "unwind_phase2(ex_ojb=%p): start_ip=0x%" PRIxPTR ", func=%s, sp=0x%" PRIxPTR ", " + "lsda=0x%" PRIxPTR ", personality=0x%" PRIxPTR "", + static_cast(exception_object), frameInfo.start_ip, + functionName, sp, frameInfo.lsda, + frameInfo.handler); } // If there is a personality routine, tell it we are unwinding. if (frameInfo.handler != 0) { __personality_routine p = (__personality_routine)(long)(frameInfo.handler); struct _Unwind_Context *context = (struct _Unwind_Context *)(cursor); // EHABI #7.2 exception_object->pr_cache.fnstart = frameInfo.start_ip; exception_object->pr_cache.ehtp = (_Unwind_EHT_Header *)frameInfo.unwind_info; exception_object->pr_cache.additional = frameInfo.flags; _Unwind_Reason_Code personalityResult = (*p)(state, exception_object, context); switch (personalityResult) { case _URC_CONTINUE_UNWIND: // Continue unwinding _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2(ex_ojb=%p): _URC_CONTINUE_UNWIND", static_cast(exception_object)); // EHABI #7.2 if (sp == exception_object->barrier_cache.sp) { // Phase 1 said we would stop at this frame, but we did not... _LIBUNWIND_ABORT("during phase1 personality function said it would " "stop here, but now in phase2 it did not stop here"); } break; case _URC_INSTALL_CONTEXT: _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2(ex_ojb=%p): _URC_INSTALL_CONTEXT", static_cast(exception_object)); // Personality routine says to transfer control to landing pad. // We may get control back if landing pad calls _Unwind_Resume(). if (_LIBUNWIND_TRACING_UNWINDING) { unw_word_t pc; unw_get_reg(cursor, UNW_REG_IP, &pc); unw_get_reg(cursor, UNW_REG_SP, &sp); _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): re-entering " - "user code with ip=0x%llX, sp=0x%llX", + "user code with ip=0x%" PRIxPTR ", sp=0x%" PRIxPTR, static_cast(exception_object), - (long long)pc, (long long)sp); + pc, sp); } { // EHABI #7.4.1 says we need to preserve pc for when _Unwind_Resume // is called back, to find this same frame. unw_word_t pc; unw_get_reg(cursor, UNW_REG_IP, &pc); exception_object->unwinder_cache.reserved2 = (uint32_t)pc; } unw_resume(cursor); // unw_resume() only returns if there was an error. return _URC_FATAL_PHASE2_ERROR; // # EHABI #7.4.3 case _URC_FAILURE: abort(); default: // Personality routine returned an unknown result code. _LIBUNWIND_DEBUG_LOG("personality function returned unknown result %d", personalityResult); return _URC_FATAL_PHASE2_ERROR; } } frame_count++; } // Clean up phase did not resume at the frame that the search phase // said it would... return _URC_FATAL_PHASE2_ERROR; } /// Called by __cxa_throw. Only returns if there is a fatal error. _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_RaiseException(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_RaiseException(ex_obj=%p)", static_cast(exception_object)); unw_context_t uc; unw_cursor_t cursor; unw_getcontext(&uc); // This field for is for compatibility with GCC to say this isn't a forced // unwind. EHABI #7.2 exception_object->unwinder_cache.reserved1 = 0; // phase 1: the search phase _Unwind_Reason_Code phase1 = unwind_phase1(&uc, &cursor, exception_object); if (phase1 != _URC_NO_REASON) return phase1; // phase 2: the clean up phase return unwind_phase2(&uc, &cursor, exception_object, false); } _LIBUNWIND_EXPORT void _Unwind_Complete(_Unwind_Exception* exception_object) { // This is to be called when exception handling completes to give us a chance // to perform any housekeeping. EHABI #7.2. But we have nothing to do here. (void)exception_object; } /// When _Unwind_RaiseException() is in phase2, it hands control /// to the personality function at each frame. The personality /// may force a jump to a landing pad in that function, the landing /// pad code may then call _Unwind_Resume() to continue with the /// unwinding. Note: the call to _Unwind_Resume() is from compiler /// geneated user code. All other _Unwind_* routines are called /// by the C++ runtime __cxa_* routines. /// /// Note: re-throwing an exception (as opposed to continuing the unwind) /// is implemented by having the code call __cxa_rethrow() which /// in turn calls _Unwind_Resume_or_Rethrow(). _LIBUNWIND_EXPORT void _Unwind_Resume(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_Resume(ex_obj=%p)", static_cast(exception_object)); unw_context_t uc; unw_cursor_t cursor; unw_getcontext(&uc); // _Unwind_RaiseException on EHABI will always set the reserved1 field to 0, // which is in the same position as private_1 below. // TODO(ajwong): Who wronte the above? Why is it true? unwind_phase2(&uc, &cursor, exception_object, true); // Clients assume _Unwind_Resume() does not return, so all we can do is abort. _LIBUNWIND_ABORT("_Unwind_Resume() can't return"); } /// Called by personality handler during phase 2 to get LSDA for current frame. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_proc_info_t frameInfo; uintptr_t result = 0; if (unw_get_proc_info(cursor, &frameInfo) == UNW_ESUCCESS) result = (uintptr_t)frameInfo.lsda; _LIBUNWIND_TRACE_API( "_Unwind_GetLanguageSpecificData(context=%p) => 0x%llx", static_cast(context), (long long)result); return result; } static uint64_t ValueAsBitPattern(_Unwind_VRS_DataRepresentation representation, void* valuep) { uint64_t value = 0; switch (representation) { case _UVRSD_UINT32: case _UVRSD_FLOAT: memcpy(&value, valuep, sizeof(uint32_t)); break; case _UVRSD_VFPX: case _UVRSD_UINT64: case _UVRSD_DOUBLE: memcpy(&value, valuep, sizeof(uint64_t)); break; } return value; } -_Unwind_VRS_Result +_LIBUNWIND_EXPORT _Unwind_VRS_Result _Unwind_VRS_Set(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t regno, _Unwind_VRS_DataRepresentation representation, void *valuep) { _LIBUNWIND_TRACE_API("_Unwind_VRS_Set(context=%p, regclass=%d, reg=%d, " "rep=%d, value=0x%llX)", static_cast(context), regclass, regno, representation, ValueAsBitPattern(representation, valuep)); unw_cursor_t *cursor = (unw_cursor_t *)context; switch (regclass) { case _UVRSC_CORE: if (representation != _UVRSD_UINT32 || regno > 15) return _UVRSR_FAILED; return unw_set_reg(cursor, (unw_regnum_t)(UNW_ARM_R0 + regno), *(unw_word_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; - case _UVRSC_WMMXC: - if (representation != _UVRSD_UINT32 || regno > 3) - return _UVRSR_FAILED; - return unw_set_reg(cursor, (unw_regnum_t)(UNW_ARM_WC0 + regno), - *(unw_word_t *)valuep) == UNW_ESUCCESS - ? _UVRSR_OK - : _UVRSR_FAILED; case _UVRSC_VFP: if (representation != _UVRSD_VFPX && representation != _UVRSD_DOUBLE) return _UVRSR_FAILED; if (representation == _UVRSD_VFPX) { // Can only touch d0-15 with FSTMFDX. if (regno > 15) return _UVRSR_FAILED; unw_save_vfp_as_X(cursor); } else { if (regno > 31) return _UVRSR_FAILED; } return unw_set_fpreg(cursor, (unw_regnum_t)(UNW_ARM_D0 + regno), *(unw_fpreg_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; +#if defined(__ARM_WMMX) + case _UVRSC_WMMXC: + if (representation != _UVRSD_UINT32 || regno > 3) + return _UVRSR_FAILED; + return unw_set_reg(cursor, (unw_regnum_t)(UNW_ARM_WC0 + regno), + *(unw_word_t *)valuep) == UNW_ESUCCESS + ? _UVRSR_OK + : _UVRSR_FAILED; case _UVRSC_WMMXD: if (representation != _UVRSD_DOUBLE || regno > 31) return _UVRSR_FAILED; return unw_set_fpreg(cursor, (unw_regnum_t)(UNW_ARM_WR0 + regno), *(unw_fpreg_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; +#else + case _UVRSC_WMMXC: + case _UVRSC_WMMXD: + break; +#endif } _LIBUNWIND_ABORT("unsupported register class"); } static _Unwind_VRS_Result _Unwind_VRS_Get_Internal(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t regno, _Unwind_VRS_DataRepresentation representation, void *valuep) { unw_cursor_t *cursor = (unw_cursor_t *)context; switch (regclass) { case _UVRSC_CORE: if (representation != _UVRSD_UINT32 || regno > 15) return _UVRSR_FAILED; return unw_get_reg(cursor, (unw_regnum_t)(UNW_ARM_R0 + regno), (unw_word_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; - case _UVRSC_WMMXC: - if (representation != _UVRSD_UINT32 || regno > 3) - return _UVRSR_FAILED; - return unw_get_reg(cursor, (unw_regnum_t)(UNW_ARM_WC0 + regno), - (unw_word_t *)valuep) == UNW_ESUCCESS - ? _UVRSR_OK - : _UVRSR_FAILED; case _UVRSC_VFP: if (representation != _UVRSD_VFPX && representation != _UVRSD_DOUBLE) return _UVRSR_FAILED; if (representation == _UVRSD_VFPX) { // Can only touch d0-15 with FSTMFDX. if (regno > 15) return _UVRSR_FAILED; unw_save_vfp_as_X(cursor); } else { if (regno > 31) return _UVRSR_FAILED; } return unw_get_fpreg(cursor, (unw_regnum_t)(UNW_ARM_D0 + regno), (unw_fpreg_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; +#if defined(__ARM_WMMX) + case _UVRSC_WMMXC: + if (representation != _UVRSD_UINT32 || regno > 3) + return _UVRSR_FAILED; + return unw_get_reg(cursor, (unw_regnum_t)(UNW_ARM_WC0 + regno), + (unw_word_t *)valuep) == UNW_ESUCCESS + ? _UVRSR_OK + : _UVRSR_FAILED; case _UVRSC_WMMXD: if (representation != _UVRSD_DOUBLE || regno > 31) return _UVRSR_FAILED; return unw_get_fpreg(cursor, (unw_regnum_t)(UNW_ARM_WR0 + regno), (unw_fpreg_t *)valuep) == UNW_ESUCCESS ? _UVRSR_OK : _UVRSR_FAILED; +#else + case _UVRSC_WMMXC: + case _UVRSC_WMMXD: + break; +#endif } _LIBUNWIND_ABORT("unsupported register class"); } -_Unwind_VRS_Result _Unwind_VRS_Get( - _Unwind_Context *context, - _Unwind_VRS_RegClass regclass, - uint32_t regno, - _Unwind_VRS_DataRepresentation representation, - void *valuep) { +_LIBUNWIND_EXPORT _Unwind_VRS_Result +_Unwind_VRS_Get(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, + uint32_t regno, _Unwind_VRS_DataRepresentation representation, + void *valuep) { _Unwind_VRS_Result result = _Unwind_VRS_Get_Internal(context, regclass, regno, representation, valuep); _LIBUNWIND_TRACE_API("_Unwind_VRS_Get(context=%p, regclass=%d, reg=%d, " "rep=%d, value=0x%llX, result = %d)", static_cast(context), regclass, regno, representation, ValueAsBitPattern(representation, valuep), result); return result; } _Unwind_VRS_Result _Unwind_VRS_Pop(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t discriminator, _Unwind_VRS_DataRepresentation representation) { _LIBUNWIND_TRACE_API("_Unwind_VRS_Pop(context=%p, regclass=%d, " "discriminator=%d, representation=%d)", static_cast(context), regclass, discriminator, representation); switch (regclass) { - case _UVRSC_CORE: - case _UVRSC_WMMXC: { + case _UVRSC_WMMXC: +#if !defined(__ARM_WMMX) + break; +#endif + case _UVRSC_CORE: { if (representation != _UVRSD_UINT32) return _UVRSR_FAILED; // When popping SP from the stack, we don't want to override it from the // computed new stack location. See EHABI #7.5.4 table 3. bool poppedSP = false; uint32_t* sp; if (_Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp) != _UVRSR_OK) { return _UVRSR_FAILED; } for (uint32_t i = 0; i < 16; ++i) { if (!(discriminator & static_cast(1 << i))) continue; uint32_t value = *sp++; if (regclass == _UVRSC_CORE && i == 13) poppedSP = true; if (_Unwind_VRS_Set(context, regclass, i, _UVRSD_UINT32, &value) != _UVRSR_OK) { return _UVRSR_FAILED; } } if (!poppedSP) { return _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); } return _UVRSR_OK; } - case _UVRSC_VFP: - case _UVRSC_WMMXD: { + case _UVRSC_WMMXD: +#if !defined(__ARM_WMMX) + break; +#endif + case _UVRSC_VFP: { if (representation != _UVRSD_VFPX && representation != _UVRSD_DOUBLE) return _UVRSR_FAILED; uint32_t first = discriminator >> 16; uint32_t count = discriminator & 0xffff; uint32_t end = first+count; uint32_t* sp; if (_Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp) != _UVRSR_OK) { return _UVRSR_FAILED; } // For _UVRSD_VFPX, we're assuming the data is stored in FSTMX "standard // format 1", which is equivalent to FSTMD + a padding word. for (uint32_t i = first; i < end; ++i) { // SP is only 32-bit aligned so don't copy 64-bit at a time. uint64_t value = *sp++; value |= ((uint64_t)(*sp++)) << 32; if (_Unwind_VRS_Set(context, regclass, i, representation, &value) != _UVRSR_OK) return _UVRSR_FAILED; } if (representation == _UVRSD_VFPX) ++sp; return _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp); } } _LIBUNWIND_ABORT("unsupported register class"); } /// Called by personality handler during phase 2 to find the start of the /// function. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetRegionStart(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_proc_info_t frameInfo; uintptr_t result = 0; if (unw_get_proc_info(cursor, &frameInfo) == UNW_ESUCCESS) result = (uintptr_t)frameInfo.start_ip; _LIBUNWIND_TRACE_API("_Unwind_GetRegionStart(context=%p) => 0x%llX", static_cast(context), (long long)result); return result; } /// Called by personality handler during phase 2 if a foreign exception // is caught. _LIBUNWIND_EXPORT void _Unwind_DeleteException(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_DeleteException(ex_obj=%p)", static_cast(exception_object)); if (exception_object->exception_cleanup != NULL) (*exception_object->exception_cleanup)(_URC_FOREIGN_EXCEPTION_CAUGHT, exception_object); } extern "C" _LIBUNWIND_EXPORT _Unwind_Reason_Code __gnu_unwind_frame(_Unwind_Exception *exception_object, struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; if (unw_step(cursor) != UNW_STEP_SUCCESS) return _URC_FAILURE; return _URC_OK; } -#endif // _LIBUNWIND_ARM_EHABI +#endif // defined(_LIBUNWIND_ARM_EHABI) Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-EHABI.h (revision 345026) @@ -1,51 +1,51 @@ //===------------------------- Unwind-EHABI.hpp ---------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // //===----------------------------------------------------------------------===// #ifndef __UNWIND_EHABI_H__ #define __UNWIND_EHABI_H__ #include <__libunwind_config.h> -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) #include #include // Unable to unwind in the ARM index table (section 5 EHABI). #define UNW_EXIDX_CANTUNWIND 0x1 static inline uint32_t signExtendPrel31(uint32_t data) { return data | ((data & 0x40000000u) << 1); } static inline uint32_t readPrel31(const uint32_t *data) { return (((uint32_t)(uintptr_t)data) + signExtendPrel31(*data)); } #if defined(__cplusplus) extern "C" { #endif extern _Unwind_Reason_Code __aeabi_unwind_cpp_pr0( _Unwind_State state, _Unwind_Control_Block *ucbp, _Unwind_Context *context); extern _Unwind_Reason_Code __aeabi_unwind_cpp_pr1( _Unwind_State state, _Unwind_Control_Block *ucbp, _Unwind_Context *context); extern _Unwind_Reason_Code __aeabi_unwind_cpp_pr2( _Unwind_State state, _Unwind_Control_Block *ucbp, _Unwind_Context *context); #if defined(__cplusplus) } // extern "C" #endif -#endif // _LIBUNWIND_ARM_EHABI +#endif // defined(_LIBUNWIND_ARM_EHABI) #endif // __UNWIND_EHABI_H__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-seh.cpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-seh.cpp (nonexistent) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-seh.cpp (revision 345026) @@ -0,0 +1,491 @@ +//===--------------------------- Unwind-seh.cpp ---------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements SEH-based Itanium C++ exceptions. +// +//===----------------------------------------------------------------------===// + +#include "config.h" + +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "libunwind_ext.h" +#include "UnwindCursor.hpp" + +using namespace libunwind; + +#define STATUS_USER_DEFINED (1u << 29) + +#define STATUS_GCC_MAGIC (('G' << 16) | ('C' << 8) | 'C') + +#define MAKE_CUSTOM_STATUS(s, c) \ + ((NTSTATUS)(((s) << 30) | STATUS_USER_DEFINED | (c))) +#define MAKE_GCC_EXCEPTION(c) \ + MAKE_CUSTOM_STATUS(STATUS_SEVERITY_SUCCESS, STATUS_GCC_MAGIC | ((c) << 24)) + +/// SEH exception raised by libunwind when the program calls +/// \c _Unwind_RaiseException. +#define STATUS_GCC_THROW MAKE_GCC_EXCEPTION(0) // 0x20474343 +/// SEH exception raised by libunwind to initiate phase 2 of exception +/// handling. +#define STATUS_GCC_UNWIND MAKE_GCC_EXCEPTION(1) // 0x21474343 + +/// Class of foreign exceptions based on unrecognized SEH exceptions. +static const uint64_t kSEHExceptionClass = 0x434C4E4753454800; // CLNGSEH\0 + +/// Exception cleanup routine used by \c _GCC_specific_handler to +/// free foreign exceptions. +static void seh_exc_cleanup(_Unwind_Reason_Code urc, _Unwind_Exception *exc) { + if (exc->exception_class != kSEHExceptionClass) + _LIBUNWIND_ABORT("SEH cleanup called on non-SEH exception"); + free(exc); +} + +static int _unw_init_seh(unw_cursor_t *cursor, CONTEXT *ctx); +static DISPATCHER_CONTEXT *_unw_seh_get_disp_ctx(unw_cursor_t *cursor); +static void _unw_seh_set_disp_ctx(unw_cursor_t *cursor, DISPATCHER_CONTEXT *disp); + +/// Common implementation of SEH-style handler functions used by Itanium- +/// style frames. Depending on how and why it was called, it may do one of: +/// a) Delegate to the given Itanium-style personality function; or +/// b) Initiate a collided unwind to halt unwinding. +_LIBUNWIND_EXPORT EXCEPTION_DISPOSITION +_GCC_specific_handler(PEXCEPTION_RECORD ms_exc, PVOID frame, PCONTEXT ms_ctx, + DISPATCHER_CONTEXT *disp, __personality_routine pers) { + unw_context_t uc; + unw_cursor_t cursor; + _Unwind_Exception *exc; + _Unwind_Action action; + struct _Unwind_Context *ctx = nullptr; + _Unwind_Reason_Code urc; + uintptr_t retval, target; + bool ours = false; + + _LIBUNWIND_TRACE_UNWINDING("_GCC_specific_handler(%#010x(%x), %p)", ms_exc->ExceptionCode, ms_exc->ExceptionFlags, frame); + if (ms_exc->ExceptionCode == STATUS_GCC_UNWIND) { + if (IS_TARGET_UNWIND(ms_exc->ExceptionFlags)) { + // Set up the upper return value (the lower one and the target PC + // were set in the call to RtlUnwindEx()) for the landing pad. +#ifdef __x86_64__ + disp->ContextRecord->Rdx = ms_exc->ExceptionInformation[3]; +#elif defined(__arm__) + disp->ContextRecord->R1 = ms_exc->ExceptionInformation[3]; +#elif defined(__aarch64__) + disp->ContextRecord->X1 = ms_exc->ExceptionInformation[3]; +#endif + } + // This is the collided unwind to the landing pad. Nothing to do. + return ExceptionContinueSearch; + } + + if (ms_exc->ExceptionCode == STATUS_GCC_THROW) { + // This is (probably) a libunwind-controlled exception/unwind. Recover the + // parameters which we set below, and pass them to the personality function. + ours = true; + exc = (_Unwind_Exception *)ms_exc->ExceptionInformation[0]; + if (!IS_UNWINDING(ms_exc->ExceptionFlags) && ms_exc->NumberParameters > 1) { + ctx = (struct _Unwind_Context *)ms_exc->ExceptionInformation[1]; + action = (_Unwind_Action)ms_exc->ExceptionInformation[2]; + } + } else { + // Foreign exception. + exc = (_Unwind_Exception *)malloc(sizeof(_Unwind_Exception)); + exc->exception_class = kSEHExceptionClass; + exc->exception_cleanup = seh_exc_cleanup; + memset(exc->private_, 0, sizeof(exc->private_)); + } + if (!ctx) { + _unw_init_seh(&cursor, disp->ContextRecord); + _unw_seh_set_disp_ctx(&cursor, disp); + unw_set_reg(&cursor, UNW_REG_IP, disp->ControlPc-1); + ctx = (struct _Unwind_Context *)&cursor; + + if (!IS_UNWINDING(ms_exc->ExceptionFlags)) { + if (ours && ms_exc->NumberParameters > 1) + action = (_Unwind_Action)(_UA_CLEANUP_PHASE | _UA_FORCE_UNWIND); + else + action = _UA_SEARCH_PHASE; + } else { + if (ours && ms_exc->ExceptionInformation[1] == (ULONG_PTR)frame) + action = (_Unwind_Action)(_UA_CLEANUP_PHASE | _UA_HANDLER_FRAME); + else + action = _UA_CLEANUP_PHASE; + } + } + + _LIBUNWIND_TRACE_UNWINDING("_GCC_specific_handler() calling personality function %p(1, %d, %llx, %p, %p)", pers, action, exc->exception_class, exc, ctx); + urc = pers(1, action, exc->exception_class, exc, ctx); + _LIBUNWIND_TRACE_UNWINDING("_GCC_specific_handler() personality returned %d", urc); + switch (urc) { + case _URC_CONTINUE_UNWIND: + // If we're in phase 2, and the personality routine said to continue + // at the target frame, we're in real trouble. + if (action & _UA_HANDLER_FRAME) + _LIBUNWIND_ABORT("Personality continued unwind at the target frame!"); + return ExceptionContinueSearch; + case _URC_HANDLER_FOUND: + // If we were called by __libunwind_seh_personality(), indicate that + // a handler was found; otherwise, initiate phase 2 by unwinding. + if (ours && ms_exc->NumberParameters > 1) + return 4 /* ExecptionExecuteHandler in mingw */; + // This should never happen in phase 2. + if (IS_UNWINDING(ms_exc->ExceptionFlags)) + _LIBUNWIND_ABORT("Personality indicated exception handler in phase 2!"); + exc->private_[1] = (ULONG_PTR)frame; + if (ours) { + ms_exc->NumberParameters = 4; + ms_exc->ExceptionInformation[1] = (ULONG_PTR)frame; + } + // FIXME: Indicate target frame in foreign case! + // phase 2: the clean up phase + RtlUnwindEx(frame, (PVOID)disp->ControlPc, ms_exc, exc, ms_ctx, disp->HistoryTable); + _LIBUNWIND_ABORT("RtlUnwindEx() failed"); + case _URC_INSTALL_CONTEXT: { + // If we were called by __libunwind_seh_personality(), indicate that + // a handler was found; otherwise, it's time to initiate a collided + // unwind to the target. + if (ours && !IS_UNWINDING(ms_exc->ExceptionFlags) && ms_exc->NumberParameters > 1) + return 4 /* ExecptionExecuteHandler in mingw */; + // This should never happen in phase 1. + if (!IS_UNWINDING(ms_exc->ExceptionFlags)) + _LIBUNWIND_ABORT("Personality installed context during phase 1!"); +#ifdef __x86_64__ + exc->private_[2] = disp->TargetIp; + unw_get_reg(&cursor, UNW_X86_64_RAX, &retval); + unw_get_reg(&cursor, UNW_X86_64_RDX, &exc->private_[3]); +#elif defined(__arm__) + exc->private_[2] = disp->TargetPc; + unw_get_reg(&cursor, UNW_ARM_R0, &retval); + unw_get_reg(&cursor, UNW_ARM_R1, &exc->private_[3]); +#elif defined(__aarch64__) + exc->private_[2] = disp->TargetPc; + unw_get_reg(&cursor, UNW_ARM64_X0, &retval); + unw_get_reg(&cursor, UNW_ARM64_X1, &exc->private_[3]); +#endif + unw_get_reg(&cursor, UNW_REG_IP, &target); + ms_exc->ExceptionCode = STATUS_GCC_UNWIND; +#ifdef __x86_64__ + ms_exc->ExceptionInformation[2] = disp->TargetIp; +#elif defined(__arm__) || defined(__aarch64__) + ms_exc->ExceptionInformation[2] = disp->TargetPc; +#endif + ms_exc->ExceptionInformation[3] = exc->private_[3]; + // Give NTRTL some scratch space to keep track of the collided unwind. + // Don't use the one that was passed in; we don't want to overwrite the + // context in the DISPATCHER_CONTEXT. + CONTEXT new_ctx; + RtlUnwindEx(frame, (PVOID)target, ms_exc, (PVOID)retval, &new_ctx, disp->HistoryTable); + _LIBUNWIND_ABORT("RtlUnwindEx() failed"); + } + // Anything else indicates a serious problem. + default: return ExceptionContinueExecution; + } +} + +/// Personality function returned by \c unw_get_proc_info() in SEH contexts. +/// This is a wrapper that calls the real SEH handler function, which in +/// turn (at least, for Itanium-style frames) calls the real Itanium +/// personality function (see \c _GCC_specific_handler()). +extern "C" _Unwind_Reason_Code +__libunwind_seh_personality(int version, _Unwind_Action state, + uint64_t klass, _Unwind_Exception *exc, + struct _Unwind_Context *context) { + EXCEPTION_RECORD ms_exc; + bool phase2 = (state & (_UA_SEARCH_PHASE|_UA_CLEANUP_PHASE)) == _UA_CLEANUP_PHASE; + ms_exc.ExceptionCode = STATUS_GCC_THROW; + ms_exc.ExceptionFlags = 0; + ms_exc.NumberParameters = 3; + ms_exc.ExceptionInformation[0] = (ULONG_PTR)exc; + ms_exc.ExceptionInformation[1] = (ULONG_PTR)context; + ms_exc.ExceptionInformation[2] = state; + DISPATCHER_CONTEXT *disp_ctx = _unw_seh_get_disp_ctx((unw_cursor_t *)context); + EXCEPTION_DISPOSITION ms_act = disp_ctx->LanguageHandler(&ms_exc, + (PVOID)disp_ctx->EstablisherFrame, + disp_ctx->ContextRecord, + disp_ctx); + switch (ms_act) { + case ExceptionContinueSearch: return _URC_CONTINUE_UNWIND; + case 4 /*ExceptionExecuteHandler*/: + return phase2 ? _URC_INSTALL_CONTEXT : _URC_HANDLER_FOUND; + default: + return phase2 ? _URC_FATAL_PHASE2_ERROR : _URC_FATAL_PHASE1_ERROR; + } +} + +static _Unwind_Reason_Code +unwind_phase2_forced(unw_context_t *uc, + _Unwind_Exception *exception_object, + _Unwind_Stop_Fn stop, void *stop_parameter) { + unw_cursor_t cursor2; + unw_init_local(&cursor2, uc); + + // Walk each frame until we reach where search phase said to stop + while (unw_step(&cursor2) > 0) { + + // Update info about this frame. + unw_proc_info_t frameInfo; + if (unw_get_proc_info(&cursor2, &frameInfo) != UNW_ESUCCESS) { + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): unw_step " + "failed => _URC_END_OF_STACK", + (void *)exception_object); + return _URC_FATAL_PHASE2_ERROR; + } + + // When tracing, print state information. + if (_LIBUNWIND_TRACING_UNWINDING) { + char functionBuf[512]; + const char *functionName = functionBuf; + unw_word_t offset; + if ((unw_get_proc_name(&cursor2, functionBuf, sizeof(functionBuf), + &offset) != UNW_ESUCCESS) || + (frameInfo.start_ip + offset > frameInfo.end_ip)) + functionName = ".anonymous."; + _LIBUNWIND_TRACE_UNWINDING( + "unwind_phase2_forced(ex_ojb=%p): start_ip=0x%" PRIx64 + ", func=%s, lsda=0x%" PRIx64 ", personality=0x%" PRIx64, + (void *)exception_object, frameInfo.start_ip, functionName, + frameInfo.lsda, frameInfo.handler); + } + + // Call stop function at each frame. + _Unwind_Action action = + (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE); + _Unwind_Reason_Code stopResult = + (*stop)(1, action, exception_object->exception_class, exception_object, + (struct _Unwind_Context *)(&cursor2), stop_parameter); + _LIBUNWIND_TRACE_UNWINDING( + "unwind_phase2_forced(ex_ojb=%p): stop function returned %d", + (void *)exception_object, stopResult); + if (stopResult != _URC_NO_REASON) { + _LIBUNWIND_TRACE_UNWINDING( + "unwind_phase2_forced(ex_ojb=%p): stopped by stop function", + (void *)exception_object); + return _URC_FATAL_PHASE2_ERROR; + } + + // If there is a personality routine, tell it we are unwinding. + if (frameInfo.handler != 0) { + __personality_routine p = + (__personality_routine)(intptr_t)(frameInfo.handler); + _LIBUNWIND_TRACE_UNWINDING( + "unwind_phase2_forced(ex_ojb=%p): calling personality function %p", + (void *)exception_object, (void *)(uintptr_t)p); + _Unwind_Reason_Code personalityResult = + (*p)(1, action, exception_object->exception_class, exception_object, + (struct _Unwind_Context *)(&cursor2)); + switch (personalityResult) { + case _URC_CONTINUE_UNWIND: + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " + "personality returned " + "_URC_CONTINUE_UNWIND", + (void *)exception_object); + // Destructors called, continue unwinding + break; + case _URC_INSTALL_CONTEXT: + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " + "personality returned " + "_URC_INSTALL_CONTEXT", + (void *)exception_object); + // We may get control back if landing pad calls _Unwind_Resume(). + unw_resume(&cursor2); + break; + default: + // Personality routine returned an unknown result code. + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " + "personality returned %d, " + "_URC_FATAL_PHASE2_ERROR", + (void *)exception_object, personalityResult); + return _URC_FATAL_PHASE2_ERROR; + } + } + } + + // Call stop function one last time and tell it we've reached the end + // of the stack. + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): calling stop " + "function with _UA_END_OF_STACK", + (void *)exception_object); + _Unwind_Action lastAction = + (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE | _UA_END_OF_STACK); + (*stop)(1, lastAction, exception_object->exception_class, exception_object, + (struct _Unwind_Context *)(&cursor2), stop_parameter); + + // Clean up phase did not resume at the frame that the search phase said it + // would. + return _URC_FATAL_PHASE2_ERROR; +} + +/// Called by \c __cxa_throw(). Only returns if there is a fatal error. +_LIBUNWIND_EXPORT _Unwind_Reason_Code +_Unwind_RaiseException(_Unwind_Exception *exception_object) { + _LIBUNWIND_TRACE_API("_Unwind_RaiseException(ex_obj=%p)", + (void *)exception_object); + + // Mark that this is a non-forced unwind, so _Unwind_Resume() + // can do the right thing. + memset(exception_object->private_, 0, sizeof(exception_object->private_)); + + // phase 1: the search phase + // We'll let the system do that for us. + RaiseException(STATUS_GCC_THROW, 0, 1, (ULONG_PTR *)&exception_object); + + // If we get here, either something went horribly wrong or we reached the + // top of the stack. Either way, let libc++abi call std::terminate(). + return _URC_END_OF_STACK; +} + +/// When \c _Unwind_RaiseException() is in phase2, it hands control +/// to the personality function at each frame. The personality +/// may force a jump to a landing pad in that function; the landing +/// pad code may then call \c _Unwind_Resume() to continue with the +/// unwinding. Note: the call to \c _Unwind_Resume() is from compiler +/// geneated user code. All other \c _Unwind_* routines are called +/// by the C++ runtime \c __cxa_* routines. +/// +/// Note: re-throwing an exception (as opposed to continuing the unwind) +/// is implemented by having the code call \c __cxa_rethrow() which +/// in turn calls \c _Unwind_Resume_or_Rethrow(). +_LIBUNWIND_EXPORT void +_Unwind_Resume(_Unwind_Exception *exception_object) { + _LIBUNWIND_TRACE_API("_Unwind_Resume(ex_obj=%p)", (void *)exception_object); + + if (exception_object->private_[0] != 0) { + unw_context_t uc; + + unw_getcontext(&uc); + unwind_phase2_forced(&uc, exception_object, + (_Unwind_Stop_Fn) exception_object->private_[0], + (void *)exception_object->private_[4]); + } else { + // Recover the parameters for the unwind from the exception object + // so we can start unwinding again. + EXCEPTION_RECORD ms_exc; + CONTEXT ms_ctx; + UNWIND_HISTORY_TABLE hist; + + memset(&ms_exc, 0, sizeof(ms_exc)); + memset(&hist, 0, sizeof(hist)); + ms_exc.ExceptionCode = STATUS_GCC_THROW; + ms_exc.ExceptionFlags = EXCEPTION_NONCONTINUABLE; + ms_exc.NumberParameters = 4; + ms_exc.ExceptionInformation[0] = (ULONG_PTR)exception_object; + ms_exc.ExceptionInformation[1] = exception_object->private_[1]; + ms_exc.ExceptionInformation[2] = exception_object->private_[2]; + ms_exc.ExceptionInformation[3] = exception_object->private_[3]; + RtlUnwindEx((PVOID)exception_object->private_[1], + (PVOID)exception_object->private_[2], &ms_exc, + exception_object, &ms_ctx, &hist); + } + + // Clients assume _Unwind_Resume() does not return, so all we can do is abort. + _LIBUNWIND_ABORT("_Unwind_Resume() can't return"); +} + +/// Not used by C++. +/// Unwinds stack, calling "stop" function at each frame. +/// Could be used to implement \c longjmp(). +_LIBUNWIND_EXPORT _Unwind_Reason_Code +_Unwind_ForcedUnwind(_Unwind_Exception *exception_object, + _Unwind_Stop_Fn stop, void *stop_parameter) { + _LIBUNWIND_TRACE_API("_Unwind_ForcedUnwind(ex_obj=%p, stop=%p)", + (void *)exception_object, (void *)(uintptr_t)stop); + unw_context_t uc; + unw_getcontext(&uc); + + // Mark that this is a forced unwind, so _Unwind_Resume() can do + // the right thing. + exception_object->private_[0] = (uintptr_t) stop; + exception_object->private_[4] = (uintptr_t) stop_parameter; + + // do it + return unwind_phase2_forced(&uc, exception_object, stop, stop_parameter); +} + +/// Called by personality handler during phase 2 to get LSDA for current frame. +_LIBUNWIND_EXPORT uintptr_t +_Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) { + uintptr_t result = (uintptr_t)_unw_seh_get_disp_ctx((unw_cursor_t *)context)->HandlerData; + _LIBUNWIND_TRACE_API( + "_Unwind_GetLanguageSpecificData(context=%p) => 0x%" PRIxPTR, + (void *)context, result); + return result; +} + +/// Called by personality handler during phase 2 to find the start of the +/// function. +_LIBUNWIND_EXPORT uintptr_t +_Unwind_GetRegionStart(struct _Unwind_Context *context) { + DISPATCHER_CONTEXT *disp = _unw_seh_get_disp_ctx((unw_cursor_t *)context); + uintptr_t result = (uintptr_t)disp->FunctionEntry->BeginAddress + disp->ImageBase; + _LIBUNWIND_TRACE_API("_Unwind_GetRegionStart(context=%p) => 0x%" PRIxPTR, + (void *)context, result); + return result; +} + +static int +_unw_init_seh(unw_cursor_t *cursor, CONTEXT *context) { +#ifdef _LIBUNWIND_TARGET_X86_64 + new ((void *)cursor) UnwindCursor( + context, LocalAddressSpace::sThisAddressSpace); + auto *co = reinterpret_cast(cursor); + co->setInfoBasedOnIPRegister(); + return UNW_ESUCCESS; +#elif defined(_LIBUNWIND_TARGET_ARM) + new ((void *)cursor) UnwindCursor( + context, LocalAddressSpace::sThisAddressSpace); + auto *co = reinterpret_cast(cursor); + co->setInfoBasedOnIPRegister(); + return UNW_ESUCCESS; +#elif defined(_LIBUNWIND_TARGET_AARCH64) + new ((void *)cursor) UnwindCursor( + context, LocalAddressSpace::sThisAddressSpace); + auto *co = reinterpret_cast(cursor); + co->setInfoBasedOnIPRegister(); + return UNW_ESUCCESS; +#else + return UNW_EINVAL; +#endif +} + +static DISPATCHER_CONTEXT * +_unw_seh_get_disp_ctx(unw_cursor_t *cursor) { +#ifdef _LIBUNWIND_TARGET_X86_64 + return reinterpret_cast *>(cursor)->getDispatcherContext(); +#elif defined(_LIBUNWIND_TARGET_ARM) + return reinterpret_cast *>(cursor)->getDispatcherContext(); +#elif defined(_LIBUNWIND_TARGET_AARCH64) + return reinterpret_cast *>(cursor)->getDispatcherContext(); +#else + return nullptr; +#endif +} + +static void +_unw_seh_set_disp_ctx(unw_cursor_t *cursor, DISPATCHER_CONTEXT *disp) { +#ifdef _LIBUNWIND_TARGET_X86_64 + reinterpret_cast *>(cursor)->setDispatcherContext(disp); +#elif defined(_LIBUNWIND_TARGET_ARM) + reinterpret_cast *>(cursor)->setDispatcherContext(disp); +#elif defined(_LIBUNWIND_TARGET_AARCH64) + reinterpret_cast *>(cursor)->setDispatcherContext(disp); +#endif +} + +#endif // defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) Property changes on: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-seh.cpp ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-sjlj.c =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-sjlj.c (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind-sjlj.c (revision 345026) @@ -1,468 +1,504 @@ //===--------------------------- Unwind-sjlj.c ----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Implements setjump-longjump based C++ exceptions // //===----------------------------------------------------------------------===// #include #include #include #include #include "config.h" -#include "unwind_ext.h" -// -// 32-bit iOS uses setjump/longjump based C++ exceptions. -// Other architectures use "zero cost" exceptions. -// -// With SJLJ based exceptions, any function that has a catch clause or needs to -// do any clean up when an exception propagates through it, needs to call -// _Unwind_SjLj_Register() at the start of the function and -// _Unwind_SjLj_Unregister() at the end. The register function is called with -// the address of a block of memory in the function's stack frame. The runtime -// keeps a linked list (stack) of these blocks - one per thread. The calling -// function also sets the personality and lsda fields of the block. -// +/// With SJLJ based exceptions, any function that has a catch clause or needs to +/// do any clean up when an exception propagates through it, needs to call +/// \c _Unwind_SjLj_Register at the start of the function and +/// \c _Unwind_SjLj_Unregister at the end. The register function is called with +/// the address of a block of memory in the function's stack frame. The runtime +/// keeps a linked list (stack) of these blocks - one per thread. The calling +/// function also sets the personality and lsda fields of the block. -#if _LIBUNWIND_BUILD_SJLJ_APIS +#if defined(_LIBUNWIND_BUILD_SJLJ_APIS) struct _Unwind_FunctionContext { // next function in stack of handlers struct _Unwind_FunctionContext *prev; // set by calling function before registering to be the landing pad - uintptr_t resumeLocation; + uint32_t resumeLocation; // set by personality handler to be parameters passed to landing pad function - uintptr_t resumeParameters[4]; + uint32_t resumeParameters[4]; // set by calling function before registering __personality_routine personality; // arm offset=24 uintptr_t lsda; // arm offset=28 // variable length array, contains registers to restore // 0 = r7, 1 = pc, 2 = sp void *jbuf[]; }; +#if defined(_LIBUNWIND_HAS_NO_THREADS) +# define _LIBUNWIND_THREAD_LOCAL +#else +# if __STDC_VERSION__ >= 201112L +# define _LIBUNWIND_THREAD_LOCAL _Thread_local +# elif defined(_WIN32) +# define _LIBUNWIND_THREAD_LOCAL __declspec(thread) +# elif defined(__GNUC__) || defined(__clang__) +# define _LIBUNWIND_THREAD_LOCAL __thread +# else +# error Unable to create thread local storage +# endif +#endif + +#if !defined(FOR_DYLD) + +#if defined(__APPLE__) +#include +#else +static _LIBUNWIND_THREAD_LOCAL struct _Unwind_FunctionContext *stack = NULL; +#endif + +static struct _Unwind_FunctionContext *__Unwind_SjLj_GetTopOfFunctionStack() { +#if defined(__APPLE__) + return _pthread_getspecific_direct(__PTK_LIBC_DYLD_Unwind_SjLj_Key); +#else + return stack; +#endif +} + +static void +__Unwind_SjLj_SetTopOfFunctionStack(struct _Unwind_FunctionContext *fc) { +#if defined(__APPLE__) + _pthread_setspecific_direct(__PTK_LIBC_DYLD_Unwind_SjLj_Key, fc); +#else + stack = fc; +#endif +} + +#endif + + /// Called at start of each function that catches exceptions _LIBUNWIND_EXPORT void _Unwind_SjLj_Register(struct _Unwind_FunctionContext *fc) { fc->prev = __Unwind_SjLj_GetTopOfFunctionStack(); __Unwind_SjLj_SetTopOfFunctionStack(fc); } /// Called at end of each function that catches exceptions _LIBUNWIND_EXPORT void _Unwind_SjLj_Unregister(struct _Unwind_FunctionContext *fc) { __Unwind_SjLj_SetTopOfFunctionStack(fc->prev); } static _Unwind_Reason_Code unwind_phase1(struct _Unwind_Exception *exception_object) { _Unwind_FunctionContext_t c = __Unwind_SjLj_GetTopOfFunctionStack(); _LIBUNWIND_TRACE_UNWINDING("unwind_phase1: initial function-context=%p", c); // walk each frame looking for a place to stop for (bool handlerNotFound = true; handlerNotFound; c = c->prev) { // check for no more frames if (c == NULL) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): reached " "bottom => _URC_END_OF_STACK", exception_object); return _URC_END_OF_STACK; } _LIBUNWIND_TRACE_UNWINDING("unwind_phase1: function-context=%p", c); // if there is a personality routine, ask it if it will want to stop at this // frame if (c->personality != NULL) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): calling " "personality function %p", exception_object, c->personality); _Unwind_Reason_Code personalityResult = (*c->personality)( 1, _UA_SEARCH_PHASE, exception_object->exception_class, exception_object, (struct _Unwind_Context *)c); switch (personalityResult) { case _URC_HANDLER_FOUND: // found a catch clause or locals that need destructing in this frame // stop search and remember function context handlerNotFound = false; exception_object->private_2 = (uintptr_t) c; _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): " "_URC_HANDLER_FOUND", exception_object); return _URC_NO_REASON; case _URC_CONTINUE_UNWIND: _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): " "_URC_CONTINUE_UNWIND", exception_object); // continue unwinding break; default: // something went wrong _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_FATAL_PHASE1_ERROR", exception_object); return _URC_FATAL_PHASE1_ERROR; } } } return _URC_NO_REASON; } static _Unwind_Reason_Code unwind_phase2(struct _Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p)", exception_object); // walk each frame until we reach where search phase said to stop _Unwind_FunctionContext_t c = __Unwind_SjLj_GetTopOfFunctionStack(); while (true) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2s(ex_ojb=%p): context=%p", exception_object, c); // check for no more frames if (c == NULL) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_step() reached " "bottom => _URC_END_OF_STACK", exception_object); return _URC_END_OF_STACK; } // if there is a personality routine, tell it we are unwinding if (c->personality != NULL) { _Unwind_Action action = _UA_CLEANUP_PHASE; if ((uintptr_t) c == exception_object->private_2) action = (_Unwind_Action)( _UA_CLEANUP_PHASE | _UA_HANDLER_FRAME); // tell personality this was the frame it marked // in phase 1 _Unwind_Reason_Code personalityResult = (*c->personality)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)c); switch (personalityResult) { case _URC_CONTINUE_UNWIND: // continue unwinding _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2(ex_ojb=%p): _URC_CONTINUE_UNWIND", exception_object); if ((uintptr_t) c == exception_object->private_2) { // phase 1 said we would stop at this frame, but we did not... _LIBUNWIND_ABORT("during phase1 personality function said it would " "stop here, but now if phase2 it did not stop here"); } break; case _URC_INSTALL_CONTEXT: _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): " "_URC_INSTALL_CONTEXT, will resume at " "landing pad %p", exception_object, c->jbuf[1]); // personality routine says to transfer control to landing pad // we may get control back if landing pad calls _Unwind_Resume() __Unwind_SjLj_SetTopOfFunctionStack(c); __builtin_longjmp(c->jbuf, 1); // unw_resume() only returns if there was an error return _URC_FATAL_PHASE2_ERROR; default: // something went wrong _LIBUNWIND_DEBUG_LOG("personality function returned unknown result %d", personalityResult); return _URC_FATAL_PHASE2_ERROR; } } c = c->prev; } // clean up phase did not resume at the frame that the search phase said it // would return _URC_FATAL_PHASE2_ERROR; } static _Unwind_Reason_Code unwind_phase2_forced(struct _Unwind_Exception *exception_object, _Unwind_Stop_Fn stop, void *stop_parameter) { // walk each frame until we reach where search phase said to stop _Unwind_FunctionContext_t c = __Unwind_SjLj_GetTopOfFunctionStack(); while (true) { // get next frame (skip over first which is _Unwind_RaiseException) if (c == NULL) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_step() reached " "bottom => _URC_END_OF_STACK", exception_object); return _URC_END_OF_STACK; } // call stop function at each frame _Unwind_Action action = (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE); _Unwind_Reason_Code stopResult = (*stop)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)c, stop_parameter); _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "stop function returned %d", exception_object, stopResult); if (stopResult != _URC_NO_REASON) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "stopped by stop function", exception_object); return _URC_FATAL_PHASE2_ERROR; } // if there is a personality routine, tell it we are unwinding if (c->personality != NULL) { __personality_routine p = (__personality_routine) c->personality; _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "calling personality function %p", exception_object, p); _Unwind_Reason_Code personalityResult = (*p)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)c); switch (personalityResult) { case _URC_CONTINUE_UNWIND: _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned _URC_CONTINUE_UNWIND", exception_object); // destructors called, continue unwinding break; case _URC_INSTALL_CONTEXT: _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned _URC_INSTALL_CONTEXT", exception_object); // we may get control back if landing pad calls _Unwind_Resume() __Unwind_SjLj_SetTopOfFunctionStack(c); __builtin_longjmp(c->jbuf, 1); break; default: // something went wrong _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned %d, " "_URC_FATAL_PHASE2_ERROR", exception_object, personalityResult); return _URC_FATAL_PHASE2_ERROR; } } c = c->prev; } // call stop function one last time and tell it we've reached the end of the // stack _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): calling stop " "function with _UA_END_OF_STACK", exception_object); _Unwind_Action lastAction = (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE | _UA_END_OF_STACK); (*stop)(1, lastAction, exception_object->exception_class, exception_object, (struct _Unwind_Context *)c, stop_parameter); // clean up phase did not resume at the frame that the search phase said it // would return _URC_FATAL_PHASE2_ERROR; } /// Called by __cxa_throw. Only returns if there is a fatal error _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_SjLj_RaiseException(struct _Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_SjLj_RaiseException(ex_obj=%p)", exception_object); // mark that this is a non-forced unwind, so _Unwind_Resume() can do the right // thing exception_object->private_1 = 0; exception_object->private_2 = 0; // phase 1: the search phase _Unwind_Reason_Code phase1 = unwind_phase1(exception_object); if (phase1 != _URC_NO_REASON) return phase1; // phase 2: the clean up phase return unwind_phase2(exception_object); } /// When _Unwind_RaiseException() is in phase2, it hands control /// to the personality function at each frame. The personality /// may force a jump to a landing pad in that function, the landing /// pad code may then call _Unwind_Resume() to continue with the /// unwinding. Note: the call to _Unwind_Resume() is from compiler /// geneated user code. All other _Unwind_* routines are called /// by the C++ runtime __cxa_* routines. /// /// Re-throwing an exception is implemented by having the code call /// __cxa_rethrow() which in turn calls _Unwind_Resume_or_Rethrow() _LIBUNWIND_EXPORT void _Unwind_SjLj_Resume(struct _Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_SjLj_Resume(ex_obj=%p)", exception_object); if (exception_object->private_1 != 0) unwind_phase2_forced(exception_object, (_Unwind_Stop_Fn) exception_object->private_1, (void *)exception_object->private_2); else unwind_phase2(exception_object); // clients assume _Unwind_Resume() does not return, so all we can do is abort. _LIBUNWIND_ABORT("_Unwind_SjLj_Resume() can't return"); } /// Called by __cxa_rethrow(). _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_SjLj_Resume_or_Rethrow(struct _Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("__Unwind_SjLj_Resume_or_Rethrow(ex_obj=%p), " "private_1=%ld", exception_object, exception_object->private_1); // If this is non-forced and a stopping place was found, then this is a // re-throw. // Call _Unwind_RaiseException() as if this was a new exception. if (exception_object->private_1 == 0) { return _Unwind_SjLj_RaiseException(exception_object); // should return if there is no catch clause, so that __cxa_rethrow can call // std::terminate() } // Call through to _Unwind_Resume() which distiguishes between forced and // regular exceptions. _Unwind_SjLj_Resume(exception_object); _LIBUNWIND_ABORT("__Unwind_SjLj_Resume_or_Rethrow() called " "_Unwind_SjLj_Resume() which unexpectedly returned"); } /// Called by personality handler during phase 2 to get LSDA for current frame. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) { _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; _LIBUNWIND_TRACE_API("_Unwind_GetLanguageSpecificData(context=%p) " "=> 0x%0lX", context, ufc->lsda); return ufc->lsda; } /// Called by personality handler during phase 2 to get register values. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetGR(struct _Unwind_Context *context, int index) { _LIBUNWIND_TRACE_API("_Unwind_GetGR(context=%p, reg=%d)", context, index); _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; return ufc->resumeParameters[index]; } /// Called by personality handler during phase 2 to alter register values. _LIBUNWIND_EXPORT void _Unwind_SetGR(struct _Unwind_Context *context, int index, uintptr_t new_value) { _LIBUNWIND_TRACE_API("_Unwind_SetGR(context=%p, reg=%d, value=0x%0lX)" , context, index, new_value); _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; ufc->resumeParameters[index] = new_value; } /// Called by personality handler during phase 2 to get instruction pointer. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) { _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; _LIBUNWIND_TRACE_API("_Unwind_GetIP(context=%p) => 0x%lX", context, ufc->resumeLocation + 1); return ufc->resumeLocation + 1; } /// Called by personality handler during phase 2 to get instruction pointer. /// ipBefore is a boolean that says if IP is already adjusted to be the call /// site address. Normally IP is the return address. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIPInfo(struct _Unwind_Context *context, int *ipBefore) { _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; *ipBefore = 0; _LIBUNWIND_TRACE_API("_Unwind_GetIPInfo(context=%p, %p) => 0x%lX", context, ipBefore, ufc->resumeLocation + 1); return ufc->resumeLocation + 1; } /// Called by personality handler during phase 2 to alter instruction pointer. _LIBUNWIND_EXPORT void _Unwind_SetIP(struct _Unwind_Context *context, uintptr_t new_value) { _LIBUNWIND_TRACE_API("_Unwind_SetIP(context=%p, value=0x%0lX)", context, new_value); _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; ufc->resumeLocation = new_value - 1; } /// Called by personality handler during phase 2 to find the start of the /// function. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetRegionStart(struct _Unwind_Context *context) { // Not supported or needed for sjlj based unwinding (void)context; _LIBUNWIND_TRACE_API("_Unwind_GetRegionStart(context=%p)", context); return 0; } /// Called by personality handler during phase 2 if a foreign exception /// is caught. _LIBUNWIND_EXPORT void _Unwind_DeleteException(struct _Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_DeleteException(ex_obj=%p)", exception_object); if (exception_object->exception_cleanup != NULL) (*exception_object->exception_cleanup)(_URC_FOREIGN_EXCEPTION_CAUGHT, exception_object); } /// Called by personality handler during phase 2 to get base address for data /// relative encodings. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetDataRelBase(struct _Unwind_Context *context) { // Not supported or needed for sjlj based unwinding (void)context; _LIBUNWIND_TRACE_API("_Unwind_GetDataRelBase(context=%p)", context); _LIBUNWIND_ABORT("_Unwind_GetDataRelBase() not implemented"); } /// Called by personality handler during phase 2 to get base address for text /// relative encodings. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetTextRelBase(struct _Unwind_Context *context) { // Not supported or needed for sjlj based unwinding (void)context; _LIBUNWIND_TRACE_API("_Unwind_GetTextRelBase(context=%p)", context); _LIBUNWIND_ABORT("_Unwind_GetTextRelBase() not implemented"); } /// Called by personality handler to get "Call Frame Area" for current frame. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetCFA(struct _Unwind_Context *context) { _LIBUNWIND_TRACE_API("_Unwind_GetCFA(context=%p)", context); if (context != NULL) { _Unwind_FunctionContext_t ufc = (_Unwind_FunctionContext_t) context; // Setjmp/longjmp based exceptions don't have a true CFA. // Instead, the SP in the jmpbuf is the closest approximation. return (uintptr_t) ufc->jbuf[2]; } return 0; } -#endif // _LIBUNWIND_BUILD_SJLJ_APIS +#endif // defined(_LIBUNWIND_BUILD_SJLJ_APIS) Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindCursor.hpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindCursor.hpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindCursor.hpp (revision 345026) @@ -1,1408 +1,1981 @@ //===------------------------- UnwindCursor.hpp ---------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // C++ interface to lower levels of libunwind //===----------------------------------------------------------------------===// #ifndef __UNWINDCURSOR_HPP__ #define __UNWINDCURSOR_HPP__ #include #include #include #include -#include #include +#ifdef _WIN32 + #include + #include +#endif #ifdef __APPLE__ #include #endif +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) +// Provide a definition for the DISPATCHER_CONTEXT struct for old (Win7 and +// earlier) SDKs. +// MinGW-w64 has always provided this struct. + #if defined(_WIN32) && defined(_LIBUNWIND_TARGET_X86_64) && \ + !defined(__MINGW32__) && VER_PRODUCTBUILD < 8000 +struct _DISPATCHER_CONTEXT { + ULONG64 ControlPc; + ULONG64 ImageBase; + PRUNTIME_FUNCTION FunctionEntry; + ULONG64 EstablisherFrame; + ULONG64 TargetIp; + PCONTEXT ContextRecord; + PEXCEPTION_ROUTINE LanguageHandler; + PVOID HandlerData; + PUNWIND_HISTORY_TABLE HistoryTable; + ULONG ScopeIndex; + ULONG Fill0; +}; + #endif + +struct UNWIND_INFO { + uint8_t Version : 3; + uint8_t Flags : 5; + uint8_t SizeOfProlog; + uint8_t CountOfCodes; + uint8_t FrameRegister : 4; + uint8_t FrameOffset : 4; + uint16_t UnwindCodes[2]; +}; + +extern "C" _Unwind_Reason_Code __libunwind_seh_personality( + int, _Unwind_Action, uint64_t, _Unwind_Exception *, + struct _Unwind_Context *); + +#endif + #include "config.h" #include "AddressSpace.hpp" #include "CompactUnwinder.hpp" #include "config.h" #include "DwarfInstructions.hpp" #include "EHHeaderParser.hpp" #include "libunwind.h" #include "Registers.hpp" +#include "RWMutex.hpp" #include "Unwind-EHABI.h" namespace libunwind { -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) /// Cache of recently found FDEs. template class _LIBUNWIND_HIDDEN DwarfFDECache { typedef typename A::pint_t pint_t; public: static pint_t findFDE(pint_t mh, pint_t pc); static void add(pint_t mh, pint_t ip_start, pint_t ip_end, pint_t fde); static void removeAllIn(pint_t mh); static void iterateCacheEntries(void (*func)(unw_word_t ip_start, unw_word_t ip_end, unw_word_t fde, unw_word_t mh)); private: struct entry { pint_t mh; pint_t ip_start; pint_t ip_end; pint_t fde; }; // These fields are all static to avoid needing an initializer. // There is only one instance of this class per process. - static pthread_rwlock_t _lock; + static RWMutex _lock; #ifdef __APPLE__ static void dyldUnloadHook(const struct mach_header *mh, intptr_t slide); static bool _registeredForDyldUnloads; #endif // Can't use std::vector<> here because this code is below libc++. static entry *_buffer; static entry *_bufferUsed; static entry *_bufferEnd; static entry _initialBuffer[64]; }; template typename DwarfFDECache::entry * DwarfFDECache::_buffer = _initialBuffer; template typename DwarfFDECache::entry * DwarfFDECache::_bufferUsed = _initialBuffer; template typename DwarfFDECache::entry * DwarfFDECache::_bufferEnd = &_initialBuffer[64]; template typename DwarfFDECache::entry DwarfFDECache::_initialBuffer[64]; template -pthread_rwlock_t DwarfFDECache::_lock = PTHREAD_RWLOCK_INITIALIZER; +RWMutex DwarfFDECache::_lock; #ifdef __APPLE__ template bool DwarfFDECache::_registeredForDyldUnloads = false; #endif template typename A::pint_t DwarfFDECache::findFDE(pint_t mh, pint_t pc) { pint_t result = 0; - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_rdlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.lock_shared()); for (entry *p = _buffer; p < _bufferUsed; ++p) { if ((mh == p->mh) || (mh == 0)) { if ((p->ip_start <= pc) && (pc < p->ip_end)) { result = p->fde; break; } } } - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_unlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.unlock_shared()); return result; } template void DwarfFDECache::add(pint_t mh, pint_t ip_start, pint_t ip_end, pint_t fde) { #if !defined(_LIBUNWIND_NO_HEAP) - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_wrlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.lock()); if (_bufferUsed >= _bufferEnd) { size_t oldSize = (size_t)(_bufferEnd - _buffer); size_t newSize = oldSize * 4; // Can't use operator new (we are below it). entry *newBuffer = (entry *)malloc(newSize * sizeof(entry)); memcpy(newBuffer, _buffer, oldSize * sizeof(entry)); if (_buffer != _initialBuffer) free(_buffer); _buffer = newBuffer; _bufferUsed = &newBuffer[oldSize]; _bufferEnd = &newBuffer[newSize]; } _bufferUsed->mh = mh; _bufferUsed->ip_start = ip_start; _bufferUsed->ip_end = ip_end; _bufferUsed->fde = fde; ++_bufferUsed; #ifdef __APPLE__ if (!_registeredForDyldUnloads) { _dyld_register_func_for_remove_image(&dyldUnloadHook); _registeredForDyldUnloads = true; } #endif - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_unlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.unlock()); #endif } template void DwarfFDECache::removeAllIn(pint_t mh) { - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_wrlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.lock()); entry *d = _buffer; for (const entry *s = _buffer; s < _bufferUsed; ++s) { if (s->mh != mh) { if (d != s) *d = *s; ++d; } } _bufferUsed = d; - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_unlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.unlock()); } #ifdef __APPLE__ template void DwarfFDECache::dyldUnloadHook(const struct mach_header *mh, intptr_t ) { removeAllIn((pint_t) mh); } #endif template void DwarfFDECache::iterateCacheEntries(void (*func)( unw_word_t ip_start, unw_word_t ip_end, unw_word_t fde, unw_word_t mh)) { - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_wrlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.lock()); for (entry *p = _buffer; p < _bufferUsed; ++p) { (*func)(p->ip_start, p->ip_end, p->fde, p->mh); } - _LIBUNWIND_LOG_NON_ZERO(::pthread_rwlock_unlock(&_lock)); + _LIBUNWIND_LOG_IF_FALSE(_lock.unlock()); } -#endif // _LIBUNWIND_SUPPORT_DWARF_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) #define arrayoffsetof(type, index, field) ((size_t)(&((type *)0)[index].field)) -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) template class UnwindSectionHeader { public: UnwindSectionHeader(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t version() const { return _addressSpace.get32(_addr + offsetof(unwind_info_section_header, version)); } uint32_t commonEncodingsArraySectionOffset() const { return _addressSpace.get32(_addr + offsetof(unwind_info_section_header, commonEncodingsArraySectionOffset)); } uint32_t commonEncodingsArrayCount() const { return _addressSpace.get32(_addr + offsetof(unwind_info_section_header, commonEncodingsArrayCount)); } uint32_t personalityArraySectionOffset() const { return _addressSpace.get32(_addr + offsetof(unwind_info_section_header, personalityArraySectionOffset)); } uint32_t personalityArrayCount() const { return _addressSpace.get32( _addr + offsetof(unwind_info_section_header, personalityArrayCount)); } uint32_t indexSectionOffset() const { return _addressSpace.get32( _addr + offsetof(unwind_info_section_header, indexSectionOffset)); } uint32_t indexCount() const { return _addressSpace.get32( _addr + offsetof(unwind_info_section_header, indexCount)); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionIndexArray { public: UnwindSectionIndexArray(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t functionOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_section_header_index_entry, index, functionOffset)); } uint32_t secondLevelPagesSectionOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_section_header_index_entry, index, secondLevelPagesSectionOffset)); } uint32_t lsdaIndexArraySectionOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_section_header_index_entry, index, lsdaIndexArraySectionOffset)); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionRegularPageHeader { public: UnwindSectionRegularPageHeader(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t kind() const { return _addressSpace.get32( _addr + offsetof(unwind_info_regular_second_level_page_header, kind)); } uint16_t entryPageOffset() const { return _addressSpace.get16( _addr + offsetof(unwind_info_regular_second_level_page_header, entryPageOffset)); } uint16_t entryCount() const { return _addressSpace.get16( _addr + offsetof(unwind_info_regular_second_level_page_header, entryCount)); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionRegularArray { public: UnwindSectionRegularArray(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t functionOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_regular_second_level_entry, index, functionOffset)); } uint32_t encoding(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_regular_second_level_entry, index, encoding)); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionCompressedPageHeader { public: UnwindSectionCompressedPageHeader(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t kind() const { return _addressSpace.get32( _addr + offsetof(unwind_info_compressed_second_level_page_header, kind)); } uint16_t entryPageOffset() const { return _addressSpace.get16( _addr + offsetof(unwind_info_compressed_second_level_page_header, entryPageOffset)); } uint16_t entryCount() const { return _addressSpace.get16( _addr + offsetof(unwind_info_compressed_second_level_page_header, entryCount)); } uint16_t encodingsPageOffset() const { return _addressSpace.get16( _addr + offsetof(unwind_info_compressed_second_level_page_header, encodingsPageOffset)); } uint16_t encodingsCount() const { return _addressSpace.get16( _addr + offsetof(unwind_info_compressed_second_level_page_header, encodingsCount)); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionCompressedArray { public: UnwindSectionCompressedArray(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t functionOffset(uint32_t index) const { return UNWIND_INFO_COMPRESSED_ENTRY_FUNC_OFFSET( _addressSpace.get32(_addr + index * sizeof(uint32_t))); } uint16_t encodingIndex(uint32_t index) const { return UNWIND_INFO_COMPRESSED_ENTRY_ENCODING_INDEX( _addressSpace.get32(_addr + index * sizeof(uint32_t))); } private: A &_addressSpace; typename A::pint_t _addr; }; template class UnwindSectionLsdaArray { public: UnwindSectionLsdaArray(A &addressSpace, typename A::pint_t addr) : _addressSpace(addressSpace), _addr(addr) {} uint32_t functionOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_section_header_lsda_index_entry, index, functionOffset)); } uint32_t lsdaOffset(uint32_t index) const { return _addressSpace.get32( _addr + arrayoffsetof(unwind_info_section_header_lsda_index_entry, index, lsdaOffset)); } private: A &_addressSpace; typename A::pint_t _addr; }; -#endif // _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) class _LIBUNWIND_HIDDEN AbstractUnwindCursor { public: // NOTE: provide a class specific placement deallocation function (S5.3.4 p20) // This avoids an unnecessary dependency to libc++abi. void operator delete(void *, size_t) {} virtual ~AbstractUnwindCursor() {} virtual bool validReg(int) { _LIBUNWIND_ABORT("validReg not implemented"); } virtual unw_word_t getReg(int) { _LIBUNWIND_ABORT("getReg not implemented"); } virtual void setReg(int, unw_word_t) { _LIBUNWIND_ABORT("setReg not implemented"); } virtual bool validFloatReg(int) { _LIBUNWIND_ABORT("validFloatReg not implemented"); } virtual unw_fpreg_t getFloatReg(int) { _LIBUNWIND_ABORT("getFloatReg not implemented"); } virtual void setFloatReg(int, unw_fpreg_t) { _LIBUNWIND_ABORT("setFloatReg not implemented"); } virtual int step() { _LIBUNWIND_ABORT("step not implemented"); } virtual void getInfo(unw_proc_info_t *) { _LIBUNWIND_ABORT("getInfo not implemented"); } virtual void jumpto() { _LIBUNWIND_ABORT("jumpto not implemented"); } virtual bool isSignalFrame() { _LIBUNWIND_ABORT("isSignalFrame not implemented"); } virtual bool getFunctionName(char *, size_t, unw_word_t *) { _LIBUNWIND_ABORT("getFunctionName not implemented"); } virtual void setInfoBasedOnIPRegister(bool = false) { _LIBUNWIND_ABORT("setInfoBasedOnIPRegister not implemented"); } virtual const char *getRegisterName(int) { _LIBUNWIND_ABORT("getRegisterName not implemented"); } #ifdef __arm__ virtual void saveVFPAsX() { _LIBUNWIND_ABORT("saveVFPAsX not implemented"); } #endif }; +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) && defined(_WIN32) + +/// \c UnwindCursor contains all state (including all register values) during +/// an unwind. This is normally stack-allocated inside a unw_cursor_t. +template +class UnwindCursor : public AbstractUnwindCursor { + typedef typename A::pint_t pint_t; +public: + UnwindCursor(unw_context_t *context, A &as); + UnwindCursor(CONTEXT *context, A &as); + UnwindCursor(A &as, void *threadArg); + virtual ~UnwindCursor() {} + virtual bool validReg(int); + virtual unw_word_t getReg(int); + virtual void setReg(int, unw_word_t); + virtual bool validFloatReg(int); + virtual unw_fpreg_t getFloatReg(int); + virtual void setFloatReg(int, unw_fpreg_t); + virtual int step(); + virtual void getInfo(unw_proc_info_t *); + virtual void jumpto(); + virtual bool isSignalFrame(); + virtual bool getFunctionName(char *buf, size_t len, unw_word_t *off); + virtual void setInfoBasedOnIPRegister(bool isReturnAddress = false); + virtual const char *getRegisterName(int num); +#ifdef __arm__ + virtual void saveVFPAsX(); +#endif + + DISPATCHER_CONTEXT *getDispatcherContext() { return &_dispContext; } + void setDispatcherContext(DISPATCHER_CONTEXT *disp) { _dispContext = *disp; } + +private: + + pint_t getLastPC() const { return _dispContext.ControlPc; } + void setLastPC(pint_t pc) { _dispContext.ControlPc = pc; } + RUNTIME_FUNCTION *lookUpSEHUnwindInfo(pint_t pc, pint_t *base) { + _dispContext.FunctionEntry = RtlLookupFunctionEntry(pc, + &_dispContext.ImageBase, + _dispContext.HistoryTable); + *base = _dispContext.ImageBase; + return _dispContext.FunctionEntry; + } + bool getInfoFromSEH(pint_t pc); + int stepWithSEHData() { + _dispContext.LanguageHandler = RtlVirtualUnwind(UNW_FLAG_UHANDLER, + _dispContext.ImageBase, + _dispContext.ControlPc, + _dispContext.FunctionEntry, + _dispContext.ContextRecord, + &_dispContext.HandlerData, + &_dispContext.EstablisherFrame, + NULL); + // Update some fields of the unwind info now, since we have them. + _info.lsda = reinterpret_cast(_dispContext.HandlerData); + if (_dispContext.LanguageHandler) { + _info.handler = reinterpret_cast(__libunwind_seh_personality); + } else + _info.handler = 0; + return UNW_STEP_SUCCESS; + } + + A &_addressSpace; + unw_proc_info_t _info; + DISPATCHER_CONTEXT _dispContext; + CONTEXT _msContext; + UNWIND_HISTORY_TABLE _histTable; + bool _unwindInfoMissing; +}; + + +template +UnwindCursor::UnwindCursor(unw_context_t *context, A &as) + : _addressSpace(as), _unwindInfoMissing(false) { + static_assert((check_fit, unw_cursor_t>::does_fit), + "UnwindCursor<> does not fit in unw_cursor_t"); + memset(&_info, 0, sizeof(_info)); + memset(&_histTable, 0, sizeof(_histTable)); + _dispContext.ContextRecord = &_msContext; + _dispContext.HistoryTable = &_histTable; + // Initialize MS context from ours. + R r(context); + _msContext.ContextFlags = CONTEXT_CONTROL|CONTEXT_INTEGER|CONTEXT_FLOATING_POINT; +#if defined(_LIBUNWIND_TARGET_X86_64) + _msContext.Rax = r.getRegister(UNW_X86_64_RAX); + _msContext.Rcx = r.getRegister(UNW_X86_64_RCX); + _msContext.Rdx = r.getRegister(UNW_X86_64_RDX); + _msContext.Rbx = r.getRegister(UNW_X86_64_RBX); + _msContext.Rsp = r.getRegister(UNW_X86_64_RSP); + _msContext.Rbp = r.getRegister(UNW_X86_64_RBP); + _msContext.Rsi = r.getRegister(UNW_X86_64_RSI); + _msContext.Rdi = r.getRegister(UNW_X86_64_RDI); + _msContext.R8 = r.getRegister(UNW_X86_64_R8); + _msContext.R9 = r.getRegister(UNW_X86_64_R9); + _msContext.R10 = r.getRegister(UNW_X86_64_R10); + _msContext.R11 = r.getRegister(UNW_X86_64_R11); + _msContext.R12 = r.getRegister(UNW_X86_64_R12); + _msContext.R13 = r.getRegister(UNW_X86_64_R13); + _msContext.R14 = r.getRegister(UNW_X86_64_R14); + _msContext.R15 = r.getRegister(UNW_X86_64_R15); + _msContext.Rip = r.getRegister(UNW_REG_IP); + union { + v128 v; + M128A m; + } t; + t.v = r.getVectorRegister(UNW_X86_64_XMM0); + _msContext.Xmm0 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM1); + _msContext.Xmm1 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM2); + _msContext.Xmm2 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM3); + _msContext.Xmm3 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM4); + _msContext.Xmm4 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM5); + _msContext.Xmm5 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM6); + _msContext.Xmm6 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM7); + _msContext.Xmm7 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM8); + _msContext.Xmm8 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM9); + _msContext.Xmm9 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM10); + _msContext.Xmm10 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM11); + _msContext.Xmm11 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM12); + _msContext.Xmm12 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM13); + _msContext.Xmm13 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM14); + _msContext.Xmm14 = t.m; + t.v = r.getVectorRegister(UNW_X86_64_XMM15); + _msContext.Xmm15 = t.m; +#elif defined(_LIBUNWIND_TARGET_ARM) + _msContext.R0 = r.getRegister(UNW_ARM_R0); + _msContext.R1 = r.getRegister(UNW_ARM_R1); + _msContext.R2 = r.getRegister(UNW_ARM_R2); + _msContext.R3 = r.getRegister(UNW_ARM_R3); + _msContext.R4 = r.getRegister(UNW_ARM_R4); + _msContext.R5 = r.getRegister(UNW_ARM_R5); + _msContext.R6 = r.getRegister(UNW_ARM_R6); + _msContext.R7 = r.getRegister(UNW_ARM_R7); + _msContext.R8 = r.getRegister(UNW_ARM_R8); + _msContext.R9 = r.getRegister(UNW_ARM_R9); + _msContext.R10 = r.getRegister(UNW_ARM_R10); + _msContext.R11 = r.getRegister(UNW_ARM_R11); + _msContext.R12 = r.getRegister(UNW_ARM_R12); + _msContext.Sp = r.getRegister(UNW_ARM_SP); + _msContext.Lr = r.getRegister(UNW_ARM_LR); + _msContext.Pc = r.getRegister(UNW_ARM_IP); + for (int i = UNW_ARM_D0; i <= UNW_ARM_D31; ++i) { + union { + uint64_t w; + double d; + } d; + d.d = r.getFloatRegister(i); + _msContext.D[i - UNW_ARM_D0] = d.w; + } +#elif defined(_LIBUNWIND_TARGET_AARCH64) + for (int i = UNW_ARM64_X0; i <= UNW_ARM64_X30; ++i) + _msContext.X[i - UNW_ARM64_X0] = r.getRegister(i); + _msContext.Sp = r.getRegister(UNW_REG_SP); + _msContext.Pc = r.getRegister(UNW_REG_IP); + for (int i = UNW_ARM64_D0; i <= UNW_ARM64_D31; ++i) + _msContext.V[i - UNW_ARM64_D0].D[0] = r.getFloatRegister(i); +#endif +} + +template +UnwindCursor::UnwindCursor(CONTEXT *context, A &as) + : _addressSpace(as), _unwindInfoMissing(false) { + static_assert((check_fit, unw_cursor_t>::does_fit), + "UnwindCursor<> does not fit in unw_cursor_t"); + memset(&_info, 0, sizeof(_info)); + memset(&_histTable, 0, sizeof(_histTable)); + _dispContext.ContextRecord = &_msContext; + _dispContext.HistoryTable = &_histTable; + _msContext = *context; +} + + +template +bool UnwindCursor::validReg(int regNum) { + if (regNum == UNW_REG_IP || regNum == UNW_REG_SP) return true; +#if defined(_LIBUNWIND_TARGET_X86_64) + if (regNum >= UNW_X86_64_RAX && regNum <= UNW_X86_64_R15) return true; +#elif defined(_LIBUNWIND_TARGET_ARM) + if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R15) return true; +#elif defined(_LIBUNWIND_TARGET_AARCH64) + if (regNum >= UNW_ARM64_X0 && regNum <= UNW_ARM64_X30) return true; +#endif + return false; +} + +template +unw_word_t UnwindCursor::getReg(int regNum) { + switch (regNum) { +#if defined(_LIBUNWIND_TARGET_X86_64) + case UNW_REG_IP: return _msContext.Rip; + case UNW_X86_64_RAX: return _msContext.Rax; + case UNW_X86_64_RDX: return _msContext.Rdx; + case UNW_X86_64_RCX: return _msContext.Rcx; + case UNW_X86_64_RBX: return _msContext.Rbx; + case UNW_REG_SP: + case UNW_X86_64_RSP: return _msContext.Rsp; + case UNW_X86_64_RBP: return _msContext.Rbp; + case UNW_X86_64_RSI: return _msContext.Rsi; + case UNW_X86_64_RDI: return _msContext.Rdi; + case UNW_X86_64_R8: return _msContext.R8; + case UNW_X86_64_R9: return _msContext.R9; + case UNW_X86_64_R10: return _msContext.R10; + case UNW_X86_64_R11: return _msContext.R11; + case UNW_X86_64_R12: return _msContext.R12; + case UNW_X86_64_R13: return _msContext.R13; + case UNW_X86_64_R14: return _msContext.R14; + case UNW_X86_64_R15: return _msContext.R15; +#elif defined(_LIBUNWIND_TARGET_ARM) + case UNW_ARM_R0: return _msContext.R0; + case UNW_ARM_R1: return _msContext.R1; + case UNW_ARM_R2: return _msContext.R2; + case UNW_ARM_R3: return _msContext.R3; + case UNW_ARM_R4: return _msContext.R4; + case UNW_ARM_R5: return _msContext.R5; + case UNW_ARM_R6: return _msContext.R6; + case UNW_ARM_R7: return _msContext.R7; + case UNW_ARM_R8: return _msContext.R8; + case UNW_ARM_R9: return _msContext.R9; + case UNW_ARM_R10: return _msContext.R10; + case UNW_ARM_R11: return _msContext.R11; + case UNW_ARM_R12: return _msContext.R12; + case UNW_REG_SP: + case UNW_ARM_SP: return _msContext.Sp; + case UNW_ARM_LR: return _msContext.Lr; + case UNW_REG_IP: + case UNW_ARM_IP: return _msContext.Pc; +#elif defined(_LIBUNWIND_TARGET_AARCH64) + case UNW_REG_SP: return _msContext.Sp; + case UNW_REG_IP: return _msContext.Pc; + default: return _msContext.X[regNum - UNW_ARM64_X0]; +#endif + } + _LIBUNWIND_ABORT("unsupported register"); +} + +template +void UnwindCursor::setReg(int regNum, unw_word_t value) { + switch (regNum) { +#if defined(_LIBUNWIND_TARGET_X86_64) + case UNW_REG_IP: _msContext.Rip = value; break; + case UNW_X86_64_RAX: _msContext.Rax = value; break; + case UNW_X86_64_RDX: _msContext.Rdx = value; break; + case UNW_X86_64_RCX: _msContext.Rcx = value; break; + case UNW_X86_64_RBX: _msContext.Rbx = value; break; + case UNW_REG_SP: + case UNW_X86_64_RSP: _msContext.Rsp = value; break; + case UNW_X86_64_RBP: _msContext.Rbp = value; break; + case UNW_X86_64_RSI: _msContext.Rsi = value; break; + case UNW_X86_64_RDI: _msContext.Rdi = value; break; + case UNW_X86_64_R8: _msContext.R8 = value; break; + case UNW_X86_64_R9: _msContext.R9 = value; break; + case UNW_X86_64_R10: _msContext.R10 = value; break; + case UNW_X86_64_R11: _msContext.R11 = value; break; + case UNW_X86_64_R12: _msContext.R12 = value; break; + case UNW_X86_64_R13: _msContext.R13 = value; break; + case UNW_X86_64_R14: _msContext.R14 = value; break; + case UNW_X86_64_R15: _msContext.R15 = value; break; +#elif defined(_LIBUNWIND_TARGET_ARM) + case UNW_ARM_R0: _msContext.R0 = value; break; + case UNW_ARM_R1: _msContext.R1 = value; break; + case UNW_ARM_R2: _msContext.R2 = value; break; + case UNW_ARM_R3: _msContext.R3 = value; break; + case UNW_ARM_R4: _msContext.R4 = value; break; + case UNW_ARM_R5: _msContext.R5 = value; break; + case UNW_ARM_R6: _msContext.R6 = value; break; + case UNW_ARM_R7: _msContext.R7 = value; break; + case UNW_ARM_R8: _msContext.R8 = value; break; + case UNW_ARM_R9: _msContext.R9 = value; break; + case UNW_ARM_R10: _msContext.R10 = value; break; + case UNW_ARM_R11: _msContext.R11 = value; break; + case UNW_ARM_R12: _msContext.R12 = value; break; + case UNW_REG_SP: + case UNW_ARM_SP: _msContext.Sp = value; break; + case UNW_ARM_LR: _msContext.Lr = value; break; + case UNW_REG_IP: + case UNW_ARM_IP: _msContext.Pc = value; break; +#elif defined(_LIBUNWIND_TARGET_AARCH64) + case UNW_REG_SP: _msContext.Sp = value; break; + case UNW_REG_IP: _msContext.Pc = value; break; + case UNW_ARM64_X0: + case UNW_ARM64_X1: + case UNW_ARM64_X2: + case UNW_ARM64_X3: + case UNW_ARM64_X4: + case UNW_ARM64_X5: + case UNW_ARM64_X6: + case UNW_ARM64_X7: + case UNW_ARM64_X8: + case UNW_ARM64_X9: + case UNW_ARM64_X10: + case UNW_ARM64_X11: + case UNW_ARM64_X12: + case UNW_ARM64_X13: + case UNW_ARM64_X14: + case UNW_ARM64_X15: + case UNW_ARM64_X16: + case UNW_ARM64_X17: + case UNW_ARM64_X18: + case UNW_ARM64_X19: + case UNW_ARM64_X20: + case UNW_ARM64_X21: + case UNW_ARM64_X22: + case UNW_ARM64_X23: + case UNW_ARM64_X24: + case UNW_ARM64_X25: + case UNW_ARM64_X26: + case UNW_ARM64_X27: + case UNW_ARM64_X28: + case UNW_ARM64_FP: + case UNW_ARM64_LR: _msContext.X[regNum - UNW_ARM64_X0] = value; break; +#endif + default: + _LIBUNWIND_ABORT("unsupported register"); + } +} + +template +bool UnwindCursor::validFloatReg(int regNum) { +#if defined(_LIBUNWIND_TARGET_ARM) + if (regNum >= UNW_ARM_S0 && regNum <= UNW_ARM_S31) return true; + if (regNum >= UNW_ARM_D0 && regNum <= UNW_ARM_D31) return true; +#elif defined(_LIBUNWIND_TARGET_AARCH64) + if (regNum >= UNW_ARM64_D0 && regNum <= UNW_ARM64_D31) return true; +#endif + return false; +} + +template +unw_fpreg_t UnwindCursor::getFloatReg(int regNum) { +#if defined(_LIBUNWIND_TARGET_ARM) + if (regNum >= UNW_ARM_S0 && regNum <= UNW_ARM_S31) { + union { + uint32_t w; + float f; + } d; + d.w = _msContext.S[regNum - UNW_ARM_S0]; + return d.f; + } + if (regNum >= UNW_ARM_D0 && regNum <= UNW_ARM_D31) { + union { + uint64_t w; + double d; + } d; + d.w = _msContext.D[regNum - UNW_ARM_D0]; + return d.d; + } + _LIBUNWIND_ABORT("unsupported float register"); +#elif defined(_LIBUNWIND_TARGET_AARCH64) + return _msContext.V[regNum - UNW_ARM64_D0].D[0]; +#else + _LIBUNWIND_ABORT("float registers unimplemented"); +#endif +} + +template +void UnwindCursor::setFloatReg(int regNum, unw_fpreg_t value) { +#if defined(_LIBUNWIND_TARGET_ARM) + if (regNum >= UNW_ARM_S0 && regNum <= UNW_ARM_S31) { + union { + uint32_t w; + float f; + } d; + d.f = value; + _msContext.S[regNum - UNW_ARM_S0] = d.w; + } + if (regNum >= UNW_ARM_D0 && regNum <= UNW_ARM_D31) { + union { + uint64_t w; + double d; + } d; + d.d = value; + _msContext.D[regNum - UNW_ARM_D0] = d.w; + } + _LIBUNWIND_ABORT("unsupported float register"); +#elif defined(_LIBUNWIND_TARGET_AARCH64) + _msContext.V[regNum - UNW_ARM64_D0].D[0] = value; +#else + _LIBUNWIND_ABORT("float registers unimplemented"); +#endif +} + +template void UnwindCursor::jumpto() { + RtlRestoreContext(&_msContext, nullptr); +} + +#ifdef __arm__ +template void UnwindCursor::saveVFPAsX() {} +#endif + +template +const char *UnwindCursor::getRegisterName(int regNum) { + return R::getRegisterName(regNum); +} + +template bool UnwindCursor::isSignalFrame() { + return false; +} + +#else // !defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) || !defined(_WIN32) + /// UnwindCursor contains all state (including all register values) during /// an unwind. This is normally stack allocated inside a unw_cursor_t. template class UnwindCursor : public AbstractUnwindCursor{ typedef typename A::pint_t pint_t; public: UnwindCursor(unw_context_t *context, A &as); UnwindCursor(A &as, void *threadArg); virtual ~UnwindCursor() {} virtual bool validReg(int); virtual unw_word_t getReg(int); virtual void setReg(int, unw_word_t); virtual bool validFloatReg(int); virtual unw_fpreg_t getFloatReg(int); virtual void setFloatReg(int, unw_fpreg_t); virtual int step(); virtual void getInfo(unw_proc_info_t *); virtual void jumpto(); virtual bool isSignalFrame(); virtual bool getFunctionName(char *buf, size_t len, unw_word_t *off); virtual void setInfoBasedOnIPRegister(bool isReturnAddress = false); virtual const char *getRegisterName(int num); #ifdef __arm__ virtual void saveVFPAsX(); #endif private: -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) bool getInfoFromEHABISection(pint_t pc, const UnwindInfoSections §s); int stepWithEHABI() { size_t len = 0; size_t off = 0; // FIXME: Calling decode_eht_entry() here is violating the libunwind // abstraction layer. const uint32_t *ehtp = decode_eht_entry(reinterpret_cast(_info.unwind_info), &off, &len); if (_Unwind_VRS_Interpret((_Unwind_Context *)this, ehtp, off, len) != _URC_CONTINUE_UNWIND) return UNW_STEP_END; return UNW_STEP_SUCCESS; } #endif -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) bool getInfoFromDwarfSection(pint_t pc, const UnwindInfoSections §s, uint32_t fdeSectionOffsetHint=0); int stepWithDwarfFDE() { return DwarfInstructions::stepWithDwarf(_addressSpace, (pint_t)this->getReg(UNW_REG_IP), (pint_t)_info.unwind_info, _registers); } #endif -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) bool getInfoFromCompactEncodingSection(pint_t pc, const UnwindInfoSections §s); int stepWithCompactEncoding() { - #if _LIBUNWIND_SUPPORT_DWARF_UNWIND + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) if ( compactSaysUseDwarf() ) return stepWithDwarfFDE(); #endif R dummy; return stepWithCompactEncoding(dummy); } #if defined(_LIBUNWIND_TARGET_X86_64) int stepWithCompactEncoding(Registers_x86_64 &) { return CompactUnwinder_x86_64::stepWithCompactEncoding( _info.format, _info.start_ip, _addressSpace, _registers); } #endif #if defined(_LIBUNWIND_TARGET_I386) int stepWithCompactEncoding(Registers_x86 &) { return CompactUnwinder_x86::stepWithCompactEncoding( _info.format, (uint32_t)_info.start_ip, _addressSpace, _registers); } #endif #if defined(_LIBUNWIND_TARGET_PPC) int stepWithCompactEncoding(Registers_ppc &) { return UNW_EINVAL; } #endif +#if defined(_LIBUNWIND_TARGET_PPC64) + int stepWithCompactEncoding(Registers_ppc64 &) { + return UNW_EINVAL; + } +#endif + + #if defined(_LIBUNWIND_TARGET_AARCH64) int stepWithCompactEncoding(Registers_arm64 &) { return CompactUnwinder_arm64::stepWithCompactEncoding( _info.format, _info.start_ip, _addressSpace, _registers); } #endif #if defined(_LIBUNWIND_TARGET_MIPS_O32) int stepWithCompactEncoding(Registers_mips_o32 &) { return UNW_EINVAL; } #endif #if defined(_LIBUNWIND_TARGET_MIPS_NEWABI) int stepWithCompactEncoding(Registers_mips_newabi &) { return UNW_EINVAL; } #endif +#if defined(_LIBUNWIND_TARGET_SPARC) + int stepWithCompactEncoding(Registers_sparc &) { return UNW_EINVAL; } +#endif + bool compactSaysUseDwarf(uint32_t *offset=NULL) const { R dummy; return compactSaysUseDwarf(dummy, offset); } #if defined(_LIBUNWIND_TARGET_X86_64) bool compactSaysUseDwarf(Registers_x86_64 &, uint32_t *offset) const { if ((_info.format & UNWIND_X86_64_MODE_MASK) == UNWIND_X86_64_MODE_DWARF) { if (offset) *offset = (_info.format & UNWIND_X86_64_DWARF_SECTION_OFFSET); return true; } return false; } #endif #if defined(_LIBUNWIND_TARGET_I386) bool compactSaysUseDwarf(Registers_x86 &, uint32_t *offset) const { if ((_info.format & UNWIND_X86_MODE_MASK) == UNWIND_X86_MODE_DWARF) { if (offset) *offset = (_info.format & UNWIND_X86_DWARF_SECTION_OFFSET); return true; } return false; } #endif #if defined(_LIBUNWIND_TARGET_PPC) bool compactSaysUseDwarf(Registers_ppc &, uint32_t *) const { return true; } #endif +#if defined(_LIBUNWIND_TARGET_PPC64) + bool compactSaysUseDwarf(Registers_ppc64 &, uint32_t *) const { + return true; + } +#endif + #if defined(_LIBUNWIND_TARGET_AARCH64) bool compactSaysUseDwarf(Registers_arm64 &, uint32_t *offset) const { if ((_info.format & UNWIND_ARM64_MODE_MASK) == UNWIND_ARM64_MODE_DWARF) { if (offset) *offset = (_info.format & UNWIND_ARM64_DWARF_SECTION_OFFSET); return true; } return false; } #endif #if defined(_LIBUNWIND_TARGET_MIPS_O32) bool compactSaysUseDwarf(Registers_mips_o32 &, uint32_t *) const { return true; } #endif #if defined(_LIBUNWIND_TARGET_MIPS_NEWABI) bool compactSaysUseDwarf(Registers_mips_newabi &, uint32_t *) const { return true; } #endif -#endif // _LIBUNWIND_SUPPORT_COMPACT_UNWIND -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_TARGET_SPARC) + bool compactSaysUseDwarf(Registers_sparc &, uint32_t *) const { return true; } +#endif + +#endif // defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) + +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) compact_unwind_encoding_t dwarfEncoding() const { R dummy; return dwarfEncoding(dummy); } #if defined(_LIBUNWIND_TARGET_X86_64) compact_unwind_encoding_t dwarfEncoding(Registers_x86_64 &) const { return UNWIND_X86_64_MODE_DWARF; } #endif #if defined(_LIBUNWIND_TARGET_I386) compact_unwind_encoding_t dwarfEncoding(Registers_x86 &) const { return UNWIND_X86_MODE_DWARF; } #endif #if defined(_LIBUNWIND_TARGET_PPC) compact_unwind_encoding_t dwarfEncoding(Registers_ppc &) const { return 0; } #endif +#if defined(_LIBUNWIND_TARGET_PPC64) + compact_unwind_encoding_t dwarfEncoding(Registers_ppc64 &) const { + return 0; + } +#endif + #if defined(_LIBUNWIND_TARGET_AARCH64) compact_unwind_encoding_t dwarfEncoding(Registers_arm64 &) const { return UNWIND_ARM64_MODE_DWARF; } #endif +#if defined(_LIBUNWIND_TARGET_ARM) + compact_unwind_encoding_t dwarfEncoding(Registers_arm &) const { + return 0; + } +#endif + #if defined (_LIBUNWIND_TARGET_OR1K) compact_unwind_encoding_t dwarfEncoding(Registers_or1k &) const { return 0; } #endif #if defined (_LIBUNWIND_TARGET_RISCV) compact_unwind_encoding_t dwarfEncoding(Registers_riscv &) const { return 0; } #endif #if defined (_LIBUNWIND_TARGET_MIPS_O32) compact_unwind_encoding_t dwarfEncoding(Registers_mips_o32 &) const { return 0; } #endif #if defined (_LIBUNWIND_TARGET_MIPS_NEWABI) compact_unwind_encoding_t dwarfEncoding(Registers_mips_newabi &) const { return 0; } #endif -#endif // _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_TARGET_SPARC) + compact_unwind_encoding_t dwarfEncoding(Registers_sparc &) const { return 0; } +#endif +#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) + +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + // For runtime environments using SEH unwind data without Windows runtime + // support. + pint_t getLastPC() const { /* FIXME: Implement */ return 0; } + void setLastPC(pint_t pc) { /* FIXME: Implement */ } + RUNTIME_FUNCTION *lookUpSEHUnwindInfo(pint_t pc, pint_t *base) { + /* FIXME: Implement */ + *base = 0; + return nullptr; + } + bool getInfoFromSEH(pint_t pc); + int stepWithSEHData() { /* FIXME: Implement */ return 0; } +#endif // defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + + A &_addressSpace; R _registers; unw_proc_info_t _info; bool _unwindInfoMissing; bool _isSignalFrame; }; template UnwindCursor::UnwindCursor(unw_context_t *context, A &as) : _addressSpace(as), _registers(context), _unwindInfoMissing(false), _isSignalFrame(false) { static_assert((check_fit, unw_cursor_t>::does_fit), "UnwindCursor<> does not fit in unw_cursor_t"); memset(&_info, 0, sizeof(_info)); } template UnwindCursor::UnwindCursor(A &as, void *) : _addressSpace(as), _unwindInfoMissing(false), _isSignalFrame(false) { memset(&_info, 0, sizeof(_info)); // FIXME // fill in _registers from thread arg } template bool UnwindCursor::validReg(int regNum) { return _registers.validRegister(regNum); } template unw_word_t UnwindCursor::getReg(int regNum) { return _registers.getRegister(regNum); } template void UnwindCursor::setReg(int regNum, unw_word_t value) { _registers.setRegister(regNum, (typename A::pint_t)value); } template bool UnwindCursor::validFloatReg(int regNum) { return _registers.validFloatRegister(regNum); } template unw_fpreg_t UnwindCursor::getFloatReg(int regNum) { return _registers.getFloatRegister(regNum); } template void UnwindCursor::setFloatReg(int regNum, unw_fpreg_t value) { _registers.setFloatRegister(regNum, value); } template void UnwindCursor::jumpto() { _registers.jumpto(); } #ifdef __arm__ template void UnwindCursor::saveVFPAsX() { _registers.saveVFPAsX(); } #endif template const char *UnwindCursor::getRegisterName(int regNum) { return _registers.getRegisterName(regNum); } template bool UnwindCursor::isSignalFrame() { return _isSignalFrame; } -#if _LIBUNWIND_ARM_EHABI +#endif // defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + +#if defined(_LIBUNWIND_ARM_EHABI) struct EHABIIndexEntry { uint32_t functionOffset; uint32_t data; }; template struct EHABISectionIterator { typedef EHABISectionIterator _Self; typedef std::random_access_iterator_tag iterator_category; typedef typename A::pint_t value_type; typedef typename A::pint_t* pointer; typedef typename A::pint_t& reference; typedef size_t size_type; typedef size_t difference_type; static _Self begin(A& addressSpace, const UnwindInfoSections& sects) { return _Self(addressSpace, sects, 0); } static _Self end(A& addressSpace, const UnwindInfoSections& sects) { - return _Self(addressSpace, sects, sects.arm_section_length); + return _Self(addressSpace, sects, + sects.arm_section_length / sizeof(EHABIIndexEntry)); } EHABISectionIterator(A& addressSpace, const UnwindInfoSections& sects, size_t i) : _i(i), _addressSpace(&addressSpace), _sects(§s) {} _Self& operator++() { ++_i; return *this; } _Self& operator+=(size_t a) { _i += a; return *this; } _Self& operator--() { assert(_i > 0); --_i; return *this; } _Self& operator-=(size_t a) { assert(_i >= a); _i -= a; return *this; } _Self operator+(size_t a) { _Self out = *this; out._i += a; return out; } _Self operator-(size_t a) { assert(_i >= a); _Self out = *this; out._i -= a; return out; } size_t operator-(const _Self& other) { return _i - other._i; } bool operator==(const _Self& other) const { assert(_addressSpace == other._addressSpace); assert(_sects == other._sects); return _i == other._i; } typename A::pint_t operator*() const { return functionAddress(); } typename A::pint_t functionAddress() const { typename A::pint_t indexAddr = _sects->arm_section + arrayoffsetof( EHABIIndexEntry, _i, functionOffset); return indexAddr + signExtendPrel31(_addressSpace->get32(indexAddr)); } typename A::pint_t dataAddress() { typename A::pint_t indexAddr = _sects->arm_section + arrayoffsetof( EHABIIndexEntry, _i, data); return indexAddr; } private: size_t _i; A* _addressSpace; const UnwindInfoSections* _sects; }; template bool UnwindCursor::getInfoFromEHABISection( pint_t pc, const UnwindInfoSections §s) { EHABISectionIterator begin = EHABISectionIterator::begin(_addressSpace, sects); EHABISectionIterator end = EHABISectionIterator::end(_addressSpace, sects); + if (begin == end) + return false; EHABISectionIterator itNextPC = std::upper_bound(begin, end, pc); - if (itNextPC == begin || itNextPC == end) + if (itNextPC == begin) return false; EHABISectionIterator itThisPC = itNextPC - 1; pint_t thisPC = itThisPC.functionAddress(); - pint_t nextPC = itNextPC.functionAddress(); + // If an exception is thrown from a function, corresponding to the last entry + // in the table, we don't really know the function extent and have to choose a + // value for nextPC. Choosing max() will allow the range check during trace to + // succeed. + pint_t nextPC = (itNextPC == end) ? std::numeric_limits::max() + : itNextPC.functionAddress(); pint_t indexDataAddr = itThisPC.dataAddress(); if (indexDataAddr == 0) return false; uint32_t indexData = _addressSpace.get32(indexDataAddr); if (indexData == UNW_EXIDX_CANTUNWIND) return false; // If the high bit is set, the exception handling table entry is inline inside // the index table entry on the second word (aka |indexDataAddr|). Otherwise, // the table points at an offset in the exception handling table (section 5 EHABI). pint_t exceptionTableAddr; uint32_t exceptionTableData; bool isSingleWordEHT; if (indexData & 0x80000000) { exceptionTableAddr = indexDataAddr; // TODO(ajwong): Should this data be 0? exceptionTableData = indexData; isSingleWordEHT = true; } else { exceptionTableAddr = indexDataAddr + signExtendPrel31(indexData); exceptionTableData = _addressSpace.get32(exceptionTableAddr); isSingleWordEHT = false; } // Now we know the 3 things: // exceptionTableAddr -- exception handler table entry. // exceptionTableData -- the data inside the first word of the eht entry. // isSingleWordEHT -- whether the entry is in the index. unw_word_t personalityRoutine = 0xbadf00d; bool scope32 = false; uintptr_t lsda; // If the high bit in the exception handling table entry is set, the entry is // in compact form (section 6.3 EHABI). if (exceptionTableData & 0x80000000) { // Grab the index of the personality routine from the compact form. uint32_t choice = (exceptionTableData & 0x0f000000) >> 24; uint32_t extraWords = 0; switch (choice) { case 0: personalityRoutine = (unw_word_t) &__aeabi_unwind_cpp_pr0; extraWords = 0; scope32 = false; lsda = isSingleWordEHT ? 0 : (exceptionTableAddr + 4); break; case 1: personalityRoutine = (unw_word_t) &__aeabi_unwind_cpp_pr1; extraWords = (exceptionTableData & 0x00ff0000) >> 16; scope32 = false; lsda = exceptionTableAddr + (extraWords + 1) * 4; break; case 2: personalityRoutine = (unw_word_t) &__aeabi_unwind_cpp_pr2; extraWords = (exceptionTableData & 0x00ff0000) >> 16; scope32 = true; lsda = exceptionTableAddr + (extraWords + 1) * 4; break; default: _LIBUNWIND_ABORT("unknown personality routine"); return false; } if (isSingleWordEHT) { if (extraWords != 0) { _LIBUNWIND_ABORT("index inlined table detected but pr function " "requires extra words"); return false; } } } else { pint_t personalityAddr = exceptionTableAddr + signExtendPrel31(exceptionTableData); personalityRoutine = personalityAddr; // ARM EHABI # 6.2, # 9.2 // // +---- ehtp // v // +--------------------------------------+ // | +--------+--------+--------+-------+ | // | |0| prel31 to personalityRoutine | | // | +--------+--------+--------+-------+ | // | | N | unwind opcodes | | <-- UnwindData // | +--------+--------+--------+-------+ | // | | Word 2 unwind opcodes | | // | +--------+--------+--------+-------+ | // | ... | // | +--------+--------+--------+-------+ | // | | Word N unwind opcodes | | // | +--------+--------+--------+-------+ | // | | LSDA | | <-- lsda // | | ... | | // | +--------+--------+--------+-------+ | // +--------------------------------------+ uint32_t *UnwindData = reinterpret_cast(exceptionTableAddr) + 1; uint32_t FirstDataWord = *UnwindData; size_t N = ((FirstDataWord >> 24) & 0xff); size_t NDataWords = N + 1; lsda = reinterpret_cast(UnwindData + NDataWords); } _info.start_ip = thisPC; _info.end_ip = nextPC; _info.handler = personalityRoutine; _info.unwind_info = exceptionTableAddr; _info.lsda = lsda; // flags is pr_cache.additional. See EHABI #7.2 for definition of bit 0. _info.flags = isSingleWordEHT ? 1 : 0 | scope32 ? 0x2 : 0; // Use enum? return true; } #endif -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) template bool UnwindCursor::getInfoFromDwarfSection(pint_t pc, const UnwindInfoSections §s, uint32_t fdeSectionOffsetHint) { typename CFI_Parser::FDE_Info fdeInfo; typename CFI_Parser::CIE_Info cieInfo; bool foundFDE = false; bool foundInCache = false; // If compact encoding table gave offset into dwarf section, go directly there if (fdeSectionOffsetHint != 0) { foundFDE = CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, (uint32_t)sects.dwarf_section_length, sects.dwarf_section + fdeSectionOffsetHint, &fdeInfo, &cieInfo); } -#if _LIBUNWIND_SUPPORT_DWARF_INDEX +#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) if (!foundFDE && (sects.dwarf_index_section != 0)) { foundFDE = EHHeaderParser::findFDE( _addressSpace, pc, sects.dwarf_index_section, (uint32_t)sects.dwarf_index_section_length, &fdeInfo, &cieInfo); } #endif if (!foundFDE) { // otherwise, search cache of previously found FDEs. pint_t cachedFDE = DwarfFDECache::findFDE(sects.dso_base, pc); if (cachedFDE != 0) { foundFDE = CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, (uint32_t)sects.dwarf_section_length, cachedFDE, &fdeInfo, &cieInfo); foundInCache = foundFDE; } } if (!foundFDE) { // Still not found, do full scan of __eh_frame section. foundFDE = CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, (uint32_t)sects.dwarf_section_length, 0, &fdeInfo, &cieInfo); } if (foundFDE) { typename CFI_Parser::PrologInfo prolog; if (CFI_Parser::parseFDEInstructions(_addressSpace, fdeInfo, cieInfo, pc, - &prolog)) { + R::getArch(), &prolog)) { // Save off parsed FDE info _info.start_ip = fdeInfo.pcStart; _info.end_ip = fdeInfo.pcEnd; _info.lsda = fdeInfo.lsda; _info.handler = cieInfo.personality; _info.gp = prolog.spExtraArgSize; _info.flags = 0; _info.format = dwarfEncoding(); _info.unwind_info = fdeInfo.fdeStart; _info.unwind_info_size = (uint32_t)fdeInfo.fdeLength; _info.extra = (unw_word_t) sects.dso_base; // Add to cache (to make next lookup faster) if we had no hint // and there was no index. if (!foundInCache && (fdeSectionOffsetHint == 0)) { - #if _LIBUNWIND_SUPPORT_DWARF_INDEX + #if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) if (sects.dwarf_index_section == 0) #endif DwarfFDECache::add(sects.dso_base, fdeInfo.pcStart, fdeInfo.pcEnd, fdeInfo.fdeStart); } return true; } } //_LIBUNWIND_DEBUG_LOG("can't find/use FDE for pc=0x%llX", (uint64_t)pc); return false; } -#endif // _LIBUNWIND_SUPPORT_DWARF_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) template bool UnwindCursor::getInfoFromCompactEncodingSection(pint_t pc, const UnwindInfoSections §s) { const bool log = false; if (log) fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX, mh=0x%llX)\n", (uint64_t)pc, (uint64_t)sects.dso_base); const UnwindSectionHeader sectionHeader(_addressSpace, sects.compact_unwind_section); if (sectionHeader.version() != UNWIND_SECTION_VERSION) return false; // do a binary search of top level index to find page with unwind info pint_t targetFunctionOffset = pc - sects.dso_base; const UnwindSectionIndexArray topIndex(_addressSpace, sects.compact_unwind_section + sectionHeader.indexSectionOffset()); uint32_t low = 0; uint32_t high = sectionHeader.indexCount(); uint32_t last = high - 1; while (low < high) { uint32_t mid = (low + high) / 2; //if ( log ) fprintf(stderr, "\tmid=%d, low=%d, high=%d, *mid=0x%08X\n", //mid, low, high, topIndex.functionOffset(mid)); if (topIndex.functionOffset(mid) <= targetFunctionOffset) { if ((mid == last) || (topIndex.functionOffset(mid + 1) > targetFunctionOffset)) { low = mid; break; } else { low = mid + 1; } } else { high = mid; } } const uint32_t firstLevelFunctionOffset = topIndex.functionOffset(low); const uint32_t firstLevelNextPageFunctionOffset = topIndex.functionOffset(low + 1); const pint_t secondLevelAddr = sects.compact_unwind_section + topIndex.secondLevelPagesSectionOffset(low); const pint_t lsdaArrayStartAddr = sects.compact_unwind_section + topIndex.lsdaIndexArraySectionOffset(low); const pint_t lsdaArrayEndAddr = sects.compact_unwind_section + topIndex.lsdaIndexArraySectionOffset(low+1); if (log) fprintf(stderr, "\tfirst level search for result index=%d " "to secondLevelAddr=0x%llX\n", low, (uint64_t) secondLevelAddr); // do a binary search of second level page index uint32_t encoding = 0; pint_t funcStart = 0; pint_t funcEnd = 0; pint_t lsda = 0; pint_t personality = 0; uint32_t pageKind = _addressSpace.get32(secondLevelAddr); if (pageKind == UNWIND_SECOND_LEVEL_REGULAR) { // regular page UnwindSectionRegularPageHeader pageHeader(_addressSpace, secondLevelAddr); UnwindSectionRegularArray pageIndex( _addressSpace, secondLevelAddr + pageHeader.entryPageOffset()); // binary search looks for entry with e where index[e].offset <= pc < // index[e+1].offset if (log) fprintf(stderr, "\tbinary search for targetFunctionOffset=0x%08llX in " "regular page starting at secondLevelAddr=0x%llX\n", (uint64_t) targetFunctionOffset, (uint64_t) secondLevelAddr); low = 0; high = pageHeader.entryCount(); while (low < high) { uint32_t mid = (low + high) / 2; if (pageIndex.functionOffset(mid) <= targetFunctionOffset) { if (mid == (uint32_t)(pageHeader.entryCount() - 1)) { // at end of table low = mid; funcEnd = firstLevelNextPageFunctionOffset + sects.dso_base; break; } else if (pageIndex.functionOffset(mid + 1) > targetFunctionOffset) { // next is too big, so we found it low = mid; funcEnd = pageIndex.functionOffset(low + 1) + sects.dso_base; break; } else { low = mid + 1; } } else { high = mid; } } encoding = pageIndex.encoding(low); funcStart = pageIndex.functionOffset(low) + sects.dso_base; if (pc < funcStart) { if (log) fprintf( stderr, "\tpc not in table, pc=0x%llX, funcStart=0x%llX, funcEnd=0x%llX\n", (uint64_t) pc, (uint64_t) funcStart, (uint64_t) funcEnd); return false; } if (pc > funcEnd) { if (log) fprintf( stderr, "\tpc not in table, pc=0x%llX, funcStart=0x%llX, funcEnd=0x%llX\n", (uint64_t) pc, (uint64_t) funcStart, (uint64_t) funcEnd); return false; } } else if (pageKind == UNWIND_SECOND_LEVEL_COMPRESSED) { // compressed page UnwindSectionCompressedPageHeader pageHeader(_addressSpace, secondLevelAddr); UnwindSectionCompressedArray pageIndex( _addressSpace, secondLevelAddr + pageHeader.entryPageOffset()); const uint32_t targetFunctionPageOffset = (uint32_t)(targetFunctionOffset - firstLevelFunctionOffset); // binary search looks for entry with e where index[e].offset <= pc < // index[e+1].offset if (log) fprintf(stderr, "\tbinary search of compressed page starting at " "secondLevelAddr=0x%llX\n", (uint64_t) secondLevelAddr); low = 0; last = pageHeader.entryCount() - 1; high = pageHeader.entryCount(); while (low < high) { uint32_t mid = (low + high) / 2; if (pageIndex.functionOffset(mid) <= targetFunctionPageOffset) { if ((mid == last) || (pageIndex.functionOffset(mid + 1) > targetFunctionPageOffset)) { low = mid; break; } else { low = mid + 1; } } else { high = mid; } } funcStart = pageIndex.functionOffset(low) + firstLevelFunctionOffset + sects.dso_base; if (low < last) funcEnd = pageIndex.functionOffset(low + 1) + firstLevelFunctionOffset + sects.dso_base; else funcEnd = firstLevelNextPageFunctionOffset + sects.dso_base; if (pc < funcStart) { _LIBUNWIND_DEBUG_LOG("malformed __unwind_info, pc=0x%llX not in second " "level compressed unwind table. funcStart=0x%llX", (uint64_t) pc, (uint64_t) funcStart); return false; } if (pc > funcEnd) { _LIBUNWIND_DEBUG_LOG("malformed __unwind_info, pc=0x%llX not in second " "level compressed unwind table. funcEnd=0x%llX", (uint64_t) pc, (uint64_t) funcEnd); return false; } uint16_t encodingIndex = pageIndex.encodingIndex(low); if (encodingIndex < sectionHeader.commonEncodingsArrayCount()) { // encoding is in common table in section header encoding = _addressSpace.get32( sects.compact_unwind_section + sectionHeader.commonEncodingsArraySectionOffset() + encodingIndex * sizeof(uint32_t)); } else { // encoding is in page specific table uint16_t pageEncodingIndex = encodingIndex - (uint16_t)sectionHeader.commonEncodingsArrayCount(); encoding = _addressSpace.get32(secondLevelAddr + pageHeader.encodingsPageOffset() + pageEncodingIndex * sizeof(uint32_t)); } } else { _LIBUNWIND_DEBUG_LOG("malformed __unwind_info at 0x%0llX bad second " "level page", (uint64_t) sects.compact_unwind_section); return false; } // look up LSDA, if encoding says function has one if (encoding & UNWIND_HAS_LSDA) { UnwindSectionLsdaArray lsdaIndex(_addressSpace, lsdaArrayStartAddr); uint32_t funcStartOffset = (uint32_t)(funcStart - sects.dso_base); low = 0; high = (uint32_t)(lsdaArrayEndAddr - lsdaArrayStartAddr) / sizeof(unwind_info_section_header_lsda_index_entry); // binary search looks for entry with exact match for functionOffset if (log) fprintf(stderr, "\tbinary search of lsda table for targetFunctionOffset=0x%08X\n", funcStartOffset); while (low < high) { uint32_t mid = (low + high) / 2; if (lsdaIndex.functionOffset(mid) == funcStartOffset) { lsda = lsdaIndex.lsdaOffset(mid) + sects.dso_base; break; } else if (lsdaIndex.functionOffset(mid) < funcStartOffset) { low = mid + 1; } else { high = mid; } } if (lsda == 0) { _LIBUNWIND_DEBUG_LOG("found encoding 0x%08X with HAS_LSDA bit set for " "pc=0x%0llX, but lsda table has no entry", encoding, (uint64_t) pc); return false; } } // extact personality routine, if encoding says function has one uint32_t personalityIndex = (encoding & UNWIND_PERSONALITY_MASK) >> (__builtin_ctz(UNWIND_PERSONALITY_MASK)); if (personalityIndex != 0) { --personalityIndex; // change 1-based to zero-based index if (personalityIndex > sectionHeader.personalityArrayCount()) { _LIBUNWIND_DEBUG_LOG("found encoding 0x%08X with personality index %d, " "but personality table has only %d entires", encoding, personalityIndex, sectionHeader.personalityArrayCount()); return false; } int32_t personalityDelta = (int32_t)_addressSpace.get32( sects.compact_unwind_section + sectionHeader.personalityArraySectionOffset() + personalityIndex * sizeof(uint32_t)); pint_t personalityPointer = sects.dso_base + (pint_t)personalityDelta; personality = _addressSpace.getP(personalityPointer); if (log) fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX), " "personalityDelta=0x%08X, personality=0x%08llX\n", (uint64_t) pc, personalityDelta, (uint64_t) personality); } if (log) fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX), " "encoding=0x%08X, lsda=0x%08llX for funcStart=0x%llX\n", (uint64_t) pc, encoding, (uint64_t) lsda, (uint64_t) funcStart); _info.start_ip = funcStart; _info.end_ip = funcEnd; _info.lsda = lsda; _info.handler = personality; _info.gp = 0; _info.flags = 0; _info.format = encoding; _info.unwind_info = 0; _info.unwind_info_size = 0; _info.extra = sects.dso_base; return true; } -#endif // _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) template +bool UnwindCursor::getInfoFromSEH(pint_t pc) { + pint_t base; + RUNTIME_FUNCTION *unwindEntry = lookUpSEHUnwindInfo(pc, &base); + if (!unwindEntry) { + _LIBUNWIND_DEBUG_LOG("\tpc not in table, pc=0x%llX", (uint64_t) pc); + return false; + } + _info.gp = 0; + _info.flags = 0; + _info.format = 0; + _info.unwind_info_size = sizeof(RUNTIME_FUNCTION); + _info.unwind_info = reinterpret_cast(unwindEntry); + _info.extra = base; + _info.start_ip = base + unwindEntry->BeginAddress; +#ifdef _LIBUNWIND_TARGET_X86_64 + _info.end_ip = base + unwindEntry->EndAddress; + // Only fill in the handler and LSDA if they're stale. + if (pc != getLastPC()) { + UNWIND_INFO *xdata = reinterpret_cast(base + unwindEntry->UnwindData); + if (xdata->Flags & (UNW_FLAG_EHANDLER|UNW_FLAG_UHANDLER)) { + // The personality is given in the UNWIND_INFO itself. The LSDA immediately + // follows the UNWIND_INFO. (This follows how both Clang and MSVC emit + // these structures.) + // N.B. UNWIND_INFO structs are DWORD-aligned. + uint32_t lastcode = (xdata->CountOfCodes + 1) & ~1; + const uint32_t *handler = reinterpret_cast(&xdata->UnwindCodes[lastcode]); + _info.lsda = reinterpret_cast(handler+1); + if (*handler) { + _info.handler = reinterpret_cast(__libunwind_seh_personality); + } else + _info.handler = 0; + } else { + _info.lsda = 0; + _info.handler = 0; + } + } +#elif defined(_LIBUNWIND_TARGET_ARM) + _info.end_ip = _info.start_ip + unwindEntry->FunctionLength; + _info.lsda = 0; // FIXME + _info.handler = 0; // FIXME +#endif + setLastPC(pc); + return true; +} +#endif + + +template void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { pint_t pc = (pint_t)this->getReg(UNW_REG_IP); -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) // Remove the thumb bit so the IP represents the actual instruction address. // This matches the behaviour of _Unwind_GetIP on arm. pc &= (pint_t)~0x1; #endif // If the last line of a function is a "throw" the compiler sometimes // emits no instructions after the call to __cxa_throw. This means // the return address is actually the start of the next function. // To disambiguate this, back up the pc when we know it is a return // address. if (isReturnAddress) --pc; // Ask address space object to find unwind sections for this pc. UnwindInfoSections sects; if (_addressSpace.findUnwindSections(pc, sects)) { -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) // If there is a compact unwind encoding table, look there first. if (sects.compact_unwind_section != 0) { if (this->getInfoFromCompactEncodingSection(pc, sects)) { - #if _LIBUNWIND_SUPPORT_DWARF_UNWIND + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) // Found info in table, done unless encoding says to use dwarf. uint32_t dwarfOffset; if ((sects.dwarf_section != 0) && compactSaysUseDwarf(&dwarfOffset)) { if (this->getInfoFromDwarfSection(pc, sects, dwarfOffset)) { // found info in dwarf, done return; } } #endif // If unwind table has entry, but entry says there is no unwind info, // record that we have no unwind info. if (_info.format == 0) _unwindInfoMissing = true; return; } } -#endif // _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + // If there is SEH unwind info, look there next. + if (this->getInfoFromSEH(pc)) + return; +#endif + +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) // If there is dwarf unwind info, look there next. if (sects.dwarf_section != 0) { if (this->getInfoFromDwarfSection(pc, sects)) { // found info in dwarf, done return; } } #endif -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) // If there is ARM EHABI unwind info, look there next. if (sects.arm_section != 0 && this->getInfoFromEHABISection(pc, sects)) return; #endif } -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) // There is no static unwind info for this pc. Look to see if an FDE was // dynamically registered for it. pint_t cachedFDE = DwarfFDECache::findFDE(0, pc); if (cachedFDE != 0) { CFI_Parser::FDE_Info fdeInfo; CFI_Parser::CIE_Info cieInfo; const char *msg = CFI_Parser::decodeFDE(_addressSpace, cachedFDE, &fdeInfo, &cieInfo); if (msg == NULL) { typename CFI_Parser::PrologInfo prolog; if (CFI_Parser::parseFDEInstructions(_addressSpace, fdeInfo, cieInfo, - pc, &prolog)) { + pc, R::getArch(), &prolog)) { // save off parsed FDE info _info.start_ip = fdeInfo.pcStart; _info.end_ip = fdeInfo.pcEnd; _info.lsda = fdeInfo.lsda; _info.handler = cieInfo.personality; _info.gp = prolog.spExtraArgSize; // Some frameless functions need SP // altered when resuming in function. _info.flags = 0; _info.format = dwarfEncoding(); _info.unwind_info = fdeInfo.fdeStart; _info.unwind_info_size = (uint32_t)fdeInfo.fdeLength; _info.extra = 0; return; } } } // Lastly, ask AddressSpace object about platform specific ways to locate // other FDEs. pint_t fde; if (_addressSpace.findOtherFDE(pc, fde)) { CFI_Parser::FDE_Info fdeInfo; CFI_Parser::CIE_Info cieInfo; if (!CFI_Parser::decodeFDE(_addressSpace, fde, &fdeInfo, &cieInfo)) { // Double check this FDE is for a function that includes the pc. if ((fdeInfo.pcStart <= pc) && (pc < fdeInfo.pcEnd)) { typename CFI_Parser::PrologInfo prolog; - if (CFI_Parser::parseFDEInstructions(_addressSpace, fdeInfo, - cieInfo, pc, &prolog)) { + if (CFI_Parser::parseFDEInstructions(_addressSpace, fdeInfo, cieInfo, + pc, R::getArch(), &prolog)) { // save off parsed FDE info _info.start_ip = fdeInfo.pcStart; _info.end_ip = fdeInfo.pcEnd; _info.lsda = fdeInfo.lsda; _info.handler = cieInfo.personality; _info.gp = prolog.spExtraArgSize; _info.flags = 0; _info.format = dwarfEncoding(); _info.unwind_info = fdeInfo.fdeStart; _info.unwind_info_size = (uint32_t)fdeInfo.fdeLength; _info.extra = 0; return; } } } } -#endif // #if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#endif // #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) // no unwind info, flag that we can't reliably unwind _unwindInfoMissing = true; } template int UnwindCursor::step() { // Bottom of stack is defined is when unwind info cannot be found. if (_unwindInfoMissing) return UNW_STEP_END; // Use unwinding info to modify register set as if function returned. int result; -#if _LIBUNWIND_SUPPORT_COMPACT_UNWIND +#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) result = this->stepWithCompactEncoding(); -#elif _LIBUNWIND_SUPPORT_DWARF_UNWIND +#elif defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + result = this->stepWithSEHData(); +#elif defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) result = this->stepWithDwarfFDE(); -#elif _LIBUNWIND_ARM_EHABI +#elif defined(_LIBUNWIND_ARM_EHABI) result = this->stepWithEHABI(); #else #error Need _LIBUNWIND_SUPPORT_COMPACT_UNWIND or \ + _LIBUNWIND_SUPPORT_SEH_UNWIND or \ _LIBUNWIND_SUPPORT_DWARF_UNWIND or \ _LIBUNWIND_ARM_EHABI #endif // update info based on new PC if (result == UNW_STEP_SUCCESS) { this->setInfoBasedOnIPRegister(true); if (_unwindInfoMissing) return UNW_STEP_END; - if (_info.gp) - setReg(UNW_REG_SP, getReg(UNW_REG_SP) + _info.gp); } return result; } template void UnwindCursor::getInfo(unw_proc_info_t *info) { *info = _info; } template bool UnwindCursor::getFunctionName(char *buf, size_t bufLen, unw_word_t *offset) { return _addressSpace.findFunctionName((pint_t)this->getReg(UNW_REG_IP), buf, bufLen, offset); } } // namespace libunwind #endif // __UNWINDCURSOR_HPP__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1-gcc-ext.c =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1-gcc-ext.c (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1-gcc-ext.c (revision 345026) @@ -1,358 +1,362 @@ //===--------------------- UnwindLevel1-gcc-ext.c -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Implements gcc extensions to the C++ ABI Exception Handling Level 1. // //===----------------------------------------------------------------------===// #include #include #include #include #include #include #include "config.h" #include "libunwind_ext.h" #include "libunwind.h" #include "Unwind-EHABI.h" #include "unwind.h" -#if _LIBUNWIND_BUILD_ZERO_COST_APIS +#if defined(_LIBUNWIND_BUILD_ZERO_COST_APIS) +#if defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) +#define private_1 private_[0] +#endif + /// Called by __cxa_rethrow(). _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_Resume_or_Rethrow(_Unwind_Exception *exception_object) { -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) _LIBUNWIND_TRACE_API("_Unwind_Resume_or_Rethrow(ex_obj=%p), private_1=%ld", (void *)exception_object, (long)exception_object->unwinder_cache.reserved1); #else - _LIBUNWIND_TRACE_API("_Unwind_Resume_or_Rethrow(ex_obj=%p), private_1=%ld", + _LIBUNWIND_TRACE_API("_Unwind_Resume_or_Rethrow(ex_obj=%p), private_1=%" PRIdPTR, (void *)exception_object, - (long)exception_object->private_1); + (intptr_t)exception_object->private_1); #endif -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) // _Unwind_RaiseException on EHABI will always set the reserved1 field to 0, // which is in the same position as private_1 below. return _Unwind_RaiseException(exception_object); #else // If this is non-forced and a stopping place was found, then this is a // re-throw. // Call _Unwind_RaiseException() as if this was a new exception if (exception_object->private_1 == 0) { return _Unwind_RaiseException(exception_object); // Will return if there is no catch clause, so that __cxa_rethrow can call // std::terminate(). } // Call through to _Unwind_Resume() which distiguishes between forced and // regular exceptions. _Unwind_Resume(exception_object); _LIBUNWIND_ABORT("_Unwind_Resume_or_Rethrow() called _Unwind_RaiseException()" " which unexpectedly returned"); #endif } /// Called by personality handler during phase 2 to get base address for data /// relative encodings. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetDataRelBase(struct _Unwind_Context *context) { (void)context; _LIBUNWIND_TRACE_API("_Unwind_GetDataRelBase(context=%p)", (void *)context); _LIBUNWIND_ABORT("_Unwind_GetDataRelBase() not implemented"); } /// Called by personality handler during phase 2 to get base address for text /// relative encodings. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetTextRelBase(struct _Unwind_Context *context) { (void)context; _LIBUNWIND_TRACE_API("_Unwind_GetTextRelBase(context=%p)", (void *)context); _LIBUNWIND_ABORT("_Unwind_GetTextRelBase() not implemented"); } /// Scans unwind information to find the function that contains the /// specified code address "pc". _LIBUNWIND_EXPORT void *_Unwind_FindEnclosingFunction(void *pc) { _LIBUNWIND_TRACE_API("_Unwind_FindEnclosingFunction(pc=%p)", pc); // This is slow, but works. // We create an unwind cursor then alter the IP to be pc unw_cursor_t cursor; unw_context_t uc; unw_proc_info_t info; unw_getcontext(&uc); unw_init_local(&cursor, &uc); - unw_set_reg(&cursor, UNW_REG_IP, (unw_word_t)(long) pc); + unw_set_reg(&cursor, UNW_REG_IP, (unw_word_t)(intptr_t) pc); if (unw_get_proc_info(&cursor, &info) == UNW_ESUCCESS) - return (void *)(long) info.start_ip; + return (void *)(intptr_t) info.start_ip; else return NULL; } /// Walk every frame and call trace function at each one. If trace function /// returns anything other than _URC_NO_REASON, then walk is terminated. _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_Backtrace(_Unwind_Trace_Fn callback, void *ref) { unw_cursor_t cursor; unw_context_t uc; unw_getcontext(&uc); unw_init_local(&cursor, &uc); _LIBUNWIND_TRACE_API("_Unwind_Backtrace(callback=%p)", (void *)(uintptr_t)callback); -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) // Create a mock exception object for force unwinding. _Unwind_Exception ex; memset(&ex, '\0', sizeof(ex)); ex.exception_class = 0x434C4E47554E5700; // CLNGUNW\0 #endif // walk each frame while (true) { _Unwind_Reason_Code result; -#if !_LIBUNWIND_ARM_EHABI +#if !defined(_LIBUNWIND_ARM_EHABI) // ask libunwind to get next frame (skip over first frame which is // _Unwind_Backtrace()) if (unw_step(&cursor) <= 0) { _LIBUNWIND_TRACE_UNWINDING(" _backtrace: ended because cursor reached " "bottom of stack, returning %d", _URC_END_OF_STACK); return _URC_END_OF_STACK; } #else // Get the information for this frame. unw_proc_info_t frameInfo; if (unw_get_proc_info(&cursor, &frameInfo) != UNW_ESUCCESS) { return _URC_END_OF_STACK; } // Update the pr_cache in the mock exception object. const uint32_t* unwindInfo = (uint32_t *) frameInfo.unwind_info; ex.pr_cache.fnstart = frameInfo.start_ip; ex.pr_cache.ehtp = (_Unwind_EHT_Header *) unwindInfo; ex.pr_cache.additional= frameInfo.flags; struct _Unwind_Context *context = (struct _Unwind_Context *)&cursor; // Get and call the personality function to unwind the frame. __personality_routine handler = (__personality_routine) frameInfo.handler; if (handler == NULL) { return _URC_END_OF_STACK; } if (handler(_US_VIRTUAL_UNWIND_FRAME | _US_FORCE_UNWIND, &ex, context) != _URC_CONTINUE_UNWIND) { return _URC_END_OF_STACK; } -#endif // _LIBUNWIND_ARM_EHABI +#endif // defined(_LIBUNWIND_ARM_EHABI) // debugging if (_LIBUNWIND_TRACING_UNWINDING) { char functionName[512]; unw_proc_info_t frame; unw_word_t offset; unw_get_proc_name(&cursor, functionName, 512, &offset); unw_get_proc_info(&cursor, &frame); _LIBUNWIND_TRACE_UNWINDING( - " _backtrace: start_ip=0x%llX, func=%s, lsda=0x%llX, context=%p", - (long long)frame.start_ip, functionName, (long long)frame.lsda, + " _backtrace: start_ip=0x%" PRIxPTR ", func=%s, lsda=0x%" PRIxPTR ", context=%p", + frame.start_ip, functionName, frame.lsda, (void *)&cursor); } // call trace function with this frame result = (*callback)((struct _Unwind_Context *)(&cursor), ref); if (result != _URC_NO_REASON) { _LIBUNWIND_TRACE_UNWINDING( " _backtrace: ended because callback returned %d", result); return result; } } } -/// Find dwarf unwind info for an address 'pc' in some function. +/// Find DWARF unwind info for an address 'pc' in some function. _LIBUNWIND_EXPORT const void *_Unwind_Find_FDE(const void *pc, struct dwarf_eh_bases *bases) { // This is slow, but works. // We create an unwind cursor then alter the IP to be pc unw_cursor_t cursor; unw_context_t uc; unw_proc_info_t info; unw_getcontext(&uc); unw_init_local(&cursor, &uc); - unw_set_reg(&cursor, UNW_REG_IP, (unw_word_t)(long) pc); + unw_set_reg(&cursor, UNW_REG_IP, (unw_word_t)(intptr_t) pc); unw_get_proc_info(&cursor, &info); bases->tbase = (uintptr_t)info.extra; bases->dbase = 0; // dbase not used on Mac OS X bases->func = (uintptr_t)info.start_ip; _LIBUNWIND_TRACE_API("_Unwind_Find_FDE(pc=%p) => %p", pc, - (void *)(long) info.unwind_info); - return (void *)(long) info.unwind_info; + (void *)(intptr_t) info.unwind_info); + return (void *)(intptr_t) info.unwind_info; } /// Returns the CFA (call frame area, or stack pointer at start of function) /// for the current context. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetCFA(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_word_t result; unw_get_reg(cursor, UNW_REG_SP, &result); - _LIBUNWIND_TRACE_API("_Unwind_GetCFA(context=%p) => 0x%" PRIx64, - (void *)context, (uint64_t)result); + _LIBUNWIND_TRACE_API("_Unwind_GetCFA(context=%p) => 0x%" PRIxPTR, + (void *)context, result); return (uintptr_t)result; } /// Called by personality handler during phase 2 to get instruction pointer. /// ipBefore is a boolean that says if IP is already adjusted to be the call /// site address. Normally IP is the return address. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIPInfo(struct _Unwind_Context *context, int *ipBefore) { _LIBUNWIND_TRACE_API("_Unwind_GetIPInfo(context=%p)", (void *)context); *ipBefore = 0; return _Unwind_GetIP(context); } -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) #ifdef __FreeBSD__ // Based on LLVM's lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp // and XXX should be fixed to be alignment-safe. static void processFDE(const char *addr, bool isDeregister) { uint64_t length; while ((length = *((const uint32_t *)addr)) != 0) { const char *p = addr + 4; if (length == 0xffffffff) { length = *((const uint64_t *)p); p += 8; } uint32_t offset = *((const uint32_t *)p); if (offset != 0) { if (isDeregister) _unw_remove_dynamic_fde((unw_word_t)(uintptr_t)addr); else _unw_add_dynamic_fde((unw_word_t)(uintptr_t)addr); } addr = p + length; } } /// Called by programs with dynamic code generators that want to register /// dynamically generated FDEs, with a libgcc-compatible API. _LIBUNWIND_EXPORT void __register_frame(const void *addr) { _LIBUNWIND_TRACE_API("__register_frame(%p)", addr); processFDE(addr, false); } /// Called by programs with dynamic code generators that want to unregister /// dynamically generated FDEs, with a libgcc-compatible API. _LIBUNWIND_EXPORT void __deregister_frame(const void *addr) { _LIBUNWIND_TRACE_API("__deregister_frame(%p)", addr); processFDE(addr, true); } #else /// Called by programs with dynamic code generators that want /// to register a dynamically generated FDE. /// This function has existed on Mac OS X since 10.4, but /// was broken until 10.6. _LIBUNWIND_EXPORT void __register_frame(const void *fde) { _LIBUNWIND_TRACE_API("__register_frame(%p)", fde); _unw_add_dynamic_fde((unw_word_t)(uintptr_t) fde); } /// Called by programs with dynamic code generators that want /// to unregister a dynamically generated FDE. /// This function has existed on Mac OS X since 10.4, but /// was broken until 10.6. _LIBUNWIND_EXPORT void __deregister_frame(const void *fde) { _LIBUNWIND_TRACE_API("__deregister_frame(%p)", fde); _unw_remove_dynamic_fde((unw_word_t)(uintptr_t) fde); } #endif // The following register/deregister functions are gcc extensions. // They have existed on Mac OS X, but have never worked because Mac OS X // before 10.6 used keymgr to track known FDEs, but these functions // never got updated to use keymgr. // For now, we implement these as do-nothing functions to keep any existing // applications working. We also add the not in 10.6 symbol so that nwe // application won't be able to use them. -#if _LIBUNWIND_SUPPORT_FRAME_APIS +#if defined(_LIBUNWIND_SUPPORT_FRAME_APIS) _LIBUNWIND_EXPORT void __register_frame_info_bases(const void *fde, void *ob, void *tb, void *db) { (void)fde; (void)ob; (void)tb; (void)db; _LIBUNWIND_TRACE_API("__register_frame_info_bases(%p,%p, %p, %p)", fde, ob, tb, db); // do nothing, this function never worked in Mac OS X } _LIBUNWIND_EXPORT void __register_frame_info(const void *fde, void *ob) { (void)fde; (void)ob; _LIBUNWIND_TRACE_API("__register_frame_info(%p, %p)", fde, ob); // do nothing, this function never worked in Mac OS X } _LIBUNWIND_EXPORT void __register_frame_info_table_bases(const void *fde, void *ob, void *tb, void *db) { (void)fde; (void)ob; (void)tb; (void)db; _LIBUNWIND_TRACE_API("__register_frame_info_table_bases" "(%p,%p, %p, %p)", fde, ob, tb, db); // do nothing, this function never worked in Mac OS X } _LIBUNWIND_EXPORT void __register_frame_info_table(const void *fde, void *ob) { (void)fde; (void)ob; _LIBUNWIND_TRACE_API("__register_frame_info_table(%p, %p)", fde, ob); // do nothing, this function never worked in Mac OS X } _LIBUNWIND_EXPORT void __register_frame_table(const void *fde) { (void)fde; _LIBUNWIND_TRACE_API("__register_frame_table(%p)", fde); // do nothing, this function never worked in Mac OS X } _LIBUNWIND_EXPORT void *__deregister_frame_info(const void *fde) { (void)fde; _LIBUNWIND_TRACE_API("__deregister_frame_info(%p)", fde); // do nothing, this function never worked in Mac OS X return NULL; } _LIBUNWIND_EXPORT void *__deregister_frame_info_bases(const void *fde) { (void)fde; _LIBUNWIND_TRACE_API("__deregister_frame_info_bases(%p)", fde); // do nothing, this function never worked in Mac OS X return NULL; } -#endif // _LIBUNWIND_SUPPORT_FRAME_APIS +#endif // defined(_LIBUNWIND_SUPPORT_FRAME_APIS) -#endif // _LIBUNWIND_SUPPORT_DWARF_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) -#endif // _LIBUNWIND_BUILD_ZERO_COST_APIS +#endif // defined(_LIBUNWIND_BUILD_ZERO_COST_APIS) Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1.c =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1.c (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindLevel1.c (revision 345026) @@ -1,506 +1,509 @@ //===------------------------- UnwindLevel1.c -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Implements C++ ABI Exception Handling Level 1 as documented at: // http://mentorembedded.github.io/cxx-abi/abi-eh.html // using libunwind // //===----------------------------------------------------------------------===// // ARM EHABI does not specify _Unwind_{Get,Set}{GR,IP}(). Thus, we are // defining inline functions to delegate the function calls to // _Unwind_VRS_{Get,Set}(). However, some applications might declare the // function protetype directly (instead of including ), thus we need // to export these functions from libunwind.so as well. #define _LIBUNWIND_UNWIND_LEVEL1_EXTERNAL_LINKAGE 1 #include #include #include #include #include #include #include "libunwind.h" #include "unwind.h" #include "config.h" -#if !_LIBUNWIND_ARM_EHABI +#if !defined(_LIBUNWIND_ARM_EHABI) && !defined(__USING_SJLJ_EXCEPTIONS__) +#ifndef _LIBUNWIND_SUPPORT_SEH_UNWIND + static _Unwind_Reason_Code unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object) { unw_init_local(cursor, uc); // Walk each frame looking for a place to stop. bool handlerNotFound = true; while (handlerNotFound) { // Ask libunwind to get next frame (skip over first which is // _Unwind_RaiseException). int stepResult = unw_step(cursor); if (stepResult == 0) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): unw_step() reached " "bottom => _URC_END_OF_STACK", (void *)exception_object); return _URC_END_OF_STACK; } else if (stepResult < 0) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): unw_step failed => " "_URC_FATAL_PHASE1_ERROR", (void *)exception_object); return _URC_FATAL_PHASE1_ERROR; } // See if frame has code to run (has personality routine). unw_proc_info_t frameInfo; unw_word_t sp; if (unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase1(ex_ojb=%p): unw_get_proc_info " "failed => _URC_FATAL_PHASE1_ERROR", (void *)exception_object); return _URC_FATAL_PHASE1_ERROR; } // When tracing, print state information. if (_LIBUNWIND_TRACING_UNWINDING) { char functionBuf[512]; const char *functionName = functionBuf; unw_word_t offset; if ((unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf), &offset) != UNW_ESUCCESS) || (frameInfo.start_ip + offset > frameInfo.end_ip)) functionName = ".anonymous."; unw_word_t pc; unw_get_reg(cursor, UNW_REG_IP, &pc); _LIBUNWIND_TRACE_UNWINDING( - "unwind_phase1(ex_ojb=%p): pc=0x%" PRIx64 ", start_ip=0x%" PRIx64 - ", func=%s, lsda=0x%" PRIx64 ", personality=0x%" PRIx64 "", + "unwind_phase1(ex_ojb=%p): pc=0x%" PRIxPTR ", start_ip=0x%" PRIxPTR + ", func=%s, lsda=0x%" PRIxPTR ", personality=0x%" PRIxPTR "", (void *)exception_object, pc, frameInfo.start_ip, functionName, frameInfo.lsda, frameInfo.handler); } // If there is a personality routine, ask it if it will want to stop at // this frame. if (frameInfo.handler != 0) { __personality_routine p = - (__personality_routine)(long)(frameInfo.handler); + (__personality_routine)(uintptr_t)(frameInfo.handler); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): calling personality function %p", (void *)exception_object, (void *)(uintptr_t)p); _Unwind_Reason_Code personalityResult = (*p)(1, _UA_SEARCH_PHASE, exception_object->exception_class, exception_object, (struct _Unwind_Context *)(cursor)); switch (personalityResult) { case _URC_HANDLER_FOUND: // found a catch clause or locals that need destructing in this frame // stop search and remember stack pointer at the frame handlerNotFound = false; unw_get_reg(cursor, UNW_REG_SP, &sp); exception_object->private_2 = (uintptr_t)sp; _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_HANDLER_FOUND", (void *)exception_object); return _URC_NO_REASON; case _URC_CONTINUE_UNWIND: _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_CONTINUE_UNWIND", (void *)exception_object); // continue unwinding break; default: // something went wrong _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_ojb=%p): _URC_FATAL_PHASE1_ERROR", (void *)exception_object); return _URC_FATAL_PHASE1_ERROR; } } } return _URC_NO_REASON; } static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object) { unw_init_local(cursor, uc); _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p)", (void *)exception_object); // Walk each frame until we reach where search phase said to stop. while (true) { // Ask libunwind to get next frame (skip over first which is // _Unwind_RaiseException). int stepResult = unw_step(cursor); if (stepResult == 0) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_step() reached " "bottom => _URC_END_OF_STACK", (void *)exception_object); return _URC_END_OF_STACK; } else if (stepResult < 0) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_step failed => " "_URC_FATAL_PHASE1_ERROR", (void *)exception_object); return _URC_FATAL_PHASE2_ERROR; } // Get info about this frame. unw_word_t sp; unw_proc_info_t frameInfo; unw_get_reg(cursor, UNW_REG_SP, &sp); if (unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): unw_get_proc_info " "failed => _URC_FATAL_PHASE1_ERROR", (void *)exception_object); return _URC_FATAL_PHASE2_ERROR; } // When tracing, print state information. if (_LIBUNWIND_TRACING_UNWINDING) { char functionBuf[512]; const char *functionName = functionBuf; unw_word_t offset; if ((unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf), &offset) != UNW_ESUCCESS) || (frameInfo.start_ip + offset > frameInfo.end_ip)) functionName = ".anonymous."; - _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): start_ip=0x%" PRIx64 - ", func=%s, sp=0x%" PRIx64 ", lsda=0x%" PRIx64 - ", personality=0x%" PRIx64, + _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): start_ip=0x%" PRIxPTR + ", func=%s, sp=0x%" PRIxPTR ", lsda=0x%" PRIxPTR + ", personality=0x%" PRIxPTR, (void *)exception_object, frameInfo.start_ip, functionName, sp, frameInfo.lsda, frameInfo.handler); } // If there is a personality routine, tell it we are unwinding. if (frameInfo.handler != 0) { __personality_routine p = - (__personality_routine)(long)(frameInfo.handler); + (__personality_routine)(uintptr_t)(frameInfo.handler); _Unwind_Action action = _UA_CLEANUP_PHASE; if (sp == exception_object->private_2) { // Tell personality this was the frame it marked in phase 1. action = (_Unwind_Action)(_UA_CLEANUP_PHASE | _UA_HANDLER_FRAME); } _Unwind_Reason_Code personalityResult = (*p)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)(cursor)); switch (personalityResult) { case _URC_CONTINUE_UNWIND: // Continue unwinding _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2(ex_ojb=%p): _URC_CONTINUE_UNWIND", (void *)exception_object); if (sp == exception_object->private_2) { // Phase 1 said we would stop at this frame, but we did not... _LIBUNWIND_ABORT("during phase1 personality function said it would " "stop here, but now in phase2 it did not stop here"); } break; case _URC_INSTALL_CONTEXT: _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2(ex_ojb=%p): _URC_INSTALL_CONTEXT", (void *)exception_object); // Personality routine says to transfer control to landing pad. // We may get control back if landing pad calls _Unwind_Resume(). if (_LIBUNWIND_TRACING_UNWINDING) { unw_word_t pc; unw_get_reg(cursor, UNW_REG_IP, &pc); unw_get_reg(cursor, UNW_REG_SP, &sp); _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_ojb=%p): re-entering " - "user code with ip=0x%" PRIx64 - ", sp=0x%" PRIx64, + "user code with ip=0x%" PRIxPTR + ", sp=0x%" PRIxPTR, (void *)exception_object, pc, sp); } unw_resume(cursor); // unw_resume() only returns if there was an error. return _URC_FATAL_PHASE2_ERROR; default: // Personality routine returned an unknown result code. _LIBUNWIND_DEBUG_LOG("personality function returned unknown result %d", personalityResult); return _URC_FATAL_PHASE2_ERROR; } } } // Clean up phase did not resume at the frame that the search phase // said it would... return _URC_FATAL_PHASE2_ERROR; } static _Unwind_Reason_Code unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object, _Unwind_Stop_Fn stop, void *stop_parameter) { unw_init_local(cursor, uc); // Walk each frame until we reach where search phase said to stop while (unw_step(cursor) > 0) { // Update info about this frame. unw_proc_info_t frameInfo; if (unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) { _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): unw_step " "failed => _URC_END_OF_STACK", (void *)exception_object); return _URC_FATAL_PHASE2_ERROR; } // When tracing, print state information. if (_LIBUNWIND_TRACING_UNWINDING) { char functionBuf[512]; const char *functionName = functionBuf; unw_word_t offset; if ((unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf), &offset) != UNW_ESUCCESS) || (frameInfo.start_ip + offset > frameInfo.end_ip)) functionName = ".anonymous."; _LIBUNWIND_TRACE_UNWINDING( - "unwind_phase2_forced(ex_ojb=%p): start_ip=0x%" PRIx64 - ", func=%s, lsda=0x%" PRIx64 ", personality=0x%" PRIx64, + "unwind_phase2_forced(ex_ojb=%p): start_ip=0x%" PRIxPTR + ", func=%s, lsda=0x%" PRIxPTR ", personality=0x%" PRIxPTR, (void *)exception_object, frameInfo.start_ip, functionName, frameInfo.lsda, frameInfo.handler); } // Call stop function at each frame. _Unwind_Action action = (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE); _Unwind_Reason_Code stopResult = (*stop)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)(cursor), stop_parameter); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2_forced(ex_ojb=%p): stop function returned %d", (void *)exception_object, stopResult); if (stopResult != _URC_NO_REASON) { _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2_forced(ex_ojb=%p): stopped by stop function", (void *)exception_object); return _URC_FATAL_PHASE2_ERROR; } // If there is a personality routine, tell it we are unwinding. if (frameInfo.handler != 0) { __personality_routine p = - (__personality_routine)(long)(frameInfo.handler); + (__personality_routine)(intptr_t)(frameInfo.handler); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2_forced(ex_ojb=%p): calling personality function %p", (void *)exception_object, (void *)(uintptr_t)p); _Unwind_Reason_Code personalityResult = (*p)(1, action, exception_object->exception_class, exception_object, (struct _Unwind_Context *)(cursor)); switch (personalityResult) { case _URC_CONTINUE_UNWIND: _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned " "_URC_CONTINUE_UNWIND", (void *)exception_object); // Destructors called, continue unwinding break; case _URC_INSTALL_CONTEXT: _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned " "_URC_INSTALL_CONTEXT", (void *)exception_object); // We may get control back if landing pad calls _Unwind_Resume(). unw_resume(cursor); break; default: // Personality routine returned an unknown result code. _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): " "personality returned %d, " "_URC_FATAL_PHASE2_ERROR", (void *)exception_object, personalityResult); return _URC_FATAL_PHASE2_ERROR; } } } // Call stop function one last time and tell it we've reached the end // of the stack. _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): calling stop " "function with _UA_END_OF_STACK", (void *)exception_object); _Unwind_Action lastAction = (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE | _UA_END_OF_STACK); (*stop)(1, lastAction, exception_object->exception_class, exception_object, (struct _Unwind_Context *)(cursor), stop_parameter); // Clean up phase did not resume at the frame that the search phase said it // would. return _URC_FATAL_PHASE2_ERROR; } /// Called by __cxa_throw. Only returns if there is a fatal error. _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_RaiseException(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_RaiseException(ex_obj=%p)", (void *)exception_object); unw_context_t uc; unw_cursor_t cursor; unw_getcontext(&uc); // Mark that this is a non-forced unwind, so _Unwind_Resume() // can do the right thing. exception_object->private_1 = 0; exception_object->private_2 = 0; // phase 1: the search phase _Unwind_Reason_Code phase1 = unwind_phase1(&uc, &cursor, exception_object); if (phase1 != _URC_NO_REASON) return phase1; // phase 2: the clean up phase return unwind_phase2(&uc, &cursor, exception_object); } /// When _Unwind_RaiseException() is in phase2, it hands control /// to the personality function at each frame. The personality /// may force a jump to a landing pad in that function, the landing /// pad code may then call _Unwind_Resume() to continue with the /// unwinding. Note: the call to _Unwind_Resume() is from compiler /// geneated user code. All other _Unwind_* routines are called /// by the C++ runtime __cxa_* routines. /// /// Note: re-throwing an exception (as opposed to continuing the unwind) /// is implemented by having the code call __cxa_rethrow() which /// in turn calls _Unwind_Resume_or_Rethrow(). _LIBUNWIND_EXPORT void _Unwind_Resume(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_Resume(ex_obj=%p)", (void *)exception_object); unw_context_t uc; unw_cursor_t cursor; unw_getcontext(&uc); if (exception_object->private_1 != 0) unwind_phase2_forced(&uc, &cursor, exception_object, (_Unwind_Stop_Fn) exception_object->private_1, (void *)exception_object->private_2); else unwind_phase2(&uc, &cursor, exception_object); // Clients assume _Unwind_Resume() does not return, so all we can do is abort. _LIBUNWIND_ABORT("_Unwind_Resume() can't return"); } /// Not used by C++. /// Unwinds stack, calling "stop" function at each frame. /// Could be used to implement longjmp(). _LIBUNWIND_EXPORT _Unwind_Reason_Code _Unwind_ForcedUnwind(_Unwind_Exception *exception_object, _Unwind_Stop_Fn stop, void *stop_parameter) { _LIBUNWIND_TRACE_API("_Unwind_ForcedUnwind(ex_obj=%p, stop=%p)", (void *)exception_object, (void *)(uintptr_t)stop); unw_context_t uc; unw_cursor_t cursor; unw_getcontext(&uc); // Mark that this is a forced unwind, so _Unwind_Resume() can do // the right thing. exception_object->private_1 = (uintptr_t) stop; exception_object->private_2 = (uintptr_t) stop_parameter; // do it return unwind_phase2_forced(&uc, &cursor, exception_object, stop, stop_parameter); } /// Called by personality handler during phase 2 to get LSDA for current frame. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_proc_info_t frameInfo; uintptr_t result = 0; if (unw_get_proc_info(cursor, &frameInfo) == UNW_ESUCCESS) result = (uintptr_t)frameInfo.lsda; _LIBUNWIND_TRACE_API( "_Unwind_GetLanguageSpecificData(context=%p) => 0x%" PRIxPTR, (void *)context, result); if (result != 0) { if (*((uint8_t *)result) != 0xFF) _LIBUNWIND_DEBUG_LOG("lsda at 0x%" PRIxPTR " does not start with 0xFF", result); } return result; } /// Called by personality handler during phase 2 to find the start of the /// function. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetRegionStart(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_proc_info_t frameInfo; uintptr_t result = 0; if (unw_get_proc_info(cursor, &frameInfo) == UNW_ESUCCESS) result = (uintptr_t)frameInfo.start_ip; _LIBUNWIND_TRACE_API("_Unwind_GetRegionStart(context=%p) => 0x%" PRIxPTR, (void *)context, result); return result; } +#endif // !_LIBUNWIND_SUPPORT_SEH_UNWIND /// Called by personality handler during phase 2 if a foreign exception // is caught. _LIBUNWIND_EXPORT void _Unwind_DeleteException(_Unwind_Exception *exception_object) { _LIBUNWIND_TRACE_API("_Unwind_DeleteException(ex_obj=%p)", (void *)exception_object); if (exception_object->exception_cleanup != NULL) (*exception_object->exception_cleanup)(_URC_FOREIGN_EXCEPTION_CAUGHT, exception_object); } /// Called by personality handler during phase 2 to get register values. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetGR(struct _Unwind_Context *context, int index) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_word_t result; unw_get_reg(cursor, index, &result); - _LIBUNWIND_TRACE_API("_Unwind_GetGR(context=%p, reg=%d) => 0x%" PRIx64, - (void *)context, index, (uint64_t)result); + _LIBUNWIND_TRACE_API("_Unwind_GetGR(context=%p, reg=%d) => 0x%" PRIxPTR, + (void *)context, index, result); return (uintptr_t)result; } /// Called by personality handler during phase 2 to alter register values. _LIBUNWIND_EXPORT void _Unwind_SetGR(struct _Unwind_Context *context, int index, uintptr_t value) { - _LIBUNWIND_TRACE_API("_Unwind_SetGR(context=%p, reg=%d, value=0x%0" PRIx64 + _LIBUNWIND_TRACE_API("_Unwind_SetGR(context=%p, reg=%d, value=0x%0" PRIxPTR ")", - (void *)context, index, (uint64_t)value); + (void *)context, index, value); unw_cursor_t *cursor = (unw_cursor_t *)context; unw_set_reg(cursor, index, value); } /// Called by personality handler during phase 2 to get instruction pointer. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_word_t result; unw_get_reg(cursor, UNW_REG_IP, &result); - _LIBUNWIND_TRACE_API("_Unwind_GetIP(context=%p) => 0x%" PRIx64, - (void *)context, (uint64_t)result); + _LIBUNWIND_TRACE_API("_Unwind_GetIP(context=%p) => 0x%" PRIxPTR, + (void *)context, result); return (uintptr_t)result; } /// Called by personality handler during phase 2 to alter instruction pointer, /// such as setting where the landing pad is, so _Unwind_Resume() will /// start executing in the landing pad. _LIBUNWIND_EXPORT void _Unwind_SetIP(struct _Unwind_Context *context, uintptr_t value) { - _LIBUNWIND_TRACE_API("_Unwind_SetIP(context=%p, value=0x%0" PRIx64 ")", - (void *)context, (uint64_t)value); + _LIBUNWIND_TRACE_API("_Unwind_SetIP(context=%p, value=0x%0" PRIxPTR ")", + (void *)context, value); unw_cursor_t *cursor = (unw_cursor_t *)context; unw_set_reg(cursor, UNW_REG_IP, value); } -#endif // !_LIBUNWIND_ARM_EHABI +#endif // !defined(_LIBUNWIND_ARM_EHABI) && !defined(__USING_SJLJ_EXCEPTIONS__) Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersRestore.S =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersRestore.S (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersRestore.S (revision 345026) @@ -1,764 +1,1111 @@ //===-------------------- UnwindRegistersRestore.S ------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "assembly.h" .text +#if !defined(__USING_SJLJ_EXCEPTIONS__) + #if defined(__i386__) DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_x866jumptoEv) # # void libunwind::Registers_x86::jumpto() # +#if defined(_WIN32) +# On windows, the 'this' pointer is passed in ecx instead of on the stack + movl %ecx, %eax +#else # On entry: # + + # +-----------------------+ # + thread_state pointer + # +-----------------------+ # + return address + # +-----------------------+ <-- SP # + + movl 4(%esp), %eax +#endif # set up eax and ret on new stack location movl 28(%eax), %edx # edx holds new stack pointer subl $8,%edx movl %edx, 28(%eax) movl 0(%eax), %ebx movl %ebx, 0(%edx) movl 40(%eax), %ebx movl %ebx, 4(%edx) # we now have ret and eax pushed onto where new stack will be # restore all registers movl 4(%eax), %ebx movl 8(%eax), %ecx movl 12(%eax), %edx movl 16(%eax), %edi movl 20(%eax), %esi movl 24(%eax), %ebp movl 28(%eax), %esp # skip ss # skip eflags pop %eax # eax was already pushed on new stack ret # eip was already pushed on new stack # skip cs # skip ds # skip es # skip fs # skip gs #elif defined(__x86_64__) DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind16Registers_x86_646jumptoEv) # # void libunwind::Registers_x86_64::jumpto() # +#if defined(_WIN64) +# On entry, thread_state pointer is in rcx; move it into rdi +# to share restore code below. Since this routine restores and +# overwrites all registers, we can use the same registers for +# pointers and temporaries as on unix even though win64 normally +# mustn't clobber some of them. + movq %rcx, %rdi +#else # On entry, thread_state pointer is in rdi +#endif movq 56(%rdi), %rax # rax holds new stack pointer subq $16, %rax movq %rax, 56(%rdi) movq 32(%rdi), %rbx # store new rdi on new stack movq %rbx, 0(%rax) movq 128(%rdi), %rbx # store new rip on new stack movq %rbx, 8(%rax) # restore all registers movq 0(%rdi), %rax movq 8(%rdi), %rbx movq 16(%rdi), %rcx movq 24(%rdi), %rdx # restore rdi later movq 40(%rdi), %rsi movq 48(%rdi), %rbp # restore rsp later movq 64(%rdi), %r8 movq 72(%rdi), %r9 movq 80(%rdi), %r10 movq 88(%rdi), %r11 movq 96(%rdi), %r12 movq 104(%rdi), %r13 movq 112(%rdi), %r14 movq 120(%rdi), %r15 # skip rflags # skip cs # skip fs # skip gs + +#if defined(_WIN64) + movdqu 176(%rdi),%xmm0 + movdqu 192(%rdi),%xmm1 + movdqu 208(%rdi),%xmm2 + movdqu 224(%rdi),%xmm3 + movdqu 240(%rdi),%xmm4 + movdqu 256(%rdi),%xmm5 + movdqu 272(%rdi),%xmm6 + movdqu 288(%rdi),%xmm7 + movdqu 304(%rdi),%xmm8 + movdqu 320(%rdi),%xmm9 + movdqu 336(%rdi),%xmm10 + movdqu 352(%rdi),%xmm11 + movdqu 368(%rdi),%xmm12 + movdqu 384(%rdi),%xmm13 + movdqu 400(%rdi),%xmm14 + movdqu 416(%rdi),%xmm15 +#endif movq 56(%rdi), %rsp # cut back rsp to new location pop %rdi # rdi was saved here earlier ret # rip was saved here +#elif defined(__powerpc64__) + +DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind15Registers_ppc646jumptoEv) +// +// void libunwind::Registers_ppc64::jumpto() +// +// On entry: +// thread_state pointer is in r3 +// + +// load register (GPR) +#define PPC64_LR(n) \ + ld %r##n, (8 * (n + 2))(%r3) + + // restore integral registers + // skip r0 for now + // skip r1 for now + PPC64_LR(2) + // skip r3 for now + // skip r4 for now + // skip r5 for now + PPC64_LR(6) + PPC64_LR(7) + PPC64_LR(8) + PPC64_LR(9) + PPC64_LR(10) + PPC64_LR(11) + PPC64_LR(12) + PPC64_LR(13) + PPC64_LR(14) + PPC64_LR(15) + PPC64_LR(16) + PPC64_LR(17) + PPC64_LR(18) + PPC64_LR(19) + PPC64_LR(20) + PPC64_LR(21) + PPC64_LR(22) + PPC64_LR(23) + PPC64_LR(24) + PPC64_LR(25) + PPC64_LR(26) + PPC64_LR(27) + PPC64_LR(28) + PPC64_LR(29) + PPC64_LR(30) + PPC64_LR(31) + +#ifdef PPC64_HAS_VMX + + // restore VS registers + // (note that this also restores floating point registers and V registers, + // because part of VS is mapped to these registers) + + addi %r4, %r3, PPC64_OFFS_FP + +// load VS register +#define PPC64_LVS(n) \ + lxvd2x %vs##n, 0, %r4 ;\ + addi %r4, %r4, 16 + + // restore the first 32 VS regs (and also all floating point regs) + PPC64_LVS(0) + PPC64_LVS(1) + PPC64_LVS(2) + PPC64_LVS(3) + PPC64_LVS(4) + PPC64_LVS(5) + PPC64_LVS(6) + PPC64_LVS(7) + PPC64_LVS(8) + PPC64_LVS(9) + PPC64_LVS(10) + PPC64_LVS(11) + PPC64_LVS(12) + PPC64_LVS(13) + PPC64_LVS(14) + PPC64_LVS(15) + PPC64_LVS(16) + PPC64_LVS(17) + PPC64_LVS(18) + PPC64_LVS(19) + PPC64_LVS(20) + PPC64_LVS(21) + PPC64_LVS(22) + PPC64_LVS(23) + PPC64_LVS(24) + PPC64_LVS(25) + PPC64_LVS(26) + PPC64_LVS(27) + PPC64_LVS(28) + PPC64_LVS(29) + PPC64_LVS(30) + PPC64_LVS(31) + + // use VRSAVE to conditionally restore the remaining VS regs, + // that are where the V regs are mapped + + ld %r5, PPC64_OFFS_VRSAVE(%r3) // test VRsave + cmpwi %r5, 0 + beq Lnovec + +// conditionally load VS +#define PPC64_CLVS_BOTTOM(n) \ + beq Ldone##n ;\ + addi %r4, %r3, PPC64_OFFS_FP + n * 16 ;\ + lxvd2x %vs##n, 0, %r4 ;\ +Ldone##n: + +#define PPC64_CLVSl(n) \ + andis. %r0, %r5, (1<<(47-n)) ;\ +PPC64_CLVS_BOTTOM(n) + +#define PPC64_CLVSh(n) \ + andi. %r0, %r5, (1<<(63-n)) ;\ +PPC64_CLVS_BOTTOM(n) + + PPC64_CLVSl(32) + PPC64_CLVSl(33) + PPC64_CLVSl(34) + PPC64_CLVSl(35) + PPC64_CLVSl(36) + PPC64_CLVSl(37) + PPC64_CLVSl(38) + PPC64_CLVSl(39) + PPC64_CLVSl(40) + PPC64_CLVSl(41) + PPC64_CLVSl(42) + PPC64_CLVSl(43) + PPC64_CLVSl(44) + PPC64_CLVSl(45) + PPC64_CLVSl(46) + PPC64_CLVSl(47) + PPC64_CLVSh(48) + PPC64_CLVSh(49) + PPC64_CLVSh(50) + PPC64_CLVSh(51) + PPC64_CLVSh(52) + PPC64_CLVSh(53) + PPC64_CLVSh(54) + PPC64_CLVSh(55) + PPC64_CLVSh(56) + PPC64_CLVSh(57) + PPC64_CLVSh(58) + PPC64_CLVSh(59) + PPC64_CLVSh(60) + PPC64_CLVSh(61) + PPC64_CLVSh(62) + PPC64_CLVSh(63) + +#else + +// load FP register +#define PPC64_LF(n) \ + lfd %f##n, (PPC64_OFFS_FP + n * 16)(%r3) + + // restore float registers + PPC64_LF(0) + PPC64_LF(1) + PPC64_LF(2) + PPC64_LF(3) + PPC64_LF(4) + PPC64_LF(5) + PPC64_LF(6) + PPC64_LF(7) + PPC64_LF(8) + PPC64_LF(9) + PPC64_LF(10) + PPC64_LF(11) + PPC64_LF(12) + PPC64_LF(13) + PPC64_LF(14) + PPC64_LF(15) + PPC64_LF(16) + PPC64_LF(17) + PPC64_LF(18) + PPC64_LF(19) + PPC64_LF(20) + PPC64_LF(21) + PPC64_LF(22) + PPC64_LF(23) + PPC64_LF(24) + PPC64_LF(25) + PPC64_LF(26) + PPC64_LF(27) + PPC64_LF(28) + PPC64_LF(29) + PPC64_LF(30) + PPC64_LF(31) + + // restore vector registers if any are in use + ld %r5, PPC64_OFFS_VRSAVE(%r3) // test VRsave + cmpwi %r5, 0 + beq Lnovec + + subi %r4, %r1, 16 + // r4 is now a 16-byte aligned pointer into the red zone + // the _vectorScalarRegisters may not be 16-byte aligned + // so copy via red zone temp buffer + +#define PPC64_CLV_UNALIGNED_BOTTOM(n) \ + beq Ldone##n ;\ + ld %r0, (PPC64_OFFS_V + n * 16)(%r3) ;\ + std %r0, 0(%r4) ;\ + ld %r0, (PPC64_OFFS_V + n * 16 + 8)(%r3) ;\ + std %r0, 8(%r4) ;\ + lvx %v##n, 0, %r4 ;\ +Ldone ## n: + +#define PPC64_CLV_UNALIGNEDl(n) \ + andis. %r0, %r5, (1<<(15-n)) ;\ +PPC64_CLV_UNALIGNED_BOTTOM(n) + +#define PPC64_CLV_UNALIGNEDh(n) \ + andi. %r0, %r5, (1<<(31-n)) ;\ +PPC64_CLV_UNALIGNED_BOTTOM(n) + + PPC64_CLV_UNALIGNEDl(0) + PPC64_CLV_UNALIGNEDl(1) + PPC64_CLV_UNALIGNEDl(2) + PPC64_CLV_UNALIGNEDl(3) + PPC64_CLV_UNALIGNEDl(4) + PPC64_CLV_UNALIGNEDl(5) + PPC64_CLV_UNALIGNEDl(6) + PPC64_CLV_UNALIGNEDl(7) + PPC64_CLV_UNALIGNEDl(8) + PPC64_CLV_UNALIGNEDl(9) + PPC64_CLV_UNALIGNEDl(10) + PPC64_CLV_UNALIGNEDl(11) + PPC64_CLV_UNALIGNEDl(12) + PPC64_CLV_UNALIGNEDl(13) + PPC64_CLV_UNALIGNEDl(14) + PPC64_CLV_UNALIGNEDl(15) + PPC64_CLV_UNALIGNEDh(16) + PPC64_CLV_UNALIGNEDh(17) + PPC64_CLV_UNALIGNEDh(18) + PPC64_CLV_UNALIGNEDh(19) + PPC64_CLV_UNALIGNEDh(20) + PPC64_CLV_UNALIGNEDh(21) + PPC64_CLV_UNALIGNEDh(22) + PPC64_CLV_UNALIGNEDh(23) + PPC64_CLV_UNALIGNEDh(24) + PPC64_CLV_UNALIGNEDh(25) + PPC64_CLV_UNALIGNEDh(26) + PPC64_CLV_UNALIGNEDh(27) + PPC64_CLV_UNALIGNEDh(28) + PPC64_CLV_UNALIGNEDh(29) + PPC64_CLV_UNALIGNEDh(30) + PPC64_CLV_UNALIGNEDh(31) + +#endif + +Lnovec: + ld %r0, PPC64_OFFS_CR(%r3) + mtcr %r0 + ld %r0, PPC64_OFFS_SRR0(%r3) + mtctr %r0 + + PPC64_LR(0) + PPC64_LR(5) + PPC64_LR(4) + PPC64_LR(1) + PPC64_LR(3) + bctr + #elif defined(__ppc__) DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_ppc6jumptoEv) ; ; void libunwind::Registers_ppc::jumpto() ; ; On entry: ; thread_state pointer is in r3 ; ; restore integral registerrs ; skip r0 for now ; skip r1 for now lwz r2, 16(r3) ; skip r3 for now ; skip r4 for now ; skip r5 for now lwz r6, 32(r3) lwz r7, 36(r3) lwz r8, 40(r3) lwz r9, 44(r3) lwz r10, 48(r3) lwz r11, 52(r3) lwz r12, 56(r3) lwz r13, 60(r3) lwz r14, 64(r3) lwz r15, 68(r3) lwz r16, 72(r3) lwz r17, 76(r3) lwz r18, 80(r3) lwz r19, 84(r3) lwz r20, 88(r3) lwz r21, 92(r3) lwz r22, 96(r3) lwz r23,100(r3) lwz r24,104(r3) lwz r25,108(r3) lwz r26,112(r3) lwz r27,116(r3) lwz r28,120(r3) lwz r29,124(r3) lwz r30,128(r3) lwz r31,132(r3) ; restore float registers lfd f0, 160(r3) lfd f1, 168(r3) lfd f2, 176(r3) lfd f3, 184(r3) lfd f4, 192(r3) lfd f5, 200(r3) lfd f6, 208(r3) lfd f7, 216(r3) lfd f8, 224(r3) lfd f9, 232(r3) lfd f10,240(r3) lfd f11,248(r3) lfd f12,256(r3) lfd f13,264(r3) lfd f14,272(r3) lfd f15,280(r3) lfd f16,288(r3) lfd f17,296(r3) lfd f18,304(r3) lfd f19,312(r3) lfd f20,320(r3) lfd f21,328(r3) lfd f22,336(r3) lfd f23,344(r3) lfd f24,352(r3) lfd f25,360(r3) lfd f26,368(r3) lfd f27,376(r3) lfd f28,384(r3) lfd f29,392(r3) lfd f30,400(r3) lfd f31,408(r3) ; restore vector registers if any are in use lwz r5,156(r3) ; test VRsave cmpwi r5,0 beq Lnovec subi r4,r1,16 rlwinm r4,r4,0,0,27 ; mask low 4-bits ; r4 is now a 16-byte aligned pointer into the red zone ; the _vectorRegisters may not be 16-byte aligned so copy via red zone temp buffer #define LOAD_VECTOR_UNALIGNEDl(_index) \ andis. r0,r5,(1<<(15-_index)) @\ beq Ldone ## _index @\ lwz r0, 424+_index*16(r3) @\ stw r0, 0(r4) @\ lwz r0, 424+_index*16+4(r3) @\ stw r0, 4(r4) @\ lwz r0, 424+_index*16+8(r3) @\ stw r0, 8(r4) @\ lwz r0, 424+_index*16+12(r3)@\ stw r0, 12(r4) @\ lvx v ## _index,0,r4 @\ Ldone ## _index: #define LOAD_VECTOR_UNALIGNEDh(_index) \ andi. r0,r5,(1<<(31-_index)) @\ beq Ldone ## _index @\ lwz r0, 424+_index*16(r3) @\ stw r0, 0(r4) @\ lwz r0, 424+_index*16+4(r3) @\ stw r0, 4(r4) @\ lwz r0, 424+_index*16+8(r3) @\ stw r0, 8(r4) @\ lwz r0, 424+_index*16+12(r3)@\ stw r0, 12(r4) @\ lvx v ## _index,0,r4 @\ Ldone ## _index: LOAD_VECTOR_UNALIGNEDl(0) LOAD_VECTOR_UNALIGNEDl(1) LOAD_VECTOR_UNALIGNEDl(2) LOAD_VECTOR_UNALIGNEDl(3) LOAD_VECTOR_UNALIGNEDl(4) LOAD_VECTOR_UNALIGNEDl(5) LOAD_VECTOR_UNALIGNEDl(6) LOAD_VECTOR_UNALIGNEDl(7) LOAD_VECTOR_UNALIGNEDl(8) LOAD_VECTOR_UNALIGNEDl(9) LOAD_VECTOR_UNALIGNEDl(10) LOAD_VECTOR_UNALIGNEDl(11) LOAD_VECTOR_UNALIGNEDl(12) LOAD_VECTOR_UNALIGNEDl(13) LOAD_VECTOR_UNALIGNEDl(14) LOAD_VECTOR_UNALIGNEDl(15) LOAD_VECTOR_UNALIGNEDh(16) LOAD_VECTOR_UNALIGNEDh(17) LOAD_VECTOR_UNALIGNEDh(18) LOAD_VECTOR_UNALIGNEDh(19) LOAD_VECTOR_UNALIGNEDh(20) LOAD_VECTOR_UNALIGNEDh(21) LOAD_VECTOR_UNALIGNEDh(22) LOAD_VECTOR_UNALIGNEDh(23) LOAD_VECTOR_UNALIGNEDh(24) LOAD_VECTOR_UNALIGNEDh(25) LOAD_VECTOR_UNALIGNEDh(26) LOAD_VECTOR_UNALIGNEDh(27) LOAD_VECTOR_UNALIGNEDh(28) LOAD_VECTOR_UNALIGNEDh(29) LOAD_VECTOR_UNALIGNEDh(30) LOAD_VECTOR_UNALIGNEDh(31) Lnovec: lwz r0, 136(r3) ; __cr mtocrf 255,r0 lwz r0, 148(r3) ; __ctr mtctr r0 lwz r0, 0(r3) ; __ssr0 mtctr r0 lwz r0, 8(r3) ; do r0 now lwz r5,28(r3) ; do r5 now lwz r4,24(r3) ; do r4 now lwz r1,12(r3) ; do sp now lwz r3,20(r3) ; do r3 last bctr #elif defined(__arm64__) || defined(__aarch64__) // // void libunwind::Registers_arm64::jumpto() // // On entry: // thread_state pointer is in x0 // .p2align 2 DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind15Registers_arm646jumptoEv) // skip restore of x0,x1 for now ldp x2, x3, [x0, #0x010] ldp x4, x5, [x0, #0x020] ldp x6, x7, [x0, #0x030] ldp x8, x9, [x0, #0x040] ldp x10,x11, [x0, #0x050] ldp x12,x13, [x0, #0x060] ldp x14,x15, [x0, #0x070] ldp x16,x17, [x0, #0x080] ldp x18,x19, [x0, #0x090] ldp x20,x21, [x0, #0x0A0] ldp x22,x23, [x0, #0x0B0] ldp x24,x25, [x0, #0x0C0] ldp x26,x27, [x0, #0x0D0] ldp x28,x29, [x0, #0x0E0] ldr x30, [x0, #0x100] // restore pc into lr ldr x1, [x0, #0x0F8] mov sp,x1 // restore sp ldp d0, d1, [x0, #0x110] ldp d2, d3, [x0, #0x120] ldp d4, d5, [x0, #0x130] ldp d6, d7, [x0, #0x140] ldp d8, d9, [x0, #0x150] ldp d10,d11, [x0, #0x160] ldp d12,d13, [x0, #0x170] ldp d14,d15, [x0, #0x180] ldp d16,d17, [x0, #0x190] ldp d18,d19, [x0, #0x1A0] ldp d20,d21, [x0, #0x1B0] ldp d22,d23, [x0, #0x1C0] ldp d24,d25, [x0, #0x1D0] ldp d26,d27, [x0, #0x1E0] ldp d28,d29, [x0, #0x1F0] ldr d30, [x0, #0x200] ldr d31, [x0, #0x208] ldp x0, x1, [x0, #0x000] // restore x0,x1 ret x30 // jump to pc #elif defined(__arm__) && !defined(__APPLE__) #if !defined(__ARM_ARCH_ISA_ARM) .thumb #endif @ @ void libunwind::Registers_arm::restoreCoreAndJumpTo() @ @ On entry: @ thread_state pointer is in r0 @ .p2align 2 DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm20restoreCoreAndJumpToEv) -#if !defined(__ARM_ARCH_ISA_ARM) - ldr r2, [r0, #52] - ldr r3, [r0, #60] +#if !defined(__ARM_ARCH_ISA_ARM) && __ARM_ARCH_ISA_THUMB == 1 + @ r8-r11: ldm into r1-r4, then mov to r8-r11 + adds r0, #0x20 + ldm r0!, {r1-r4} + subs r0, #0x30 + mov r8, r1 + mov r9, r2 + mov r10, r3 + mov r11, r4 + @ r12 does not need loading, it it the intra-procedure-call scratch register + ldr r2, [r0, #0x34] + ldr r3, [r0, #0x3c] mov sp, r2 mov lr, r3 @ restore pc into lr ldm r0, {r0-r7} #else @ Use lr as base so that r0 can be restored. mov lr, r0 @ 32bit thumb-2 restrictions for ldm: @ . the sp (r13) cannot be in the list @ . the pc (r15) and lr (r14) cannot both be in the list in an LDM instruction ldm lr, {r0-r12} ldr sp, [lr, #52] ldr lr, [lr, #60] @ restore pc into lr #endif JMP(lr) @ @ static void libunwind::Registers_arm::restoreVFPWithFLDMD(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3-d16 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm19restoreVFPWithFLDMDEPy) @ VFP and iwMMX instructions are only available when compiling with the flags @ that enable them. We do not want to do that in the library (because we do not @ want the compiler to generate instructions that access those) but this is @ only accessed if the personality routine needs these registers. Use of @ these registers implies they are, actually, available on the target, so @ it's ok to execute. @ So, generate the instruction using the corresponding coprocessor mnemonic. vldmia r0, {d0-d15} JMP(lr) @ @ static void libunwind::Registers_arm::restoreVFPWithFLDMX(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3-d16 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm19restoreVFPWithFLDMXEPy) vldmia r0, {d0-d15} @ fldmiax is deprecated in ARMv7+ and now behaves like vldmia JMP(lr) @ @ static void libunwind::Registers_arm::restoreVFPv3(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm12restoreVFPv3EPy) vldmia r0, {d16-d31} JMP(lr) +#if defined(__ARM_WMMX) + @ @ static void libunwind::Registers_arm::restoreiWMMX(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) + .arch armv5te +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm12restoreiWMMXEPy) -#if (!defined(__ARM_ARCH_6M__) && !defined(__ARM_ARCH_6SM__)) || defined(__ARM_WMMX) ldcl p1, cr0, [r0], #8 @ wldrd wR0, [r0], #8 ldcl p1, cr1, [r0], #8 @ wldrd wR1, [r0], #8 ldcl p1, cr2, [r0], #8 @ wldrd wR2, [r0], #8 ldcl p1, cr3, [r0], #8 @ wldrd wR3, [r0], #8 ldcl p1, cr4, [r0], #8 @ wldrd wR4, [r0], #8 ldcl p1, cr5, [r0], #8 @ wldrd wR5, [r0], #8 ldcl p1, cr6, [r0], #8 @ wldrd wR6, [r0], #8 ldcl p1, cr7, [r0], #8 @ wldrd wR7, [r0], #8 ldcl p1, cr8, [r0], #8 @ wldrd wR8, [r0], #8 ldcl p1, cr9, [r0], #8 @ wldrd wR9, [r0], #8 ldcl p1, cr10, [r0], #8 @ wldrd wR10, [r0], #8 ldcl p1, cr11, [r0], #8 @ wldrd wR11, [r0], #8 ldcl p1, cr12, [r0], #8 @ wldrd wR12, [r0], #8 ldcl p1, cr13, [r0], #8 @ wldrd wR13, [r0], #8 ldcl p1, cr14, [r0], #8 @ wldrd wR14, [r0], #8 ldcl p1, cr15, [r0], #8 @ wldrd wR15, [r0], #8 -#endif JMP(lr) @ @ static void libunwind::Registers_arm::restoreiWMMXControl(unw_uint32_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) + .arch armv5te +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm19restoreiWMMXControlEPj) -#if (!defined(__ARM_ARCH_6M__) && !defined(__ARM_ARCH_6SM__)) || defined(__ARM_WMMX) ldc2 p1, cr8, [r0], #4 @ wldrw wCGR0, [r0], #4 ldc2 p1, cr9, [r0], #4 @ wldrw wCGR1, [r0], #4 ldc2 p1, cr10, [r0], #4 @ wldrw wCGR2, [r0], #4 ldc2 p1, cr11, [r0], #4 @ wldrw wCGR3, [r0], #4 -#endif JMP(lr) +#endif + #elif defined(__or1k__) DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind14Registers_or1k6jumptoEv) # # void libunwind::Registers_or1k::jumpto() # # On entry: # thread_state pointer is in r3 # - # restore integral registerrs + # restore integral registers l.lwz r0, 0(r3) l.lwz r1, 4(r3) l.lwz r2, 8(r3) # skip r3 for now l.lwz r4, 16(r3) l.lwz r5, 20(r3) l.lwz r6, 24(r3) l.lwz r7, 28(r3) l.lwz r8, 32(r3) - l.lwz r9, 36(r3) + # skip r9 l.lwz r10, 40(r3) l.lwz r11, 44(r3) l.lwz r12, 48(r3) l.lwz r13, 52(r3) l.lwz r14, 56(r3) l.lwz r15, 60(r3) l.lwz r16, 64(r3) l.lwz r17, 68(r3) l.lwz r18, 72(r3) l.lwz r19, 76(r3) l.lwz r20, 80(r3) l.lwz r21, 84(r3) l.lwz r22, 88(r3) l.lwz r23, 92(r3) l.lwz r24, 96(r3) l.lwz r25,100(r3) l.lwz r26,104(r3) l.lwz r27,108(r3) l.lwz r28,112(r3) l.lwz r29,116(r3) l.lwz r30,120(r3) l.lwz r31,124(r3) # at last, restore r3 l.lwz r3, 12(r3) + # load new pc into ra + l.lwz r9, 128(r3) # jump to pc l.jr r9 l.nop #elif defined(__riscv) // // void libunwind::Registers_riscv::jumpto() // // On entry: // thread_state pointer is in a0 // .p2align 2 DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind15Registers_riscv6jumptoEv) #ifdef __riscv_float_abi_double fld f0, (8 * 32 + 8 * 0)(a0) fld f1, (8 * 32 + 8 * 1)(a0) fld f2, (8 * 32 + 8 * 2)(a0) fld f3, (8 * 32 + 8 * 3)(a0) fld f4, (8 * 32 + 8 * 4)(a0) fld f5, (8 * 32 + 8 * 5)(a0) fld f6, (8 * 32 + 8 * 6)(a0) fld f7, (8 * 32 + 8 * 7)(a0) fld f8, (8 * 32 + 8 * 8)(a0) fld f9, (8 * 32 + 8 * 9)(a0) fld f10, (8 * 32 + 8 * 10)(a0) fld f11, (8 * 32 + 8 * 11)(a0) fld f12, (8 * 32 + 8 * 12)(a0) fld f13, (8 * 32 + 8 * 13)(a0) fld f14, (8 * 32 + 8 * 14)(a0) fld f15, (8 * 32 + 8 * 15)(a0) fld f16, (8 * 32 + 8 * 16)(a0) fld f17, (8 * 32 + 8 * 17)(a0) fld f18, (8 * 32 + 8 * 18)(a0) fld f19, (8 * 32 + 8 * 19)(a0) fld f20, (8 * 32 + 8 * 20)(a0) fld f21, (8 * 32 + 8 * 21)(a0) fld f22, (8 * 32 + 8 * 22)(a0) fld f23, (8 * 32 + 8 * 23)(a0) fld f24, (8 * 32 + 8 * 24)(a0) fld f25, (8 * 32 + 8 * 25)(a0) fld f26, (8 * 32 + 8 * 26)(a0) fld f27, (8 * 32 + 8 * 27)(a0) fld f28, (8 * 32 + 8 * 28)(a0) fld f29, (8 * 32 + 8 * 29)(a0) fld f30, (8 * 32 + 8 * 30)(a0) fld f31, (8 * 32 + 8 * 31)(a0) #endif // x0 is zero ld x1, (8 * 1)(a0) ld x2, (8 * 2)(a0) ld x3, (8 * 3)(a0) ld x4, (8 * 4)(a0) ld x5, (8 * 5)(a0) ld x6, (8 * 6)(a0) ld x7, (8 * 7)(a0) ld x8, (8 * 8)(a0) ld x9, (8 * 9)(a0) // skip a0 for now ld x11, (8 * 11)(a0) ld x12, (8 * 12)(a0) ld x13, (8 * 13)(a0) ld x14, (8 * 14)(a0) ld x15, (8 * 15)(a0) ld x16, (8 * 16)(a0) ld x17, (8 * 17)(a0) ld x18, (8 * 18)(a0) ld x19, (8 * 19)(a0) ld x20, (8 * 20)(a0) ld x21, (8 * 21)(a0) ld x22, (8 * 22)(a0) ld x23, (8 * 23)(a0) ld x24, (8 * 24)(a0) ld x25, (8 * 25)(a0) ld x26, (8 * 26)(a0) ld x27, (8 * 27)(a0) ld x28, (8 * 28)(a0) ld x29, (8 * 29)(a0) ld x30, (8 * 30)(a0) ld x31, (8 * 31)(a0) ld x10, (8 * 10)(a0) // restore a0 ret // jump to ra #elif defined(__mips__) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 // // void libunwind::Registers_mips_o32::jumpto() // // On entry: // thread state pointer is in a0 ($4) // DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind18Registers_mips_o326jumptoEv) .set push .set noat .set noreorder .set nomacro #ifdef __mips_hard_float -#if __mips_fpr == 32 +#if __mips_fpr != 64 ldc1 $f0, (4 * 36 + 8 * 0)($4) ldc1 $f2, (4 * 36 + 8 * 2)($4) ldc1 $f4, (4 * 36 + 8 * 4)($4) ldc1 $f6, (4 * 36 + 8 * 6)($4) ldc1 $f8, (4 * 36 + 8 * 8)($4) ldc1 $f10, (4 * 36 + 8 * 10)($4) ldc1 $f12, (4 * 36 + 8 * 12)($4) ldc1 $f14, (4 * 36 + 8 * 14)($4) ldc1 $f16, (4 * 36 + 8 * 16)($4) ldc1 $f18, (4 * 36 + 8 * 18)($4) ldc1 $f20, (4 * 36 + 8 * 20)($4) ldc1 $f22, (4 * 36 + 8 * 22)($4) ldc1 $f24, (4 * 36 + 8 * 24)($4) ldc1 $f26, (4 * 36 + 8 * 26)($4) ldc1 $f28, (4 * 36 + 8 * 28)($4) ldc1 $f30, (4 * 36 + 8 * 30)($4) #else ldc1 $f0, (4 * 36 + 8 * 0)($4) ldc1 $f1, (4 * 36 + 8 * 1)($4) ldc1 $f2, (4 * 36 + 8 * 2)($4) ldc1 $f3, (4 * 36 + 8 * 3)($4) ldc1 $f4, (4 * 36 + 8 * 4)($4) ldc1 $f5, (4 * 36 + 8 * 5)($4) ldc1 $f6, (4 * 36 + 8 * 6)($4) ldc1 $f7, (4 * 36 + 8 * 7)($4) ldc1 $f8, (4 * 36 + 8 * 8)($4) ldc1 $f9, (4 * 36 + 8 * 9)($4) ldc1 $f10, (4 * 36 + 8 * 10)($4) ldc1 $f11, (4 * 36 + 8 * 11)($4) ldc1 $f12, (4 * 36 + 8 * 12)($4) ldc1 $f13, (4 * 36 + 8 * 13)($4) ldc1 $f14, (4 * 36 + 8 * 14)($4) ldc1 $f15, (4 * 36 + 8 * 15)($4) ldc1 $f16, (4 * 36 + 8 * 16)($4) ldc1 $f17, (4 * 36 + 8 * 17)($4) ldc1 $f18, (4 * 36 + 8 * 18)($4) ldc1 $f19, (4 * 36 + 8 * 19)($4) ldc1 $f20, (4 * 36 + 8 * 20)($4) ldc1 $f21, (4 * 36 + 8 * 21)($4) ldc1 $f22, (4 * 36 + 8 * 22)($4) ldc1 $f23, (4 * 36 + 8 * 23)($4) ldc1 $f24, (4 * 36 + 8 * 24)($4) ldc1 $f25, (4 * 36 + 8 * 25)($4) ldc1 $f26, (4 * 36 + 8 * 26)($4) ldc1 $f27, (4 * 36 + 8 * 27)($4) ldc1 $f28, (4 * 36 + 8 * 28)($4) ldc1 $f29, (4 * 36 + 8 * 29)($4) ldc1 $f30, (4 * 36 + 8 * 30)($4) ldc1 $f31, (4 * 36 + 8 * 31)($4) #endif #endif // restore hi and lo lw $8, (4 * 33)($4) mthi $8 lw $8, (4 * 34)($4) mtlo $8 // r0 is zero lw $1, (4 * 1)($4) lw $2, (4 * 2)($4) lw $3, (4 * 3)($4) // skip a0 for now lw $5, (4 * 5)($4) lw $6, (4 * 6)($4) lw $7, (4 * 7)($4) lw $8, (4 * 8)($4) lw $9, (4 * 9)($4) lw $10, (4 * 10)($4) lw $11, (4 * 11)($4) lw $12, (4 * 12)($4) lw $13, (4 * 13)($4) lw $14, (4 * 14)($4) lw $15, (4 * 15)($4) lw $16, (4 * 16)($4) lw $17, (4 * 17)($4) lw $18, (4 * 18)($4) lw $19, (4 * 19)($4) lw $20, (4 * 20)($4) lw $21, (4 * 21)($4) lw $22, (4 * 22)($4) lw $23, (4 * 23)($4) lw $24, (4 * 24)($4) lw $25, (4 * 25)($4) lw $26, (4 * 26)($4) lw $27, (4 * 27)($4) lw $28, (4 * 28)($4) lw $29, (4 * 29)($4) lw $30, (4 * 30)($4) // load new pc into ra lw $31, (4 * 32)($4) // jump to ra, load a0 in the delay slot jr $31 lw $4, (4 * 4)($4) .set pop #elif defined(__mips64) // // void libunwind::Registers_mips_newabi::jumpto() // // On entry: // thread state pointer is in a0 ($4) // DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind21Registers_mips_newabi6jumptoEv) .set push .set noat .set noreorder .set nomacro #ifdef __mips_hard_float ldc1 $f0, (8 * 35)($4) ldc1 $f1, (8 * 36)($4) ldc1 $f2, (8 * 37)($4) ldc1 $f3, (8 * 38)($4) ldc1 $f4, (8 * 39)($4) ldc1 $f5, (8 * 40)($4) ldc1 $f6, (8 * 41)($4) ldc1 $f7, (8 * 42)($4) ldc1 $f8, (8 * 43)($4) ldc1 $f9, (8 * 44)($4) ldc1 $f10, (8 * 45)($4) ldc1 $f11, (8 * 46)($4) ldc1 $f12, (8 * 47)($4) ldc1 $f13, (8 * 48)($4) ldc1 $f14, (8 * 49)($4) ldc1 $f15, (8 * 50)($4) ldc1 $f16, (8 * 51)($4) ldc1 $f17, (8 * 52)($4) ldc1 $f18, (8 * 53)($4) ldc1 $f19, (8 * 54)($4) ldc1 $f20, (8 * 55)($4) ldc1 $f21, (8 * 56)($4) ldc1 $f22, (8 * 57)($4) ldc1 $f23, (8 * 58)($4) ldc1 $f24, (8 * 59)($4) ldc1 $f25, (8 * 60)($4) ldc1 $f26, (8 * 61)($4) ldc1 $f27, (8 * 62)($4) ldc1 $f28, (8 * 63)($4) ldc1 $f29, (8 * 64)($4) ldc1 $f30, (8 * 65)($4) ldc1 $f31, (8 * 66)($4) #endif // restore hi and lo ld $8, (8 * 33)($4) mthi $8 ld $8, (8 * 34)($4) mtlo $8 // r0 is zero ld $1, (8 * 1)($4) ld $2, (8 * 2)($4) ld $3, (8 * 3)($4) // skip a0 for now ld $5, (8 * 5)($4) ld $6, (8 * 6)($4) ld $7, (8 * 7)($4) ld $8, (8 * 8)($4) ld $9, (8 * 9)($4) ld $10, (8 * 10)($4) ld $11, (8 * 11)($4) ld $12, (8 * 12)($4) ld $13, (8 * 13)($4) ld $14, (8 * 14)($4) ld $15, (8 * 15)($4) ld $16, (8 * 16)($4) ld $17, (8 * 17)($4) ld $18, (8 * 18)($4) ld $19, (8 * 19)($4) ld $20, (8 * 20)($4) ld $21, (8 * 21)($4) ld $22, (8 * 22)($4) ld $23, (8 * 23)($4) ld $24, (8 * 24)($4) ld $25, (8 * 25)($4) ld $26, (8 * 26)($4) ld $27, (8 * 27)($4) ld $28, (8 * 28)($4) ld $29, (8 * 29)($4) ld $30, (8 * 30)($4) // load new pc into ra ld $31, (8 * 32)($4) // jump to ra, load a0 in the delay slot jr $31 ld $4, (8 * 4)($4) .set pop +#elif defined(__sparc__) + +// +// void libunwind::Registers_sparc_o32::jumpto() +// +// On entry: +// thread_state pointer is in o0 +// +DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind15Registers_sparc6jumptoEv) + ta 3 + ldd [%o0 + 64], %l0 + ldd [%o0 + 72], %l2 + ldd [%o0 + 80], %l4 + ldd [%o0 + 88], %l6 + ldd [%o0 + 96], %i0 + ldd [%o0 + 104], %i2 + ldd [%o0 + 112], %i4 + ldd [%o0 + 120], %i6 + ld [%o0 + 60], %o7 + jmp %o7 + nop + #endif + +#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */ NO_EXEC_STACK_DIRECTIVE Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersSave.S =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersSave.S (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/UnwindRegistersSave.S (revision 345026) @@ -1,748 +1,1058 @@ //===------------------------ UnwindRegistersSave.S -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "assembly.h" .text +#if !defined(__USING_SJLJ_EXCEPTIONS__) + #if defined(__i386__) # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # + + # +-----------------------+ # + thread_state pointer + # +-----------------------+ # + return address + # +-----------------------+ <-- SP # + + # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) push %eax movl 8(%esp), %eax movl %ebx, 4(%eax) movl %ecx, 8(%eax) movl %edx, 12(%eax) movl %edi, 16(%eax) movl %esi, 20(%eax) movl %ebp, 24(%eax) movl %esp, %edx addl $8, %edx movl %edx, 28(%eax) # store what sp was at call site as esp # skip ss # skip eflags movl 4(%esp), %edx movl %edx, 40(%eax) # store return address as eip # skip cs # skip ds # skip es # skip fs # skip gs movl (%esp), %edx movl %edx, (%eax) # store original eax popl %eax xorl %eax, %eax # return UNW_ESUCCESS ret #elif defined(__x86_64__) # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # thread_state pointer is in rdi # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) - movq %rax, (%rdi) - movq %rbx, 8(%rdi) - movq %rcx, 16(%rdi) - movq %rdx, 24(%rdi) - movq %rdi, 32(%rdi) - movq %rsi, 40(%rdi) - movq %rbp, 48(%rdi) - movq %rsp, 56(%rdi) - addq $8, 56(%rdi) - movq %r8, 64(%rdi) - movq %r9, 72(%rdi) - movq %r10, 80(%rdi) - movq %r11, 88(%rdi) - movq %r12, 96(%rdi) - movq %r13,104(%rdi) - movq %r14,112(%rdi) - movq %r15,120(%rdi) - movq (%rsp),%rsi - movq %rsi,128(%rdi) # store return address as rip +#if defined(_WIN64) +#define PTR %rcx +#define TMP %rdx +#else +#define PTR %rdi +#define TMP %rsi +#endif + + movq %rax, (PTR) + movq %rbx, 8(PTR) + movq %rcx, 16(PTR) + movq %rdx, 24(PTR) + movq %rdi, 32(PTR) + movq %rsi, 40(PTR) + movq %rbp, 48(PTR) + movq %rsp, 56(PTR) + addq $8, 56(PTR) + movq %r8, 64(PTR) + movq %r9, 72(PTR) + movq %r10, 80(PTR) + movq %r11, 88(PTR) + movq %r12, 96(PTR) + movq %r13,104(PTR) + movq %r14,112(PTR) + movq %r15,120(PTR) + movq (%rsp),TMP + movq TMP,128(PTR) # store return address as rip # skip rflags # skip cs # skip fs # skip gs + +#if defined(_WIN64) + movdqu %xmm0,176(PTR) + movdqu %xmm1,192(PTR) + movdqu %xmm2,208(PTR) + movdqu %xmm3,224(PTR) + movdqu %xmm4,240(PTR) + movdqu %xmm5,256(PTR) + movdqu %xmm6,272(PTR) + movdqu %xmm7,288(PTR) + movdqu %xmm8,304(PTR) + movdqu %xmm9,320(PTR) + movdqu %xmm10,336(PTR) + movdqu %xmm11,352(PTR) + movdqu %xmm12,368(PTR) + movdqu %xmm13,384(PTR) + movdqu %xmm14,400(PTR) + movdqu %xmm15,416(PTR) +#endif xorl %eax, %eax # return UNW_ESUCCESS ret #elif defined(__mips__) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # thread_state pointer is in a0 ($4) # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) .set push .set noat .set noreorder .set nomacro sw $1, (4 * 1)($4) sw $2, (4 * 2)($4) sw $3, (4 * 3)($4) sw $4, (4 * 4)($4) sw $5, (4 * 5)($4) sw $6, (4 * 6)($4) sw $7, (4 * 7)($4) sw $8, (4 * 8)($4) sw $9, (4 * 9)($4) sw $10, (4 * 10)($4) sw $11, (4 * 11)($4) sw $12, (4 * 12)($4) sw $13, (4 * 13)($4) sw $14, (4 * 14)($4) sw $15, (4 * 15)($4) sw $16, (4 * 16)($4) sw $17, (4 * 17)($4) sw $18, (4 * 18)($4) sw $19, (4 * 19)($4) sw $20, (4 * 20)($4) sw $21, (4 * 21)($4) sw $22, (4 * 22)($4) sw $23, (4 * 23)($4) sw $24, (4 * 24)($4) sw $25, (4 * 25)($4) sw $26, (4 * 26)($4) sw $27, (4 * 27)($4) sw $28, (4 * 28)($4) sw $29, (4 * 29)($4) sw $30, (4 * 30)($4) sw $31, (4 * 31)($4) # Store return address to pc sw $31, (4 * 32)($4) # hi and lo mfhi $8 sw $8, (4 * 33)($4) mflo $8 sw $8, (4 * 34)($4) #ifdef __mips_hard_float -#if __mips_fpr == 32 +#if __mips_fpr != 64 sdc1 $f0, (4 * 36 + 8 * 0)($4) sdc1 $f2, (4 * 36 + 8 * 2)($4) sdc1 $f4, (4 * 36 + 8 * 4)($4) sdc1 $f6, (4 * 36 + 8 * 6)($4) sdc1 $f8, (4 * 36 + 8 * 8)($4) sdc1 $f10, (4 * 36 + 8 * 10)($4) sdc1 $f12, (4 * 36 + 8 * 12)($4) sdc1 $f14, (4 * 36 + 8 * 14)($4) sdc1 $f16, (4 * 36 + 8 * 16)($4) sdc1 $f18, (4 * 36 + 8 * 18)($4) sdc1 $f20, (4 * 36 + 8 * 20)($4) sdc1 $f22, (4 * 36 + 8 * 22)($4) sdc1 $f24, (4 * 36 + 8 * 24)($4) sdc1 $f26, (4 * 36 + 8 * 26)($4) sdc1 $f28, (4 * 36 + 8 * 28)($4) sdc1 $f30, (4 * 36 + 8 * 30)($4) #else sdc1 $f0, (4 * 36 + 8 * 0)($4) sdc1 $f1, (4 * 36 + 8 * 1)($4) sdc1 $f2, (4 * 36 + 8 * 2)($4) sdc1 $f3, (4 * 36 + 8 * 3)($4) sdc1 $f4, (4 * 36 + 8 * 4)($4) sdc1 $f5, (4 * 36 + 8 * 5)($4) sdc1 $f6, (4 * 36 + 8 * 6)($4) sdc1 $f7, (4 * 36 + 8 * 7)($4) sdc1 $f8, (4 * 36 + 8 * 8)($4) sdc1 $f9, (4 * 36 + 8 * 9)($4) sdc1 $f10, (4 * 36 + 8 * 10)($4) sdc1 $f11, (4 * 36 + 8 * 11)($4) sdc1 $f12, (4 * 36 + 8 * 12)($4) sdc1 $f13, (4 * 36 + 8 * 13)($4) sdc1 $f14, (4 * 36 + 8 * 14)($4) sdc1 $f15, (4 * 36 + 8 * 15)($4) sdc1 $f16, (4 * 36 + 8 * 16)($4) sdc1 $f17, (4 * 36 + 8 * 17)($4) sdc1 $f18, (4 * 36 + 8 * 18)($4) sdc1 $f19, (4 * 36 + 8 * 19)($4) sdc1 $f20, (4 * 36 + 8 * 20)($4) sdc1 $f21, (4 * 36 + 8 * 21)($4) sdc1 $f22, (4 * 36 + 8 * 22)($4) sdc1 $f23, (4 * 36 + 8 * 23)($4) sdc1 $f24, (4 * 36 + 8 * 24)($4) sdc1 $f25, (4 * 36 + 8 * 25)($4) sdc1 $f26, (4 * 36 + 8 * 26)($4) sdc1 $f27, (4 * 36 + 8 * 27)($4) sdc1 $f28, (4 * 36 + 8 * 28)($4) sdc1 $f29, (4 * 36 + 8 * 29)($4) sdc1 $f30, (4 * 36 + 8 * 30)($4) sdc1 $f31, (4 * 36 + 8 * 31)($4) #endif #endif jr $31 # return UNW_ESUCCESS or $2, $0, $0 .set pop #elif defined(__mips64) # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # thread_state pointer is in a0 ($4) # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) .set push .set noat .set noreorder .set nomacro sd $1, (8 * 1)($4) sd $2, (8 * 2)($4) sd $3, (8 * 3)($4) sd $4, (8 * 4)($4) sd $5, (8 * 5)($4) sd $6, (8 * 6)($4) sd $7, (8 * 7)($4) sd $8, (8 * 8)($4) sd $9, (8 * 9)($4) sd $10, (8 * 10)($4) sd $11, (8 * 11)($4) sd $12, (8 * 12)($4) sd $13, (8 * 13)($4) sd $14, (8 * 14)($4) sd $15, (8 * 15)($4) sd $16, (8 * 16)($4) sd $17, (8 * 17)($4) sd $18, (8 * 18)($4) sd $19, (8 * 19)($4) sd $20, (8 * 20)($4) sd $21, (8 * 21)($4) sd $22, (8 * 22)($4) sd $23, (8 * 23)($4) sd $24, (8 * 24)($4) sd $25, (8 * 25)($4) sd $26, (8 * 26)($4) sd $27, (8 * 27)($4) sd $28, (8 * 28)($4) sd $29, (8 * 29)($4) sd $30, (8 * 30)($4) sd $31, (8 * 31)($4) # Store return address to pc sd $31, (8 * 32)($4) # hi and lo mfhi $8 sd $8, (8 * 33)($4) mflo $8 sd $8, (8 * 34)($4) #ifdef __mips_hard_float sdc1 $f0, (8 * 35)($4) sdc1 $f1, (8 * 36)($4) sdc1 $f2, (8 * 37)($4) sdc1 $f3, (8 * 38)($4) sdc1 $f4, (8 * 39)($4) sdc1 $f5, (8 * 40)($4) sdc1 $f6, (8 * 41)($4) sdc1 $f7, (8 * 42)($4) sdc1 $f8, (8 * 43)($4) sdc1 $f9, (8 * 44)($4) sdc1 $f10, (8 * 45)($4) sdc1 $f11, (8 * 46)($4) sdc1 $f12, (8 * 47)($4) sdc1 $f13, (8 * 48)($4) sdc1 $f14, (8 * 49)($4) sdc1 $f15, (8 * 50)($4) sdc1 $f16, (8 * 51)($4) sdc1 $f17, (8 * 52)($4) sdc1 $f18, (8 * 53)($4) sdc1 $f19, (8 * 54)($4) sdc1 $f20, (8 * 55)($4) sdc1 $f21, (8 * 56)($4) sdc1 $f22, (8 * 57)($4) sdc1 $f23, (8 * 58)($4) sdc1 $f24, (8 * 59)($4) sdc1 $f25, (8 * 60)($4) sdc1 $f26, (8 * 61)($4) sdc1 $f27, (8 * 62)($4) sdc1 $f28, (8 * 63)($4) sdc1 $f29, (8 * 64)($4) sdc1 $f30, (8 * 65)($4) sdc1 $f31, (8 * 66)($4) #endif jr $31 # return UNW_ESUCCESS or $2, $0, $0 .set pop # elif defined(__mips__) # # extern int unw_getcontext(unw_context_t* thread_state) # # Just trap for the time being. DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) teq $0, $0 +#elif defined(__powerpc64__) + +// +// extern int unw_getcontext(unw_context_t* thread_state) +// +// On entry: +// thread_state pointer is in r3 +// +DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) + +// store register (GPR) +#define PPC64_STR(n) \ + std %r##n, (8 * (n + 2))(%r3) + + // save GPRs + PPC64_STR(0) + mflr %r0 + std %r0, PPC64_OFFS_SRR0(%r3) // store lr as ssr0 + PPC64_STR(1) + PPC64_STR(2) + PPC64_STR(3) + PPC64_STR(4) + PPC64_STR(5) + PPC64_STR(6) + PPC64_STR(7) + PPC64_STR(8) + PPC64_STR(9) + PPC64_STR(10) + PPC64_STR(11) + PPC64_STR(12) + PPC64_STR(13) + PPC64_STR(14) + PPC64_STR(15) + PPC64_STR(16) + PPC64_STR(17) + PPC64_STR(18) + PPC64_STR(19) + PPC64_STR(20) + PPC64_STR(21) + PPC64_STR(22) + PPC64_STR(23) + PPC64_STR(24) + PPC64_STR(25) + PPC64_STR(26) + PPC64_STR(27) + PPC64_STR(28) + PPC64_STR(29) + PPC64_STR(30) + PPC64_STR(31) + + mfcr %r0 + std %r0, PPC64_OFFS_CR(%r3) + mfxer %r0 + std %r0, PPC64_OFFS_XER(%r3) + mflr %r0 + std %r0, PPC64_OFFS_LR(%r3) + mfctr %r0 + std %r0, PPC64_OFFS_CTR(%r3) + mfvrsave %r0 + std %r0, PPC64_OFFS_VRSAVE(%r3) + +#ifdef PPC64_HAS_VMX + // save VS registers + // (note that this also saves floating point registers and V registers, + // because part of VS is mapped to these registers) + + addi %r4, %r3, PPC64_OFFS_FP + +// store VS register +#define PPC64_STVS(n) \ + stxvd2x %vs##n, 0, %r4 ;\ + addi %r4, %r4, 16 + + PPC64_STVS(0) + PPC64_STVS(1) + PPC64_STVS(2) + PPC64_STVS(3) + PPC64_STVS(4) + PPC64_STVS(5) + PPC64_STVS(6) + PPC64_STVS(7) + PPC64_STVS(8) + PPC64_STVS(9) + PPC64_STVS(10) + PPC64_STVS(11) + PPC64_STVS(12) + PPC64_STVS(13) + PPC64_STVS(14) + PPC64_STVS(15) + PPC64_STVS(16) + PPC64_STVS(17) + PPC64_STVS(18) + PPC64_STVS(19) + PPC64_STVS(20) + PPC64_STVS(21) + PPC64_STVS(22) + PPC64_STVS(23) + PPC64_STVS(24) + PPC64_STVS(25) + PPC64_STVS(26) + PPC64_STVS(27) + PPC64_STVS(28) + PPC64_STVS(29) + PPC64_STVS(30) + PPC64_STVS(31) + PPC64_STVS(32) + PPC64_STVS(33) + PPC64_STVS(34) + PPC64_STVS(35) + PPC64_STVS(36) + PPC64_STVS(37) + PPC64_STVS(38) + PPC64_STVS(39) + PPC64_STVS(40) + PPC64_STVS(41) + PPC64_STVS(42) + PPC64_STVS(43) + PPC64_STVS(44) + PPC64_STVS(45) + PPC64_STVS(46) + PPC64_STVS(47) + PPC64_STVS(48) + PPC64_STVS(49) + PPC64_STVS(50) + PPC64_STVS(51) + PPC64_STVS(52) + PPC64_STVS(53) + PPC64_STVS(54) + PPC64_STVS(55) + PPC64_STVS(56) + PPC64_STVS(57) + PPC64_STVS(58) + PPC64_STVS(59) + PPC64_STVS(60) + PPC64_STVS(61) + PPC64_STVS(62) + PPC64_STVS(63) + +#else + +// store FP register +#define PPC64_STF(n) \ + stfd %f##n, (PPC64_OFFS_FP + n * 16)(%r3) + + // save float registers + PPC64_STF(0) + PPC64_STF(1) + PPC64_STF(2) + PPC64_STF(3) + PPC64_STF(4) + PPC64_STF(5) + PPC64_STF(6) + PPC64_STF(7) + PPC64_STF(8) + PPC64_STF(9) + PPC64_STF(10) + PPC64_STF(11) + PPC64_STF(12) + PPC64_STF(13) + PPC64_STF(14) + PPC64_STF(15) + PPC64_STF(16) + PPC64_STF(17) + PPC64_STF(18) + PPC64_STF(19) + PPC64_STF(20) + PPC64_STF(21) + PPC64_STF(22) + PPC64_STF(23) + PPC64_STF(24) + PPC64_STF(25) + PPC64_STF(26) + PPC64_STF(27) + PPC64_STF(28) + PPC64_STF(29) + PPC64_STF(30) + PPC64_STF(31) + + // save vector registers + + // Use 16-bytes below the stack pointer as an + // aligned buffer to save each vector register. + // Note that the stack pointer is always 16-byte aligned. + subi %r4, %r1, 16 + +#define PPC64_STV_UNALIGNED(n) \ + stvx %v##n, 0, %r4 ;\ + ld %r5, 0(%r4) ;\ + std %r5, (PPC64_OFFS_V + n * 16)(%r3) ;\ + ld %r5, 8(%r4) ;\ + std %r5, (PPC64_OFFS_V + n * 16 + 8)(%r3) + + PPC64_STV_UNALIGNED(0) + PPC64_STV_UNALIGNED(1) + PPC64_STV_UNALIGNED(2) + PPC64_STV_UNALIGNED(3) + PPC64_STV_UNALIGNED(4) + PPC64_STV_UNALIGNED(5) + PPC64_STV_UNALIGNED(6) + PPC64_STV_UNALIGNED(7) + PPC64_STV_UNALIGNED(8) + PPC64_STV_UNALIGNED(9) + PPC64_STV_UNALIGNED(10) + PPC64_STV_UNALIGNED(11) + PPC64_STV_UNALIGNED(12) + PPC64_STV_UNALIGNED(13) + PPC64_STV_UNALIGNED(14) + PPC64_STV_UNALIGNED(15) + PPC64_STV_UNALIGNED(16) + PPC64_STV_UNALIGNED(17) + PPC64_STV_UNALIGNED(18) + PPC64_STV_UNALIGNED(19) + PPC64_STV_UNALIGNED(20) + PPC64_STV_UNALIGNED(21) + PPC64_STV_UNALIGNED(22) + PPC64_STV_UNALIGNED(23) + PPC64_STV_UNALIGNED(24) + PPC64_STV_UNALIGNED(25) + PPC64_STV_UNALIGNED(26) + PPC64_STV_UNALIGNED(27) + PPC64_STV_UNALIGNED(28) + PPC64_STV_UNALIGNED(29) + PPC64_STV_UNALIGNED(30) + PPC64_STV_UNALIGNED(31) + +#endif + + li %r3, 0 // return UNW_ESUCCESS + blr + + #elif defined(__ppc__) ; ; extern int unw_getcontext(unw_context_t* thread_state) ; ; On entry: ; thread_state pointer is in r3 ; DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) stw r0, 8(r3) mflr r0 stw r0, 0(r3) ; store lr as ssr0 stw r1, 12(r3) stw r2, 16(r3) stw r3, 20(r3) stw r4, 24(r3) stw r5, 28(r3) stw r6, 32(r3) stw r7, 36(r3) stw r8, 40(r3) stw r9, 44(r3) stw r10, 48(r3) stw r11, 52(r3) stw r12, 56(r3) stw r13, 60(r3) stw r14, 64(r3) stw r15, 68(r3) stw r16, 72(r3) stw r17, 76(r3) stw r18, 80(r3) stw r19, 84(r3) stw r20, 88(r3) stw r21, 92(r3) stw r22, 96(r3) stw r23,100(r3) stw r24,104(r3) stw r25,108(r3) stw r26,112(r3) stw r27,116(r3) stw r28,120(r3) stw r29,124(r3) stw r30,128(r3) stw r31,132(r3) ; save VRSave register mfspr r0,256 stw r0,156(r3) ; save CR registers mfcr r0 stw r0,136(r3) ; save CTR register mfctr r0 stw r0,148(r3) ; save float registers stfd f0, 160(r3) stfd f1, 168(r3) stfd f2, 176(r3) stfd f3, 184(r3) stfd f4, 192(r3) stfd f5, 200(r3) stfd f6, 208(r3) stfd f7, 216(r3) stfd f8, 224(r3) stfd f9, 232(r3) stfd f10,240(r3) stfd f11,248(r3) stfd f12,256(r3) stfd f13,264(r3) stfd f14,272(r3) stfd f15,280(r3) stfd f16,288(r3) stfd f17,296(r3) stfd f18,304(r3) stfd f19,312(r3) stfd f20,320(r3) stfd f21,328(r3) stfd f22,336(r3) stfd f23,344(r3) stfd f24,352(r3) stfd f25,360(r3) stfd f26,368(r3) stfd f27,376(r3) stfd f28,384(r3) stfd f29,392(r3) stfd f30,400(r3) stfd f31,408(r3) ; save vector registers subi r4,r1,16 rlwinm r4,r4,0,0,27 ; mask low 4-bits ; r4 is now a 16-byte aligned pointer into the red zone #define SAVE_VECTOR_UNALIGNED(_vec, _offset) \ stvx _vec,0,r4 @\ lwz r5, 0(r4) @\ stw r5, _offset(r3) @\ lwz r5, 4(r4) @\ stw r5, _offset+4(r3) @\ lwz r5, 8(r4) @\ stw r5, _offset+8(r3) @\ lwz r5, 12(r4) @\ stw r5, _offset+12(r3) SAVE_VECTOR_UNALIGNED( v0, 424+0x000) SAVE_VECTOR_UNALIGNED( v1, 424+0x010) SAVE_VECTOR_UNALIGNED( v2, 424+0x020) SAVE_VECTOR_UNALIGNED( v3, 424+0x030) SAVE_VECTOR_UNALIGNED( v4, 424+0x040) SAVE_VECTOR_UNALIGNED( v5, 424+0x050) SAVE_VECTOR_UNALIGNED( v6, 424+0x060) SAVE_VECTOR_UNALIGNED( v7, 424+0x070) SAVE_VECTOR_UNALIGNED( v8, 424+0x080) SAVE_VECTOR_UNALIGNED( v9, 424+0x090) SAVE_VECTOR_UNALIGNED(v10, 424+0x0A0) SAVE_VECTOR_UNALIGNED(v11, 424+0x0B0) SAVE_VECTOR_UNALIGNED(v12, 424+0x0C0) SAVE_VECTOR_UNALIGNED(v13, 424+0x0D0) SAVE_VECTOR_UNALIGNED(v14, 424+0x0E0) SAVE_VECTOR_UNALIGNED(v15, 424+0x0F0) SAVE_VECTOR_UNALIGNED(v16, 424+0x100) SAVE_VECTOR_UNALIGNED(v17, 424+0x110) SAVE_VECTOR_UNALIGNED(v18, 424+0x120) SAVE_VECTOR_UNALIGNED(v19, 424+0x130) SAVE_VECTOR_UNALIGNED(v20, 424+0x140) SAVE_VECTOR_UNALIGNED(v21, 424+0x150) SAVE_VECTOR_UNALIGNED(v22, 424+0x160) SAVE_VECTOR_UNALIGNED(v23, 424+0x170) SAVE_VECTOR_UNALIGNED(v24, 424+0x180) SAVE_VECTOR_UNALIGNED(v25, 424+0x190) SAVE_VECTOR_UNALIGNED(v26, 424+0x1A0) SAVE_VECTOR_UNALIGNED(v27, 424+0x1B0) SAVE_VECTOR_UNALIGNED(v28, 424+0x1C0) SAVE_VECTOR_UNALIGNED(v29, 424+0x1D0) SAVE_VECTOR_UNALIGNED(v30, 424+0x1E0) SAVE_VECTOR_UNALIGNED(v31, 424+0x1F0) li r3, 0 ; return UNW_ESUCCESS blr #elif defined(__arm64__) || defined(__aarch64__) // // extern int unw_getcontext(unw_context_t* thread_state) // // On entry: // thread_state pointer is in x0 // .p2align 2 DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) stp x0, x1, [x0, #0x000] stp x2, x3, [x0, #0x010] stp x4, x5, [x0, #0x020] stp x6, x7, [x0, #0x030] stp x8, x9, [x0, #0x040] stp x10,x11, [x0, #0x050] stp x12,x13, [x0, #0x060] stp x14,x15, [x0, #0x070] stp x16,x17, [x0, #0x080] stp x18,x19, [x0, #0x090] stp x20,x21, [x0, #0x0A0] stp x22,x23, [x0, #0x0B0] stp x24,x25, [x0, #0x0C0] stp x26,x27, [x0, #0x0D0] stp x28,x29, [x0, #0x0E0] str x30, [x0, #0x0F0] mov x1,sp str x1, [x0, #0x0F8] str x30, [x0, #0x100] // store return address as pc // skip cpsr stp d0, d1, [x0, #0x110] stp d2, d3, [x0, #0x120] stp d4, d5, [x0, #0x130] stp d6, d7, [x0, #0x140] stp d8, d9, [x0, #0x150] stp d10,d11, [x0, #0x160] stp d12,d13, [x0, #0x170] stp d14,d15, [x0, #0x180] stp d16,d17, [x0, #0x190] stp d18,d19, [x0, #0x1A0] stp d20,d21, [x0, #0x1B0] stp d22,d23, [x0, #0x1C0] stp d24,d25, [x0, #0x1D0] stp d26,d27, [x0, #0x1E0] stp d28,d29, [x0, #0x1F0] str d30, [x0, #0x200] str d31, [x0, #0x208] mov x0, #0 // return UNW_ESUCCESS ret #elif defined(__arm__) && !defined(__APPLE__) #if !defined(__ARM_ARCH_ISA_ARM) .thumb #endif @ @ extern int unw_getcontext(unw_context_t* thread_state) @ @ On entry: @ thread_state pointer is in r0 @ @ Per EHABI #4.7 this only saves the core integer registers. @ EHABI #7.4.5 notes that in general all VRS registers should be restored @ however this is very hard to do for VFP registers because it is unknown @ to the library how many registers are implemented by the architecture. @ Instead, VFP registers are demand saved by logic external to unw_getcontext. @ .p2align 2 DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) -#if !defined(__ARM_ARCH_ISA_ARM) - stm r0, {r0-r7} +#if !defined(__ARM_ARCH_ISA_ARM) && __ARM_ARCH_ISA_THUMB == 1 + stm r0!, {r0-r7} + mov r1, r8 + mov r2, r9 + mov r3, r10 + stm r0!, {r1-r3} + mov r1, r11 mov r2, sp mov r3, lr - str r2, [r0, #52] - str r3, [r0, #56] - str r3, [r0, #60] @ store return address as pc + str r1, [r0, #0] @ r11 + @ r12 does not need storing, it it the intra-procedure-call scratch register + str r2, [r0, #8] @ sp + str r3, [r0, #12] @ lr + str r3, [r0, #16] @ store return address as pc + @ T1 does not have a non-cpsr-clobbering register-zeroing instruction. + @ It is safe to use here though because we are about to return, and cpsr is + @ not expected to be preserved. + movs r0, #0 @ return UNW_ESUCCESS #else @ 32bit thumb-2 restrictions for stm: @ . the sp (r13) cannot be in the list @ . the pc (r15) cannot be in the list in an STM instruction stm r0, {r0-r12} str sp, [r0, #52] str lr, [r0, #56] str lr, [r0, #60] @ store return address as pc -#endif -#if __ARM_ARCH_ISA_THUMB == 1 - @ T1 does not have a non-cpsr-clobbering register-zeroing instruction. - @ It is safe to use here though because we are about to return, and cpsr is - @ not expected to be preserved. - movs r0, #0 @ return UNW_ESUCCESS -#else mov r0, #0 @ return UNW_ESUCCESS #endif JMP(lr) @ @ static void libunwind::Registers_arm::saveVFPWithFSTMD(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3-d16 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm16saveVFPWithFSTMDEPy) vstmia r0, {d0-d15} JMP(lr) @ @ static void libunwind::Registers_arm::saveVFPWithFSTMX(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3-d16 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm16saveVFPWithFSTMXEPy) vstmia r0, {d0-d15} @ fstmiax is deprecated in ARMv7+ and now behaves like vstmia JMP(lr) @ @ static void libunwind::Registers_arm::saveVFPv3(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) .fpu vfpv3 +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm9saveVFPv3EPy) @ VFP and iwMMX instructions are only available when compiling with the flags @ that enable them. We do not want to do that in the library (because we do not @ want the compiler to generate instructions that access those) but this is @ only accessed if the personality routine needs these registers. Use of @ these registers implies they are, actually, available on the target, so @ it's ok to execute. @ So, generate the instructions using the corresponding coprocessor mnemonic. vstmia r0, {d16-d31} JMP(lr) +#if defined(_LIBUNWIND_ARM_WMMX) + @ @ static void libunwind::Registers_arm::saveiWMMX(unw_fpreg_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) + .arch armv5te +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm9saveiWMMXEPy) -#if (!defined(__ARM_ARCH_6M__) && !defined(__ARM_ARCH_6SM__)) || defined(__ARM_WMMX) stcl p1, cr0, [r0], #8 @ wstrd wR0, [r0], #8 stcl p1, cr1, [r0], #8 @ wstrd wR1, [r0], #8 stcl p1, cr2, [r0], #8 @ wstrd wR2, [r0], #8 stcl p1, cr3, [r0], #8 @ wstrd wR3, [r0], #8 stcl p1, cr4, [r0], #8 @ wstrd wR4, [r0], #8 stcl p1, cr5, [r0], #8 @ wstrd wR5, [r0], #8 stcl p1, cr6, [r0], #8 @ wstrd wR6, [r0], #8 stcl p1, cr7, [r0], #8 @ wstrd wR7, [r0], #8 stcl p1, cr8, [r0], #8 @ wstrd wR8, [r0], #8 stcl p1, cr9, [r0], #8 @ wstrd wR9, [r0], #8 stcl p1, cr10, [r0], #8 @ wstrd wR10, [r0], #8 stcl p1, cr11, [r0], #8 @ wstrd wR11, [r0], #8 stcl p1, cr12, [r0], #8 @ wstrd wR12, [r0], #8 stcl p1, cr13, [r0], #8 @ wstrd wR13, [r0], #8 stcl p1, cr14, [r0], #8 @ wstrd wR14, [r0], #8 stcl p1, cr15, [r0], #8 @ wstrd wR15, [r0], #8 -#endif JMP(lr) @ @ static void libunwind::Registers_arm::saveiWMMXControl(unw_uint32_t* values) @ @ On entry: @ values pointer is in r0 @ .p2align 2 +#if defined(__ELF__) + .arch armv5te +#endif DEFINE_LIBUNWIND_PRIVATE_FUNCTION(_ZN9libunwind13Registers_arm16saveiWMMXControlEPj) -#if (!defined(__ARM_ARCH_6M__) && !defined(__ARM_ARCH_6SM__)) || defined(__ARM_WMMX) stc2 p1, cr8, [r0], #4 @ wstrw wCGR0, [r0], #4 stc2 p1, cr9, [r0], #4 @ wstrw wCGR1, [r0], #4 stc2 p1, cr10, [r0], #4 @ wstrw wCGR2, [r0], #4 stc2 p1, cr11, [r0], #4 @ wstrw wCGR3, [r0], #4 -#endif JMP(lr) +#endif + #elif defined(__or1k__) # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # thread_state pointer is in r3 # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) l.sw 0(r3), r0 l.sw 4(r3), r1 l.sw 8(r3), r2 l.sw 12(r3), r3 l.sw 16(r3), r4 l.sw 20(r3), r5 l.sw 24(r3), r6 l.sw 28(r3), r7 l.sw 32(r3), r8 l.sw 36(r3), r9 l.sw 40(r3), r10 l.sw 44(r3), r11 l.sw 48(r3), r12 l.sw 52(r3), r13 l.sw 56(r3), r14 l.sw 60(r3), r15 l.sw 64(r3), r16 l.sw 68(r3), r17 l.sw 72(r3), r18 l.sw 76(r3), r19 l.sw 80(r3), r20 l.sw 84(r3), r21 l.sw 88(r3), r22 l.sw 92(r3), r23 l.sw 96(r3), r24 l.sw 100(r3), r25 l.sw 104(r3), r26 l.sw 108(r3), r27 l.sw 112(r3), r28 l.sw 116(r3), r29 l.sw 120(r3), r30 l.sw 124(r3), r31 + # store ra to pc + l.sw 128(r3), r9 + # zero epcr + l.sw 132(r3), r0 #elif defined(__riscv) # # extern int unw_getcontext(unw_context_t* thread_state) # # On entry: # thread_state pointer is in a0 # DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) // x0 is zero sd x1, (8 * 1)(a0) sd x2, (8 * 2)(a0) sd x3, (8 * 3)(a0) sd x4, (8 * 4)(a0) sd x5, (8 * 5)(a0) sd x6, (8 * 6)(a0) sd x7, (8 * 7)(a0) sd x8, (8 * 8)(a0) sd x9, (8 * 9)(a0) sd x10, (8 * 10)(a0) sd x11, (8 * 11)(a0) sd x12, (8 * 12)(a0) sd x13, (8 * 13)(a0) sd x14, (8 * 14)(a0) sd x15, (8 * 15)(a0) sd x16, (8 * 16)(a0) sd x17, (8 * 17)(a0) sd x18, (8 * 18)(a0) sd x19, (8 * 19)(a0) sd x20, (8 * 20)(a0) sd x21, (8 * 21)(a0) sd x22, (8 * 22)(a0) sd x23, (8 * 23)(a0) sd x24, (8 * 24)(a0) sd x25, (8 * 25)(a0) sd x26, (8 * 26)(a0) sd x27, (8 * 27)(a0) sd x28, (8 * 28)(a0) sd x29, (8 * 29)(a0) sd x30, (8 * 30)(a0) sd x31, (8 * 31)(a0) #ifdef __riscv_float_abi_double fsd f0, (8 * 32 + 8 * 0)(a0) fsd f1, (8 * 32 + 8 * 1)(a0) fsd f2, (8 * 32 + 8 * 2)(a0) fsd f3, (8 * 32 + 8 * 3)(a0) fsd f4, (8 * 32 + 8 * 4)(a0) fsd f5, (8 * 32 + 8 * 5)(a0) fsd f6, (8 * 32 + 8 * 6)(a0) fsd f7, (8 * 32 + 8 * 7)(a0) fsd f8, (8 * 32 + 8 * 8)(a0) fsd f9, (8 * 32 + 8 * 9)(a0) fsd f10, (8 * 32 + 8 * 10)(a0) fsd f11, (8 * 32 + 8 * 11)(a0) fsd f12, (8 * 32 + 8 * 12)(a0) fsd f13, (8 * 32 + 8 * 13)(a0) fsd f14, (8 * 32 + 8 * 14)(a0) fsd f15, (8 * 32 + 8 * 15)(a0) fsd f16, (8 * 32 + 8 * 16)(a0) fsd f17, (8 * 32 + 8 * 17)(a0) fsd f18, (8 * 32 + 8 * 18)(a0) fsd f19, (8 * 32 + 8 * 19)(a0) fsd f20, (8 * 32 + 8 * 20)(a0) fsd f21, (8 * 32 + 8 * 21)(a0) fsd f22, (8 * 32 + 8 * 22)(a0) fsd f23, (8 * 32 + 8 * 23)(a0) fsd f24, (8 * 32 + 8 * 24)(a0) fsd f25, (8 * 32 + 8 * 25)(a0) fsd f26, (8 * 32 + 8 * 26)(a0) fsd f27, (8 * 32 + 8 * 27)(a0) fsd f28, (8 * 32 + 8 * 28)(a0) fsd f29, (8 * 32 + 8 * 29)(a0) fsd f30, (8 * 32 + 8 * 30)(a0) fsd f31, (8 * 32 + 8 * 31)(a0) #endif li a0, 0 // return UNW_ESUCCESS ret // jump to ra + +#elif defined(__sparc__) + +# +# extern int unw_getcontext(unw_context_t* thread_state) +# +# On entry: +# thread_state pointer is in o0 +# +DEFINE_LIBUNWIND_FUNCTION(unw_getcontext) + ta 3 + add %o7, 8, %o7 + std %g0, [%o0 + 0] + std %g2, [%o0 + 8] + std %g4, [%o0 + 16] + std %g6, [%o0 + 24] + std %o0, [%o0 + 32] + std %o2, [%o0 + 40] + std %o4, [%o0 + 48] + std %o6, [%o0 + 56] + std %l0, [%o0 + 64] + std %l2, [%o0 + 72] + std %l4, [%o0 + 80] + std %l6, [%o0 + 88] + std %i0, [%o0 + 96] + std %i2, [%o0 + 104] + std %i4, [%o0 + 112] + std %i6, [%o0 + 120] + jmp %o7 + clr %o0 // return UNW_ESUCCESS #endif +#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */ NO_EXEC_STACK_DIRECTIVE - Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind_AppleExtras.cpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind_AppleExtras.cpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/Unwind_AppleExtras.cpp (revision 345026) @@ -1,205 +1,184 @@ //===--------------------- Unwind_AppleExtras.cpp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // //===----------------------------------------------------------------------===// #include "config.h" +#include "AddressSpace.hpp" #include "DwarfParser.hpp" -#include "unwind_ext.h" // private keymgr stuff #define KEYMGR_GCC3_DW2_OBJ_LIST 302 extern "C" { extern void _keymgr_set_and_unlock_processwide_ptr(int key, void *ptr); extern void *_keymgr_get_and_lock_processwide_ptr(int key); } // undocumented libgcc "struct object" struct libgcc_object { void *start; void *unused1; void *unused2; void *fde; unsigned long encoding; void *fde_end; libgcc_object *next; }; // undocumented libgcc "struct km_object_info" referenced by // KEYMGR_GCC3_DW2_OBJ_LIST struct libgcc_object_info { libgcc_object *seen_objects; libgcc_object *unseen_objects; unsigned spare[2]; }; // static linker symbols to prevent wrong two level namespace for _Unwind symbols #if defined(__arm__) #define NOT_HERE_BEFORE_5_0(sym) \ extern const char sym##_tmp30 __asm("$ld$hide$os3.0$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp30 = 0; \ extern const char sym##_tmp31 __asm("$ld$hide$os3.1$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp31 = 0; \ extern const char sym##_tmp32 __asm("$ld$hide$os3.2$_" #sym );\ __attribute__((visibility("default"))) const char sym##_tmp32 = 0; \ extern const char sym##_tmp40 __asm("$ld$hide$os4.0$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp40 = 0; \ extern const char sym##_tmp41 __asm("$ld$hide$os4.1$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp41 = 0; \ extern const char sym##_tmp42 __asm("$ld$hide$os4.2$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp42 = 0; \ extern const char sym##_tmp43 __asm("$ld$hide$os4.3$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp43 = 0; #elif defined(__arm64__) #define NOT_HERE_BEFORE_10_6(sym) #define NEVER_HERE(sym) #else #define NOT_HERE_BEFORE_10_6(sym) \ extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp5 = 0; #define NEVER_HERE(sym) \ extern const char sym##_tmp4 __asm("$ld$hide$os10.4$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp4 = 0; \ extern const char sym##_tmp5 __asm("$ld$hide$os10.5$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp5 = 0; \ extern const char sym##_tmp6 __asm("$ld$hide$os10.6$_" #sym ); \ __attribute__((visibility("default"))) const char sym##_tmp6 = 0; #endif -#if _LIBUNWIND_BUILD_ZERO_COST_APIS +#if defined(_LIBUNWIND_BUILD_ZERO_COST_APIS) // // symbols in libSystem.dylib in 10.6 and later, but are in libgcc_s.dylib in // earlier versions // NOT_HERE_BEFORE_10_6(_Unwind_DeleteException) NOT_HERE_BEFORE_10_6(_Unwind_Find_FDE) NOT_HERE_BEFORE_10_6(_Unwind_ForcedUnwind) NOT_HERE_BEFORE_10_6(_Unwind_GetGR) NOT_HERE_BEFORE_10_6(_Unwind_GetIP) NOT_HERE_BEFORE_10_6(_Unwind_GetLanguageSpecificData) NOT_HERE_BEFORE_10_6(_Unwind_GetRegionStart) NOT_HERE_BEFORE_10_6(_Unwind_RaiseException) NOT_HERE_BEFORE_10_6(_Unwind_Resume) NOT_HERE_BEFORE_10_6(_Unwind_SetGR) NOT_HERE_BEFORE_10_6(_Unwind_SetIP) NOT_HERE_BEFORE_10_6(_Unwind_Backtrace) NOT_HERE_BEFORE_10_6(_Unwind_FindEnclosingFunction) NOT_HERE_BEFORE_10_6(_Unwind_GetCFA) NOT_HERE_BEFORE_10_6(_Unwind_GetDataRelBase) NOT_HERE_BEFORE_10_6(_Unwind_GetTextRelBase) NOT_HERE_BEFORE_10_6(_Unwind_Resume_or_Rethrow) NOT_HERE_BEFORE_10_6(_Unwind_GetIPInfo) NOT_HERE_BEFORE_10_6(__register_frame) NOT_HERE_BEFORE_10_6(__deregister_frame) // // symbols in libSystem.dylib for compatibility, but we don't want any new code // using them // NEVER_HERE(__register_frame_info_bases) NEVER_HERE(__register_frame_info) NEVER_HERE(__register_frame_info_table_bases) NEVER_HERE(__register_frame_info_table) NEVER_HERE(__register_frame_table) NEVER_HERE(__deregister_frame_info) NEVER_HERE(__deregister_frame_info_bases) -#endif // _LIBUNWIND_BUILD_ZERO_COST_APIS +#endif // defined(_LIBUNWIND_BUILD_ZERO_COST_APIS) -#if _LIBUNWIND_BUILD_SJLJ_APIS +#if defined(_LIBUNWIND_BUILD_SJLJ_APIS) // // symbols in libSystem.dylib in iOS 5.0 and later, but are in libgcc_s.dylib in // earlier versions // NOT_HERE_BEFORE_5_0(_Unwind_GetLanguageSpecificData) NOT_HERE_BEFORE_5_0(_Unwind_GetRegionStart) NOT_HERE_BEFORE_5_0(_Unwind_GetIP) NOT_HERE_BEFORE_5_0(_Unwind_SetGR) NOT_HERE_BEFORE_5_0(_Unwind_SetIP) NOT_HERE_BEFORE_5_0(_Unwind_DeleteException) NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Register) NOT_HERE_BEFORE_5_0(_Unwind_GetGR) NOT_HERE_BEFORE_5_0(_Unwind_GetIPInfo) NOT_HERE_BEFORE_5_0(_Unwind_GetCFA) NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Resume) NOT_HERE_BEFORE_5_0(_Unwind_SjLj_RaiseException) NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Resume_or_Rethrow) NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Unregister) -#endif // _LIBUNWIND_BUILD_SJLJ_APIS +#endif // defined(_LIBUNWIND_BUILD_SJLJ_APIS) namespace libunwind { _LIBUNWIND_HIDDEN bool checkKeyMgrRegisteredFDEs(uintptr_t pc, void *&fde) { #if __MAC_OS_X_VERSION_MIN_REQUIRED // lastly check for old style keymgr registration of dynamically generated // FDEs acquire exclusive access to libgcc_object_info libgcc_object_info *head = (libgcc_object_info *) _keymgr_get_and_lock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST); if (head != NULL) { // look at each FDE in keymgr for (libgcc_object *ob = head->unseen_objects; ob != NULL; ob = ob->next) { CFI_Parser::FDE_Info fdeInfo; CFI_Parser::CIE_Info cieInfo; const char *msg = CFI_Parser::decodeFDE( LocalAddressSpace::sThisAddressSpace, (uintptr_t)ob->fde, &fdeInfo, &cieInfo); if (msg == NULL) { // Check if this FDE is for a function that includes the pc if ((fdeInfo.pcStart <= pc) && (pc < fdeInfo.pcEnd)) { fde = (void*)fdeInfo.pcStart; _keymgr_set_and_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST, head); return true; } } } } // release libgcc_object_info _keymgr_set_and_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST, head); #else (void)pc; (void)fde; #endif return false; } } - - -#if !defined(FOR_DYLD) && _LIBUNWIND_BUILD_SJLJ_APIS - -#include - -// Accessors to get get/set linked list of frames for sjlj based execeptions. -_LIBUNWIND_HIDDEN -struct _Unwind_FunctionContext *__Unwind_SjLj_GetTopOfFunctionStack() { - return (struct _Unwind_FunctionContext *) - _pthread_getspecific_direct(__PTK_LIBC_DYLD_Unwind_SjLj_Key); -} - -_LIBUNWIND_HIDDEN -void __Unwind_SjLj_SetTopOfFunctionStack(struct _Unwind_FunctionContext *fc) { - _pthread_setspecific_direct(__PTK_LIBC_DYLD_Unwind_SjLj_Key, fc); -} -#endif - - - Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/assembly.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/assembly.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/assembly.h (revision 345026) @@ -1,95 +1,124 @@ /* ===-- assembly.h - libUnwind assembler support macros -------------------=== * * The LLVM Compiler Infrastructure * * This file is dual licensed under the MIT and the University of Illinois Open * Source Licenses. See LICENSE.TXT for details. * * ===----------------------------------------------------------------------=== * * This file defines macros for use in libUnwind assembler source. * This file is not part of the interface of this library. * * ===----------------------------------------------------------------------=== */ #ifndef UNWIND_ASSEMBLY_H #define UNWIND_ASSEMBLY_H -#if defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__) +#if defined(__powerpc64__) +#define SEPARATOR ; +#define PPC64_OFFS_SRR0 0 +#define PPC64_OFFS_CR 272 +#define PPC64_OFFS_XER 280 +#define PPC64_OFFS_LR 288 +#define PPC64_OFFS_CTR 296 +#define PPC64_OFFS_VRSAVE 304 +#define PPC64_OFFS_FP 312 +#define PPC64_OFFS_V 824 +#ifdef _ARCH_PWR8 +#define PPC64_HAS_VMX +#endif +#elif defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__) #define SEPARATOR @ #elif defined(__arm64__) #define SEPARATOR %% #else #define SEPARATOR ; #endif -#if defined(__APPLE__) -#define HIDDEN_DIRECTIVE .private_extern -#else -#define HIDDEN_DIRECTIVE .hidden -#endif - #define GLUE2(a, b) a ## b #define GLUE(a, b) GLUE2(a, b) #define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name) #if defined(__APPLE__) #define SYMBOL_IS_FUNC(name) +#define EXPORT_SYMBOL(name) +#define HIDDEN_SYMBOL(name) .private_extern name #define NO_EXEC_STACK_DIRECTIVE #elif defined(__ELF__) #if defined(__arm__) #define SYMBOL_IS_FUNC(name) .type name,%function #else #define SYMBOL_IS_FUNC(name) .type name,@function #endif +#define EXPORT_SYMBOL(name) +#define HIDDEN_SYMBOL(name) .hidden name -#if defined(__GNU__) || defined(__ANDROID__) || defined(__FreeBSD__) +#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + defined(__linux__) #define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits #else #define NO_EXEC_STACK_DIRECTIVE #endif -#else +#elif defined(_WIN32) #define SYMBOL_IS_FUNC(name) \ .def name SEPARATOR \ .scl 2 SEPARATOR \ .type 32 SEPARATOR \ .endef +#define EXPORT_SYMBOL2(name) \ + .section .drectve,"yn" SEPARATOR \ + .ascii "-export:", #name, "\0" SEPARATOR \ + .text +#if defined(_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS) +#define EXPORT_SYMBOL(name) +#else +#define EXPORT_SYMBOL(name) EXPORT_SYMBOL2(name) +#endif +#define HIDDEN_SYMBOL(name) #define NO_EXEC_STACK_DIRECTIVE +#elif defined(__sparc__) + +#else + +#error Unsupported target + #endif #define DEFINE_LIBUNWIND_FUNCTION(name) \ .globl SYMBOL_NAME(name) SEPARATOR \ + EXPORT_SYMBOL(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ SYMBOL_NAME(name): #define DEFINE_LIBUNWIND_PRIVATE_FUNCTION(name) \ .globl SYMBOL_NAME(name) SEPARATOR \ - HIDDEN_DIRECTIVE SYMBOL_NAME(name) SEPARATOR \ + HIDDEN_SYMBOL(SYMBOL_NAME(name)) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ SYMBOL_NAME(name): #if defined(__arm__) #if !defined(__ARM_ARCH) #define __ARM_ARCH 4 #endif #if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5 #define ARM_HAS_BX #endif #ifdef ARM_HAS_BX #define JMP(r) bx r #else #define JMP(r) mov pc, r #endif #endif /* __arm__ */ #endif /* UNWIND_ASSEMBLY_H */ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/config.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/config.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/config.h (revision 345026) @@ -1,144 +1,184 @@ //===----------------------------- config.h -------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Defines macros used within libunwind project. // //===----------------------------------------------------------------------===// #ifndef LIBUNWIND_CONFIG_H #define LIBUNWIND_CONFIG_H #include #include +#include #include // Define static_assert() unless already defined by compiler. #ifndef __has_feature #define __has_feature(__x) 0 #endif #if !(__has_feature(cxx_static_assert)) && !defined(static_assert) #define static_assert(__b, __m) \ extern int compile_time_assert_failed[ ( __b ) ? 1 : -1 ] \ __attribute__( ( unused ) ); #endif // Platform specific configuration defines. #ifdef __APPLE__ #if defined(FOR_DYLD) - #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 1 - #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 0 - #define _LIBUNWIND_SUPPORT_DWARF_INDEX 0 + #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND #else - #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 1 + #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 1 - #define _LIBUNWIND_SUPPORT_DWARF_INDEX 0 #endif +#elif defined(_WIN32) + #ifdef __SEH__ + #define _LIBUNWIND_SUPPORT_SEH_UNWIND 1 + #else + #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 1 + #endif #else #if defined(__ARM_DWARF_EH__) || !defined(__arm__) - #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 0 #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 1 #define _LIBUNWIND_SUPPORT_DWARF_INDEX 1 + #endif +#endif + +#if defined(_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS) + #define _LIBUNWIND_EXPORT + #define _LIBUNWIND_HIDDEN +#else + #if !defined(__ELF__) && !defined(__MACH__) + #define _LIBUNWIND_EXPORT __declspec(dllexport) + #define _LIBUNWIND_HIDDEN #else - #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 0 - #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 0 - #define _LIBUNWIND_SUPPORT_DWARF_INDEX 0 + #define _LIBUNWIND_EXPORT __attribute__((visibility("default"))) + #define _LIBUNWIND_HIDDEN __attribute__((visibility("hidden"))) #endif #endif -// FIXME: these macros are not correct for COFF targets -#define _LIBUNWIND_EXPORT __attribute__((visibility("default"))) -#define _LIBUNWIND_HIDDEN __attribute__((visibility("hidden"))) - #if (defined(__APPLE__) && defined(__arm__)) || defined(__USING_SJLJ_EXCEPTIONS__) -#define _LIBUNWIND_BUILD_SJLJ_APIS 1 -#else -#define _LIBUNWIND_BUILD_SJLJ_APIS 0 +#define _LIBUNWIND_BUILD_SJLJ_APIS #endif -#if defined(__i386__) || defined(__x86_64__) -#define _LIBUNWIND_SUPPORT_FRAME_APIS 1 -#else -#define _LIBUNWIND_SUPPORT_FRAME_APIS 0 +#if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || defined(__ppc64__) || defined(__powerpc64__) +#define _LIBUNWIND_SUPPORT_FRAME_APIS #endif #if defined(__i386__) || defined(__x86_64__) || \ + defined(__ppc__) || defined(__ppc64__) || defined(__powerpc64__) || \ (!defined(__APPLE__) && defined(__arm__)) || \ (defined(__arm64__) || defined(__aarch64__)) || \ - (defined(__mips__)) || \ + defined(__mips__) || \ defined(__riscv) -#define _LIBUNWIND_BUILD_ZERO_COST_APIS 1 -#else -#define _LIBUNWIND_BUILD_ZERO_COST_APIS 0 +#if !defined(_LIBUNWIND_BUILD_SJLJ_APIS) +#define _LIBUNWIND_BUILD_ZERO_COST_APIS #endif +#endif +#if defined(__powerpc64__) && defined(_ARCH_PWR8) +#define PPC64_HAS_VMX +#endif + +#if defined(NDEBUG) && defined(_LIBUNWIND_IS_BAREMETAL) #define _LIBUNWIND_ABORT(msg) \ do { \ + abort(); \ + } while (0) +#else +#define _LIBUNWIND_ABORT(msg) \ + do { \ fprintf(stderr, "libunwind: %s %s:%d - %s\n", __func__, __FILE__, \ __LINE__, msg); \ fflush(stderr); \ abort(); \ } while (0) -#define _LIBUNWIND_LOG(msg, ...) fprintf(stderr, "libunwind: " msg "\n", __VA_ARGS__) +#endif +#if defined(NDEBUG) && defined(_LIBUNWIND_IS_BAREMETAL) +#define _LIBUNWIND_LOG0(msg) +#define _LIBUNWIND_LOG(msg, ...) +#else +#define _LIBUNWIND_LOG0(msg) \ + fprintf(stderr, "libunwind: " msg "\n") +#define _LIBUNWIND_LOG(msg, ...) \ + fprintf(stderr, "libunwind: " msg "\n", __VA_ARGS__) +#endif + +#if defined(NDEBUG) + #define _LIBUNWIND_LOG_IF_FALSE(x) x +#else + #define _LIBUNWIND_LOG_IF_FALSE(x) \ + do { \ + bool _ret = x; \ + if (!_ret) \ + _LIBUNWIND_LOG("" #x " failed in %s", __FUNCTION__); \ + } while (0) +#endif + // Macros that define away in non-Debug builds #ifdef NDEBUG #define _LIBUNWIND_DEBUG_LOG(msg, ...) #define _LIBUNWIND_TRACE_API(msg, ...) - #define _LIBUNWIND_TRACING_UNWINDING 0 + #define _LIBUNWIND_TRACING_UNWINDING (0) + #define _LIBUNWIND_TRACING_DWARF (0) #define _LIBUNWIND_TRACE_UNWINDING(msg, ...) - #define _LIBUNWIND_LOG_NON_ZERO(x) x + #define _LIBUNWIND_TRACE_DWARF(...) #else #ifdef __cplusplus extern "C" { #endif extern bool logAPIs(); extern bool logUnwinding(); + extern bool logDWARF(); #ifdef __cplusplus } #endif #define _LIBUNWIND_DEBUG_LOG(msg, ...) _LIBUNWIND_LOG(msg, __VA_ARGS__) - #define _LIBUNWIND_LOG_NON_ZERO(x) \ - do { \ - int _err = x; \ - if ( _err != 0 ) \ - _LIBUNWIND_LOG("" #x "=%d in %s", _err, __FUNCTION__); \ - } while (0) - #define _LIBUNWIND_TRACE_API(msg, ...) \ - do { \ - if ( logAPIs() ) _LIBUNWIND_LOG(msg, __VA_ARGS__); \ - } while(0) - #define _LIBUNWIND_TRACE_UNWINDING(msg, ...) \ - do { \ - if ( logUnwinding() ) _LIBUNWIND_LOG(msg, __VA_ARGS__); \ - } while(0) + #define _LIBUNWIND_TRACE_API(msg, ...) \ + do { \ + if (logAPIs()) \ + _LIBUNWIND_LOG(msg, __VA_ARGS__); \ + } while (0) #define _LIBUNWIND_TRACING_UNWINDING logUnwinding() + #define _LIBUNWIND_TRACING_DWARF logDWARF() + #define _LIBUNWIND_TRACE_UNWINDING(msg, ...) \ + do { \ + if (logUnwinding()) \ + _LIBUNWIND_LOG(msg, __VA_ARGS__); \ + } while (0) + #define _LIBUNWIND_TRACE_DWARF(...) \ + do { \ + if (logDWARF()) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) #endif #ifdef __cplusplus // Used to fit UnwindCursor and Registers_xxx types against unw_context_t / // unw_cursor_t sized memory blocks. #if defined(_LIBUNWIND_IS_NATIVE_ONLY) # define COMP_OP == #else -# define COMP_OP < +# define COMP_OP <= #endif template struct check_fit { template struct blk_count { static const size_t count = (sizeof(T) + sizeof(uint64_t) - 1) / sizeof(uint64_t); }; static const bool does_fit = (blk_count<_Type>::count COMP_OP blk_count<_Mem>::count); }; #undef COMP_OP #endif // __cplusplus #endif // LIBUNWIND_CONFIG_H Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/dwarf2.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/dwarf2.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/dwarf2.h (revision 345026) @@ -1,237 +1,240 @@ //===------------------------------- dwarf2.h -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /* These constants were taken from version 3 of the DWARF standard, which is Copyright (c) 2005 Free Standards Group, and Copyright (c) 1992, 1993 UNIX International, Inc. */ #ifndef __DWARF2__ #define __DWARF2__ // DWARF unwind instructions enum { DW_CFA_nop = 0x0, DW_CFA_set_loc = 0x1, DW_CFA_advance_loc1 = 0x2, DW_CFA_advance_loc2 = 0x3, DW_CFA_advance_loc4 = 0x4, DW_CFA_offset_extended = 0x5, DW_CFA_restore_extended = 0x6, DW_CFA_undefined = 0x7, DW_CFA_same_value = 0x8, DW_CFA_register = 0x9, DW_CFA_remember_state = 0xA, DW_CFA_restore_state = 0xB, DW_CFA_def_cfa = 0xC, DW_CFA_def_cfa_register = 0xD, DW_CFA_def_cfa_offset = 0xE, DW_CFA_def_cfa_expression = 0xF, DW_CFA_expression = 0x10, DW_CFA_offset_extended_sf = 0x11, DW_CFA_def_cfa_sf = 0x12, DW_CFA_def_cfa_offset_sf = 0x13, DW_CFA_val_offset = 0x14, DW_CFA_val_offset_sf = 0x15, DW_CFA_val_expression = 0x16, DW_CFA_advance_loc = 0x40, // high 2 bits are 0x1, lower 6 bits are delta DW_CFA_offset = 0x80, // high 2 bits are 0x2, lower 6 bits are register DW_CFA_restore = 0xC0, // high 2 bits are 0x3, lower 6 bits are register // GNU extensions DW_CFA_GNU_window_save = 0x2D, DW_CFA_GNU_args_size = 0x2E, - DW_CFA_GNU_negative_offset_extended = 0x2F + DW_CFA_GNU_negative_offset_extended = 0x2F, + + // AARCH64 extensions + DW_CFA_AARCH64_negate_ra_state = 0x2D }; // FSF exception handling Pointer-Encoding constants // Used in CFI augmentation by GCC enum { DW_EH_PE_ptr = 0x00, DW_EH_PE_uleb128 = 0x01, DW_EH_PE_udata2 = 0x02, DW_EH_PE_udata4 = 0x03, DW_EH_PE_udata8 = 0x04, DW_EH_PE_signed = 0x08, DW_EH_PE_sleb128 = 0x09, DW_EH_PE_sdata2 = 0x0A, DW_EH_PE_sdata4 = 0x0B, DW_EH_PE_sdata8 = 0x0C, DW_EH_PE_absptr = 0x00, DW_EH_PE_pcrel = 0x10, DW_EH_PE_textrel = 0x20, DW_EH_PE_datarel = 0x30, DW_EH_PE_funcrel = 0x40, DW_EH_PE_aligned = 0x50, DW_EH_PE_indirect = 0x80, DW_EH_PE_omit = 0xFF }; // DWARF expressions enum { DW_OP_addr = 0x03, // constant address (size target specific) DW_OP_deref = 0x06, DW_OP_const1u = 0x08, // 1-byte constant DW_OP_const1s = 0x09, // 1-byte constant DW_OP_const2u = 0x0A, // 2-byte constant DW_OP_const2s = 0x0B, // 2-byte constant DW_OP_const4u = 0x0C, // 4-byte constant DW_OP_const4s = 0x0D, // 4-byte constant DW_OP_const8u = 0x0E, // 8-byte constant DW_OP_const8s = 0x0F, // 8-byte constant DW_OP_constu = 0x10, // ULEB128 constant DW_OP_consts = 0x11, // SLEB128 constant DW_OP_dup = 0x12, DW_OP_drop = 0x13, DW_OP_over = 0x14, DW_OP_pick = 0x15, // 1-byte stack index DW_OP_swap = 0x16, DW_OP_rot = 0x17, DW_OP_xderef = 0x18, DW_OP_abs = 0x19, DW_OP_and = 0x1A, DW_OP_div = 0x1B, DW_OP_minus = 0x1C, DW_OP_mod = 0x1D, DW_OP_mul = 0x1E, DW_OP_neg = 0x1F, DW_OP_not = 0x20, DW_OP_or = 0x21, DW_OP_plus = 0x22, DW_OP_plus_uconst = 0x23, // ULEB128 addend DW_OP_shl = 0x24, DW_OP_shr = 0x25, DW_OP_shra = 0x26, DW_OP_xor = 0x27, DW_OP_skip = 0x2F, // signed 2-byte constant DW_OP_bra = 0x28, // signed 2-byte constant DW_OP_eq = 0x29, DW_OP_ge = 0x2A, DW_OP_gt = 0x2B, DW_OP_le = 0x2C, DW_OP_lt = 0x2D, DW_OP_ne = 0x2E, DW_OP_lit0 = 0x30, // Literal 0 DW_OP_lit1 = 0x31, // Literal 1 DW_OP_lit2 = 0x32, // Literal 2 DW_OP_lit3 = 0x33, // Literal 3 DW_OP_lit4 = 0x34, // Literal 4 DW_OP_lit5 = 0x35, // Literal 5 DW_OP_lit6 = 0x36, // Literal 6 DW_OP_lit7 = 0x37, // Literal 7 DW_OP_lit8 = 0x38, // Literal 8 DW_OP_lit9 = 0x39, // Literal 9 DW_OP_lit10 = 0x3A, // Literal 10 DW_OP_lit11 = 0x3B, // Literal 11 DW_OP_lit12 = 0x3C, // Literal 12 DW_OP_lit13 = 0x3D, // Literal 13 DW_OP_lit14 = 0x3E, // Literal 14 DW_OP_lit15 = 0x3F, // Literal 15 DW_OP_lit16 = 0x40, // Literal 16 DW_OP_lit17 = 0x41, // Literal 17 DW_OP_lit18 = 0x42, // Literal 18 DW_OP_lit19 = 0x43, // Literal 19 DW_OP_lit20 = 0x44, // Literal 20 DW_OP_lit21 = 0x45, // Literal 21 DW_OP_lit22 = 0x46, // Literal 22 DW_OP_lit23 = 0x47, // Literal 23 DW_OP_lit24 = 0x48, // Literal 24 DW_OP_lit25 = 0x49, // Literal 25 DW_OP_lit26 = 0x4A, // Literal 26 DW_OP_lit27 = 0x4B, // Literal 27 DW_OP_lit28 = 0x4C, // Literal 28 DW_OP_lit29 = 0x4D, // Literal 29 DW_OP_lit30 = 0x4E, // Literal 30 DW_OP_lit31 = 0x4F, // Literal 31 DW_OP_reg0 = 0x50, // Contents of reg0 DW_OP_reg1 = 0x51, // Contents of reg1 DW_OP_reg2 = 0x52, // Contents of reg2 DW_OP_reg3 = 0x53, // Contents of reg3 DW_OP_reg4 = 0x54, // Contents of reg4 DW_OP_reg5 = 0x55, // Contents of reg5 DW_OP_reg6 = 0x56, // Contents of reg6 DW_OP_reg7 = 0x57, // Contents of reg7 DW_OP_reg8 = 0x58, // Contents of reg8 DW_OP_reg9 = 0x59, // Contents of reg9 DW_OP_reg10 = 0x5A, // Contents of reg10 DW_OP_reg11 = 0x5B, // Contents of reg11 DW_OP_reg12 = 0x5C, // Contents of reg12 DW_OP_reg13 = 0x5D, // Contents of reg13 DW_OP_reg14 = 0x5E, // Contents of reg14 DW_OP_reg15 = 0x5F, // Contents of reg15 DW_OP_reg16 = 0x60, // Contents of reg16 DW_OP_reg17 = 0x61, // Contents of reg17 DW_OP_reg18 = 0x62, // Contents of reg18 DW_OP_reg19 = 0x63, // Contents of reg19 DW_OP_reg20 = 0x64, // Contents of reg20 DW_OP_reg21 = 0x65, // Contents of reg21 DW_OP_reg22 = 0x66, // Contents of reg22 DW_OP_reg23 = 0x67, // Contents of reg23 DW_OP_reg24 = 0x68, // Contents of reg24 DW_OP_reg25 = 0x69, // Contents of reg25 DW_OP_reg26 = 0x6A, // Contents of reg26 DW_OP_reg27 = 0x6B, // Contents of reg27 DW_OP_reg28 = 0x6C, // Contents of reg28 DW_OP_reg29 = 0x6D, // Contents of reg29 DW_OP_reg30 = 0x6E, // Contents of reg30 DW_OP_reg31 = 0x6F, // Contents of reg31 DW_OP_breg0 = 0x70, // base register 0 + SLEB128 offset DW_OP_breg1 = 0x71, // base register 1 + SLEB128 offset DW_OP_breg2 = 0x72, // base register 2 + SLEB128 offset DW_OP_breg3 = 0x73, // base register 3 + SLEB128 offset DW_OP_breg4 = 0x74, // base register 4 + SLEB128 offset DW_OP_breg5 = 0x75, // base register 5 + SLEB128 offset DW_OP_breg6 = 0x76, // base register 6 + SLEB128 offset DW_OP_breg7 = 0x77, // base register 7 + SLEB128 offset DW_OP_breg8 = 0x78, // base register 8 + SLEB128 offset DW_OP_breg9 = 0x79, // base register 9 + SLEB128 offset DW_OP_breg10 = 0x7A, // base register 10 + SLEB128 offset DW_OP_breg11 = 0x7B, // base register 11 + SLEB128 offset DW_OP_breg12 = 0x7C, // base register 12 + SLEB128 offset DW_OP_breg13 = 0x7D, // base register 13 + SLEB128 offset DW_OP_breg14 = 0x7E, // base register 14 + SLEB128 offset DW_OP_breg15 = 0x7F, // base register 15 + SLEB128 offset DW_OP_breg16 = 0x80, // base register 16 + SLEB128 offset DW_OP_breg17 = 0x81, // base register 17 + SLEB128 offset DW_OP_breg18 = 0x82, // base register 18 + SLEB128 offset DW_OP_breg19 = 0x83, // base register 19 + SLEB128 offset DW_OP_breg20 = 0x84, // base register 20 + SLEB128 offset DW_OP_breg21 = 0x85, // base register 21 + SLEB128 offset DW_OP_breg22 = 0x86, // base register 22 + SLEB128 offset DW_OP_breg23 = 0x87, // base register 23 + SLEB128 offset DW_OP_breg24 = 0x88, // base register 24 + SLEB128 offset DW_OP_breg25 = 0x89, // base register 25 + SLEB128 offset DW_OP_breg26 = 0x8A, // base register 26 + SLEB128 offset DW_OP_breg27 = 0x8B, // base register 27 + SLEB128 offset DW_OP_breg28 = 0x8C, // base register 28 + SLEB128 offset DW_OP_breg29 = 0x8D, // base register 29 + SLEB128 offset DW_OP_breg30 = 0x8E, // base register 30 + SLEB128 offset DW_OP_breg31 = 0x8F, // base register 31 + SLEB128 offset DW_OP_regx = 0x90, // ULEB128 register DW_OP_fbreg = 0x91, // SLEB128 offset DW_OP_bregx = 0x92, // ULEB128 register followed by SLEB128 offset DW_OP_piece = 0x93, // ULEB128 size of piece addressed DW_OP_deref_size = 0x94, // 1-byte size of data retrieved DW_OP_xderef_size = 0x95, // 1-byte size of data retrieved DW_OP_nop = 0x96, DW_OP_push_object_addres = 0x97, DW_OP_call2 = 0x98, // 2-byte offset of DIE DW_OP_call4 = 0x99, // 4-byte offset of DIE DW_OP_call_ref = 0x9A, // 4- or 8-byte offset of DIE DW_OP_lo_user = 0xE0, DW_OP_APPLE_uninit = 0xF0, DW_OP_hi_user = 0xFF }; #endif Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind.cpp =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind.cpp (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind.cpp (revision 345026) @@ -1,381 +1,412 @@ //===--------------------------- libunwind.cpp ----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Implements unw_* functions from // //===----------------------------------------------------------------------===// #include #ifndef NDEBUG #include // getenv #endif #include #include #include "libunwind_ext.h" #include "config.h" #include +#if !defined(__USING_SJLJ_EXCEPTIONS__) +#include "AddressSpace.hpp" #include "UnwindCursor.hpp" using namespace libunwind; /// internal object to represent this processes address space LocalAddressSpace LocalAddressSpace::sThisAddressSpace; _LIBUNWIND_EXPORT unw_addr_space_t unw_local_addr_space = (unw_addr_space_t)&LocalAddressSpace::sThisAddressSpace; /// record the registers and stack position of the caller extern int unw_getcontext(unw_context_t *); // note: unw_getcontext() implemented in assembly /// Create a cursor of a thread in this process given 'context' recorded by /// unw_getcontext(). _LIBUNWIND_EXPORT int unw_init_local(unw_cursor_t *cursor, unw_context_t *context) { _LIBUNWIND_TRACE_API("unw_init_local(cursor=%p, context=%p)", static_cast(cursor), static_cast(context)); #if defined(__i386__) # define REGISTER_KIND Registers_x86 #elif defined(__x86_64__) # define REGISTER_KIND Registers_x86_64 +#elif defined(__powerpc64__) +# define REGISTER_KIND Registers_ppc64 #elif defined(__ppc__) # define REGISTER_KIND Registers_ppc #elif defined(__aarch64__) # define REGISTER_KIND Registers_arm64 -#elif _LIBUNWIND_ARM_EHABI +#elif defined(__arm__) # define REGISTER_KIND Registers_arm #elif defined(__or1k__) # define REGISTER_KIND Registers_or1k #elif defined(__riscv) # define REGISTER_KIND Registers_riscv #elif defined(__mips__) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define REGISTER_KIND Registers_mips_o32 #elif defined(__mips64) # define REGISTER_KIND Registers_mips_newabi #elif defined(__mips__) # warning The MIPS architecture is not supported with this ABI and environment! +#elif defined(__sparc__) +# define REGISTER_KIND Registers_sparc #else # error Architecture not supported #endif // Use "placement new" to allocate UnwindCursor in the cursor buffer. new ((void *)cursor) UnwindCursor( context, LocalAddressSpace::sThisAddressSpace); #undef REGISTER_KIND AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; co->setInfoBasedOnIPRegister(); return UNW_ESUCCESS; } #ifdef UNW_REMOTE /// Create a cursor into a thread in another process. _LIBUNWIND_EXPORT int unw_init_remote_thread(unw_cursor_t *cursor, unw_addr_space_t as, void *arg) { // special case: unw_init_remote(xx, unw_local_addr_space, xx) if (as == (unw_addr_space_t)&LocalAddressSpace::sThisAddressSpace) return unw_init_local(cursor, NULL); //FIXME // use "placement new" to allocate UnwindCursor in the cursor buffer switch (as->cpuType) { case CPU_TYPE_I386: new ((void *)cursor) - UnwindCursor >, + UnwindCursor>, Registers_x86>(((unw_addr_space_i386 *)as)->oas, arg); break; case CPU_TYPE_X86_64: - new ((void *)cursor) UnwindCursor< - OtherAddressSpace >, Registers_x86_64>( - ((unw_addr_space_x86_64 *)as)->oas, arg); + new ((void *)cursor) + UnwindCursor>, + Registers_x86_64>(((unw_addr_space_x86_64 *)as)->oas, arg); break; case CPU_TYPE_POWERPC: new ((void *)cursor) - UnwindCursor >, Registers_ppc>( - ((unw_addr_space_ppc *)as)->oas, arg); + UnwindCursor>, + Registers_ppc>(((unw_addr_space_ppc *)as)->oas, arg); break; default: return UNW_EUNSPEC; } return UNW_ESUCCESS; } static bool is64bit(task_t task) { return false; // FIXME } /// Create an address_space object for use in examining another task. _LIBUNWIND_EXPORT unw_addr_space_t unw_create_addr_space_for_task(task_t task) { #if __i386__ if (is64bit(task)) { unw_addr_space_x86_64 *as = new unw_addr_space_x86_64(task); as->taskPort = task; as->cpuType = CPU_TYPE_X86_64; //as->oas } else { unw_addr_space_i386 *as = new unw_addr_space_i386(task); as->taskPort = task; as->cpuType = CPU_TYPE_I386; //as->oas } #else // FIXME #endif } /// Delete an address_space object. _LIBUNWIND_EXPORT void unw_destroy_addr_space(unw_addr_space_t asp) { switch (asp->cpuType) { #if __i386__ || __x86_64__ case CPU_TYPE_I386: { unw_addr_space_i386 *as = (unw_addr_space_i386 *)asp; delete as; } break; case CPU_TYPE_X86_64: { unw_addr_space_x86_64 *as = (unw_addr_space_x86_64 *)asp; delete as; } break; #endif case CPU_TYPE_POWERPC: { unw_addr_space_ppc *as = (unw_addr_space_ppc *)asp; delete as; } break; } } #endif // UNW_REMOTE /// Get value of specified register at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_get_reg(unw_cursor_t *cursor, unw_regnum_t regNum, unw_word_t *value) { _LIBUNWIND_TRACE_API("unw_get_reg(cursor=%p, regNum=%d, &value=%p)", static_cast(cursor), regNum, static_cast(value)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->validReg(regNum)) { *value = co->getReg(regNum); return UNW_ESUCCESS; } return UNW_EBADREG; } /// Set value of specified register at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_set_reg(unw_cursor_t *cursor, unw_regnum_t regNum, unw_word_t value) { - _LIBUNWIND_TRACE_API("unw_set_reg(cursor=%p, regNum=%d, value=0x%llX)", - static_cast(cursor), regNum, (long long)value); + _LIBUNWIND_TRACE_API("unw_set_reg(cursor=%p, regNum=%d, value=0x%" PRIxPTR ")", + static_cast(cursor), regNum, value); typedef LocalAddressSpace::pint_t pint_t; AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->validReg(regNum)) { co->setReg(regNum, (pint_t)value); // specical case altering IP to re-find info (being called by personality // function) - if (regNum == UNW_REG_IP) + if (regNum == UNW_REG_IP) { + unw_proc_info_t info; + // First, get the FDE for the old location and then update it. + co->getInfo(&info); co->setInfoBasedOnIPRegister(false); + // If the original call expects stack adjustment, perform this now. + // Normal frame unwinding would have included the offset already in the + // CFA computation. + // Note: for PA-RISC and other platforms where the stack grows up, + // this should actually be - info.gp. LLVM doesn't currently support + // any such platforms and Clang doesn't export a macro for them. + if (info.gp) + co->setReg(UNW_REG_SP, co->getReg(UNW_REG_SP) + info.gp); + } return UNW_ESUCCESS; } return UNW_EBADREG; } /// Get value of specified float register at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_get_fpreg(unw_cursor_t *cursor, unw_regnum_t regNum, unw_fpreg_t *value) { _LIBUNWIND_TRACE_API("unw_get_fpreg(cursor=%p, regNum=%d, &value=%p)", static_cast(cursor), regNum, static_cast(value)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->validFloatReg(regNum)) { *value = co->getFloatReg(regNum); return UNW_ESUCCESS; } return UNW_EBADREG; } /// Set value of specified float register at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_set_fpreg(unw_cursor_t *cursor, unw_regnum_t regNum, unw_fpreg_t value) { -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) _LIBUNWIND_TRACE_API("unw_set_fpreg(cursor=%p, regNum=%d, value=%llX)", static_cast(cursor), regNum, value); #else _LIBUNWIND_TRACE_API("unw_set_fpreg(cursor=%p, regNum=%d, value=%g)", static_cast(cursor), regNum, value); #endif AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->validFloatReg(regNum)) { co->setFloatReg(regNum, value); return UNW_ESUCCESS; } return UNW_EBADREG; } /// Move cursor to next frame. _LIBUNWIND_EXPORT int unw_step(unw_cursor_t *cursor) { _LIBUNWIND_TRACE_API("unw_step(cursor=%p)", static_cast(cursor)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; return co->step(); } /// Get unwind info at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_get_proc_info(unw_cursor_t *cursor, unw_proc_info_t *info) { _LIBUNWIND_TRACE_API("unw_get_proc_info(cursor=%p, &info=%p)", static_cast(cursor), static_cast(info)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; co->getInfo(info); if (info->end_ip == 0) return UNW_ENOINFO; else return UNW_ESUCCESS; } /// Resume execution at cursor position (aka longjump). _LIBUNWIND_EXPORT int unw_resume(unw_cursor_t *cursor) { _LIBUNWIND_TRACE_API("unw_resume(cursor=%p)", static_cast(cursor)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; co->jumpto(); return UNW_EUNSPEC; } /// Get name of function at cursor position in stack frame. _LIBUNWIND_EXPORT int unw_get_proc_name(unw_cursor_t *cursor, char *buf, size_t bufLen, unw_word_t *offset) { _LIBUNWIND_TRACE_API("unw_get_proc_name(cursor=%p, &buf=%p, bufLen=%lu)", static_cast(cursor), static_cast(buf), static_cast(bufLen)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->getFunctionName(buf, bufLen, offset)) return UNW_ESUCCESS; else return UNW_EUNSPEC; } /// Checks if a register is a floating-point register. _LIBUNWIND_EXPORT int unw_is_fpreg(unw_cursor_t *cursor, unw_regnum_t regNum) { _LIBUNWIND_TRACE_API("unw_is_fpreg(cursor=%p, regNum=%d)", static_cast(cursor), regNum); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; return co->validFloatReg(regNum); } /// Checks if a register is a floating-point register. _LIBUNWIND_EXPORT const char *unw_regname(unw_cursor_t *cursor, unw_regnum_t regNum) { _LIBUNWIND_TRACE_API("unw_regname(cursor=%p, regNum=%d)", static_cast(cursor), regNum); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; return co->getRegisterName(regNum); } /// Checks if current frame is signal trampoline. _LIBUNWIND_EXPORT int unw_is_signal_frame(unw_cursor_t *cursor) { _LIBUNWIND_TRACE_API("unw_is_signal_frame(cursor=%p)", static_cast(cursor)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; return co->isSignalFrame(); } #ifdef __arm__ // Save VFP registers d0-d15 using FSTMIADX instead of FSTMIADD _LIBUNWIND_EXPORT void unw_save_vfp_as_X(unw_cursor_t *cursor) { _LIBUNWIND_TRACE_API("unw_fpreg_save_vfp_as_X(cursor=%p)", static_cast(cursor)); AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; return co->saveVFPAsX(); } #endif -#if _LIBUNWIND_SUPPORT_DWARF_UNWIND -/// SPI: walks cached dwarf entries +#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) +/// SPI: walks cached DWARF entries _LIBUNWIND_EXPORT void unw_iterate_dwarf_unwind_cache(void (*func)( unw_word_t ip_start, unw_word_t ip_end, unw_word_t fde, unw_word_t mh)) { _LIBUNWIND_TRACE_API("unw_iterate_dwarf_unwind_cache(func=%p)", reinterpret_cast(func)); DwarfFDECache::iterateCacheEntries(func); } /// IPI: for __register_frame() void _unw_add_dynamic_fde(unw_word_t fde) { CFI_Parser::FDE_Info fdeInfo; CFI_Parser::CIE_Info cieInfo; const char *message = CFI_Parser::decodeFDE( LocalAddressSpace::sThisAddressSpace, (LocalAddressSpace::pint_t) fde, &fdeInfo, &cieInfo); if (message == NULL) { // dynamically registered FDEs don't have a mach_header group they are in. // Use fde as mh_group unw_word_t mh_group = fdeInfo.fdeStart; DwarfFDECache::add((LocalAddressSpace::pint_t)mh_group, fdeInfo.pcStart, fdeInfo.pcEnd, fdeInfo.fdeStart); } else { _LIBUNWIND_DEBUG_LOG("_unw_add_dynamic_fde: bad fde: %s", message); } } /// IPI: for __deregister_frame() void _unw_remove_dynamic_fde(unw_word_t fde) { // fde is own mh_group DwarfFDECache::removeAllIn((LocalAddressSpace::pint_t)fde); } -#endif // _LIBUNWIND_SUPPORT_DWARF_UNWIND +#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) +#endif // !defined(__USING_SJLJ_EXCEPTIONS__) // Add logging hooks in Debug builds only #ifndef NDEBUG #include _LIBUNWIND_HIDDEN bool logAPIs() { // do manual lock to avoid use of _cxa_guard_acquire or initializers static bool checked = false; static bool log = false; if (!checked) { log = (getenv("LIBUNWIND_PRINT_APIS") != NULL); checked = true; } return log; } _LIBUNWIND_HIDDEN bool logUnwinding() { // do manual lock to avoid use of _cxa_guard_acquire or initializers static bool checked = false; static bool log = false; if (!checked) { log = (getenv("LIBUNWIND_PRINT_UNWINDING") != NULL); + checked = true; + } + return log; +} + +_LIBUNWIND_HIDDEN +bool logDWARF() { + // do manual lock to avoid use of _cxa_guard_acquire or initializers + static bool checked = false; + static bool log = false; + if (!checked) { + log = (getenv("LIBUNWIND_PRINT_DWARF") != NULL); checked = true; } return log; } #endif // NDEBUG Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind_ext.h =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind_ext.h (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind/src/libunwind_ext.h (revision 345026) @@ -1,47 +1,47 @@ //===------------------------ libunwind_ext.h -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // // // Extensions to libunwind API. // //===----------------------------------------------------------------------===// #ifndef __LIBUNWIND_EXT__ #define __LIBUNWIND_EXT__ #include "config.h" #include #include #define UNW_STEP_SUCCESS 1 #define UNW_STEP_END 0 #ifdef __cplusplus extern "C" { #endif // SPI extern void unw_iterate_dwarf_unwind_cache(void (*func)(unw_word_t ip_start, unw_word_t ip_end, unw_word_t fde, unw_word_t mh)); // IPI extern void _unw_add_dynamic_fde(unw_word_t fde); extern void _unw_remove_dynamic_fde(unw_word_t fde); -#if _LIBUNWIND_ARM_EHABI +#if defined(_LIBUNWIND_ARM_EHABI) extern const uint32_t* decode_eht_entry(const uint32_t*, size_t*, size_t*); extern _Unwind_Reason_Code _Unwind_VRS_Interpret(_Unwind_Context *context, const uint32_t *data, size_t offset, size_t len); #endif #ifdef __cplusplus } #endif #endif // __LIBUNWIND_EXT__ Index: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind (revision 345026) Property changes on: projects/import-googletest-1.8.1/contrib/llvm/projects/libunwind ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,3 ## Merged /vendor/llvm-libunwind/dist-release_80:r344939-345018 Merged /head/contrib/llvm/projects/libunwind:r344081-345025 Merged /vendor/llvm-libunwind/dist:r288151-302346,302348-344967 Index: projects/import-googletest-1.8.1/contrib/llvm =================================================================== --- projects/import-googletest-1.8.1/contrib/llvm (revision 345025) +++ projects/import-googletest-1.8.1/contrib/llvm (revision 345026) Property changes on: projects/import-googletest-1.8.1/contrib/llvm ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm:r344997-345025 Index: projects/import-googletest-1.8.1/libexec/rc/rc.d/growfs =================================================================== --- projects/import-googletest-1.8.1/libexec/rc/rc.d/growfs (revision 345025) +++ projects/import-googletest-1.8.1/libexec/rc/rc.d/growfs (revision 345026) @@ -1,118 +1,118 @@ #!/bin/sh # # Copyright 2014 John-Mark Gurney # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # # PROVIDE: growfs # BEFORE: sysctl # KEYWORD: firstboot # This allows us to distribute a image # and have it work on essentially any size drive. # # TODO: Figure out where this should really be ordered. # I suspect it should go just after fsck but before mountcritlocal. # . /etc/rc.subr name="growfs" desc="Grow root partition to fill device" start_cmd="growfs_start" stop_cmd=":" rcvar="growfs_enable" growfs_start () { echo "Growing root partition to fill device" FSTYPE=$(mount -p | awk '{ if ( $2 == "/") { print $3 }}') FSDEV=$(mount -p | awk '{ if ( $2 == "/") { print $1 }}') case "$FSTYPE" in ufs) rootdev=${FSDEV#/dev/} ;; zfs) pool=${FSDEV%%/*} - rootdev=$(zpool list -v $pool | tail -n 1 | awk '{ print $1 }') + rootdev=$(zpool list -v $pool | awk 'END { print $1 }') ;; *) echo "Don't know how to grow root filesystem type: $FSTYPE" return esac if [ x"$rootdev" = x"${rootdev%/*}" ]; then # raw device rawdev="$rootdev" else rawdev=$(glabel status | awk '$1 == "'"$rootdev"'" { print $3 }') if [ x"$rawdev" = x"" ]; then echo "Can't figure out device for: $rootdev" return fi fi sysctl -b kern.geom.conftxt | awk ' { lvl=$1 device[lvl] = $3 type[lvl] = $2 idx[lvl] = $7 parttype[lvl] = $13 if (dev == $3) { for (i = 1; i <= lvl; i++) { # resize if (type[i] == "PART") { pdev = device[i - 1] cmd[i] = "gpart resize -i " idx[i] " " pdev if (parttype[i] == "GPT") cmd[i] = "gpart recover " pdev " ; " cmd[i] } else if (type[i] == "LABEL") { continue } else { print "unhandled type: " type[i] exit 1 } } for (i = 1; i <= lvl; i++) { if (cmd[i]) system(cmd[i]) } exit 0 } }' dev="$rawdev" gpart commit "$rootdev" case "$FSTYPE" in ufs) growfs -y /dev/"$rootdev" ;; zfs) zpool online -e $pool $rootdev ;; esac } load_rc_config $name run_rc_command "$1" Index: projects/import-googletest-1.8.1/share/misc/organization.dot =================================================================== --- projects/import-googletest-1.8.1/share/misc/organization.dot (revision 345025) +++ projects/import-googletest-1.8.1/share/misc/organization.dot (revision 345026) @@ -1,99 +1,97 @@ # $FreeBSD$ # This file is meant to show the infrastructural organization of the # FreeBSD Project; what kind of teams we have and how they relate to # each other. # For a detailed description of the responsibilities and duties of the listed # teams, please see our Administration page at # https://www.freebsd.org/administration.html . # # The graphical output can be generated from this file with the following # command: # $ dot -T png -o file.png organization.dot # # The dot binary is part of the graphics/graphviz port. digraph org { node [color=lightblue2, style=filled, bgcolor=black]; # Meta-categories go here _devel [label="FreeBSD Developers"] _admin [label="FreeBSD Infrastructure Administrators"] _misc [label="Miscellaneous Hats"] # Development teams go here alphabetically sorted core [label="Core Team\ncore@FreeBSD.org\nallanjude, bcr, brooks,\nimp, hrs, jeff,\njhb, kmoore, seanc"] coresecretary [label="Core Team Secretary\ncore-secretary@FreeBSD.org\njrm"] doccommitters [label="Doc/www Committers\ndoc-committers@FreeBSD.org"] doceng [label="Documentation Engineering Team\ndoceng@FreeBSD.org\nbcr, gabor, gjb, hrs,\nblackend, ryusuke, wblock"] portscommitters [label="Ports Committers\nports-committers@FreeBSD.org"] portmgr [label="Port Management Team\nportmgr@FreeBSD.org\nadamw, antoine, bapt, bdrewery\nfeld, mat, rene, swills"] portmgrsecretary [label="Port Management Team Secretary\nportmgr-secretary@FreeBSD.org\nrene"] -re [label="Primary Release Engineering Team\nre@FreeBSD.org\ngjb, kib,\nbdrewery, blackend,\nrgrimes, delphij,\nhrs, glebius,\nmarius, rwatson"] +re [label="Primary Release Engineering Team\nre@FreeBSD.org\ngjb, kib,\nbdrewery, blackend,\nrgrimes, delphij,\nhrs, glebius,\nmarius"] secteam [label="Security Team\nsecteam@FreeBSD.org\nbenno, delphij,\ndes, emaste,\ngjb, gordon,\nremko"] portssecteam [label="Ports Security Team\nports-secteam@FreeBSD.org\ndelphij, amdmi3, eadler, feld, jgh, rea, riggs, sbz, simon, swills, zi"] secteamsecretary [label="Security Team Secretary\nsecteam-secretary@FreeBSD.org\nremko"] securityofficer [label="Security Officer Team\nsecurity-officer@FreeBSD.org\nbenno, delphij,\ndes, emaste,\ngjb, gordon,\nremko"] srccommitters [label="Src Committers\nsrc-committers@FreeBSD.org"] # Admin teams go here alphabetically sorted -accounts [label="Accounts Team\naccounts@FreeBSD.org\nmarkm, simon, kensmith,\ndhw"] -backups [label="Backup Administrators\nbackups@FreeBSD.org\nsimon, kensmith,\ndhw"] +accounts [label="Accounts Team\naccounts@FreeBSD.org\nclusteradm"] +backups [label="Backup Administrators\nbackups@FreeBSD.org\nclusteradm"] bugmeister [label="Bugmeister Team\nbugmeister@FreeBSD.org\neadler, gavin, gonzo"] clusteradm [label="Cluster Administrators\nclusteradm@FreeBSD.org\nallanjude, brd,\ndhw, gavin,\ngjb, peter,\nsbruno, simon,\nzi"] -dnsadm [label="DNS Administrators\ndnsadm@FreeBSD.org\nbillf, dg, ps,\nkensmith, peter"] -mirroradmin [label="FTP/WWW Mirror Site Coordinators\nmirror-admin@FreeBSD.org\nkuriyama, kensmith"] +dnsadm [label="DNS Administrators\ndnsadm@FreeBSD.org\nclusteradm"] +mirroradmin [label="FTP/WWW Mirror Site Coordinators\nmirror-admin@FreeBSD.org\nclusteradm,\nkuriyama"] perforceadmin [label="Perforce Repository Administrators\nperforce-admin@FreeBSD.org\nscottl, kensmith, gordon,\nrwatson, peter, dhw"] postmaster [label="Postmaster Team\npostmaster@FreeBSD.org\ndhw, krion, ler, philip, pi, rea, remko, zi"] -refadm [label="Reference Systems Administrators\nrefadm@FreeBSD.org\njake, billf, markm, simon,\nobrien, ps, kensmith,\npeter, dhw"] webmaster [label="Webmaster Team\nwebmaster@FreeBSD.org\ngjb, wblock, blackend,\ngabor, hrs, wosch"] # Misc hats go here alphabetically sorted donations [label="Donations Team\ndonations@FreeBSD.org\nwilko, gahr, pgolluci,\nobrien, ds,\nrwatson"] marketing [label="Marketing Team\nmarketing@FreeBSD.org\nSteven Beedle, Denise Ebery, deb,\njkoshy, dru, mwlucas, imp,\nKris Moore, murray, mattt,\nJeremy C. Reed, rwatson"] vendorrelations [label="Vendor Relations\nvendor-relations@FreeBSD.org\ncore, FreeBSD Foundation"] # Here are the team relationships. # Group together all the entries for the superior team. # Keep the list sorted by the superior team entry. _admin -> accounts _admin -> backups _admin -> bugmeister _admin -> clusteradm _admin -> dnsadm _admin -> mirroradmin _admin -> perforceadmin -_admin -> refadm _admin -> postmaster _admin -> webmaster _devel -> core _misc -> donations _misc -> marketing _misc -> vendorrelations core -> coresecretary core -> doceng core -> portmgr core -> re core -> securityofficer core -> srccommitters doceng -> doccommitters portmgr -> portmgrsecretary portmgr -> portscommitters securityofficer -> secteam securityofficer -> portssecteam secteam -> secteamsecretary } Index: projects/import-googletest-1.8.1/sys/arm/freescale/imx/imx_spi.c =================================================================== --- projects/import-googletest-1.8.1/sys/arm/freescale/imx/imx_spi.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/arm/freescale/imx/imx_spi.c (revision 345026) @@ -1,612 +1,613 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 Ian Lepore * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Driver for imx Enhanced Configurable SPI; master-mode only. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "spibus_if.h" #define ECSPI_RXDATA 0x00 #define ECSPI_TXDATA 0x04 #define ECSPI_CTLREG 0x08 #define CTLREG_BLEN_SHIFT 20 #define CTLREG_BLEN_MASK 0x0fff #define CTLREG_CSEL_SHIFT 18 #define CTLREG_CSEL_MASK 0x03 #define CTLREG_DRCTL_SHIFT 16 #define CTLREG_DRCTL_MASK 0x03 #define CTLREG_PREDIV_SHIFT 12 #define CTLREG_PREDIV_MASK 0x0f #define CTLREG_POSTDIV_SHIFT 8 #define CTLREG_POSTDIV_MASK 0x0f #define CTLREG_CMODE_SHIFT 4 #define CTLREG_CMODE_MASK 0x0f #define CTLREG_CMODES_MASTER (CTLREG_CMODE_MASK << CTLREG_CMODE_SHIFT) #define CTLREG_SMC (1u << 3) #define CTLREG_XCH (1u << 2) #define CTLREG_HT (1u << 1) #define CTLREG_EN (1u << 0) #define ECSPI_CFGREG 0x0c #define CFGREG_HTLEN_SHIFT 24 #define CFGREG_SCLKCTL_SHIFT 20 #define CFGREG_DATACTL_SHIFT 16 #define CFGREG_SSPOL_SHIFT 12 #define CFGREG_SSCTL_SHIFT 8 #define CFGREG_SCLKPOL_SHIFT 4 #define CFGREG_SCLKPHA_SHIFT 0 #define CFGREG_MASK 0x0f /* all CFGREG fields are 4 bits */ #define ECSPI_INTREG 0x10 #define INTREG_TCEN (1u << 7) #define INTREG_ROEN (1u << 6) #define INTREG_RFEN (1u << 5) #define INTREG_RDREN (1u << 4) #define INTREG_RREN (1u << 3) #define INTREG_TFEN (1u << 2) #define INTREG_TDREN (1u << 1) #define INTREG_TEEN (1u << 0) #define ECSPI_DMAREG 0x14 #define DMA_RX_THRESH_SHIFT 16 #define DMA_RX_THRESH_MASK 0x3f #define DMA_TX_THRESH_SHIFT 0 #define DMA_TX_THRESH_MASK 0x3f #define ECSPI_STATREG 0x18 #define SREG_TC (1u << 7) #define SREG_RO (1u << 6) #define SREG_RF (1u << 5) #define SREG_RDR (1u << 4) #define SREG_RR (1u << 3) #define SREG_TF (1u << 2) #define SREG_TDR (1u << 1) #define SREG_TE (1u << 0) #define ECSPI_PERIODREG 0x1c #define ECSPI_TESTREG 0x20 #define CS_MAX 4 /* Max number of chip selects. */ #define CS_MASK 0x03 /* Mask flag bits out of chipsel. */ #define FIFO_SIZE 64 #define FIFO_RXTHRESH 32 #define FIFO_TXTHRESH 32 struct spi_softc { device_t dev; device_t spibus; struct mtx mtx; struct resource *memres; struct resource *intres; void *inthandle; gpio_pin_t cspins[CS_MAX]; u_int debug; u_int basefreq; uint32_t ctlreg; uint32_t intreg; uint32_t fifocnt; uint8_t *rxbuf; uint32_t rxidx; uint32_t rxlen; uint8_t *txbuf; uint32_t txidx; uint32_t txlen; }; static struct ofw_compat_data compat_data[] = { {"fsl,imx51-ecspi", true}, {"fsl,imx53-ecspi", true}, {"fsl,imx6dl-ecspi", true}, {"fsl,imx6q-ecspi", true}, {"fsl,imx6sx-ecspi", true}, {"fsl,imx6ul-ecspi", true}, {NULL, false} }; static inline uint32_t RD4(struct spi_softc *sc, bus_size_t offset) { return (bus_read_4(sc->memres, offset)); } static inline void WR4(struct spi_softc *sc, bus_size_t offset, uint32_t value) { bus_write_4(sc->memres, offset, value); } static u_int spi_calc_clockdiv(struct spi_softc *sc, u_int busfreq) { u_int post, pre; /* Returning 0 effectively sets both dividers to 1. */ if (sc->basefreq <= busfreq) return (0); /* * Brute-force this; all real-world bus speeds are going to be found on * the 1st or 2nd time through this loop. */ for (post = 0; post < 16; ++post) { pre = ((sc->basefreq >> post) / busfreq) - 1; if (pre < 16) break; } if (post == 16) { /* The lowest we can go is ~115 Hz. */ pre = 15; post = 15; } if (sc->debug >= 2) { device_printf(sc->dev, "base %u bus %u; pre %u, post %u; actual busfreq %u\n", sc->basefreq, busfreq, pre, post, (sc->basefreq / (pre + 1)) / (1 << post)); } return (pre << CTLREG_PREDIV_SHIFT) | (post << CTLREG_POSTDIV_SHIFT); } static void spi_set_chipsel(struct spi_softc *sc, u_int cs, bool active) { bool pinactive; /* * This is kinda crazy... the gpio pins for chipsel are defined as * active-high in the dts, but are supposed to be treated as active-low * by this driver. So to turn on chipsel we have to invert the value * passed to gpio_pin_set_active(). Then, to make it more fun, any * slave can say its chipsel is active-high, so if that option is * on, we have to invert the value again. */ pinactive = !active ^ (bool)(cs & SPIBUS_CS_HIGH); if (sc->debug >= 2) { device_printf(sc->dev, "chipsel %u changed to %u\n", (cs & ~SPIBUS_CS_HIGH), pinactive); } /* * Change the pin, then do a dummy read of its current state to ensure * that the state change reaches the hardware before proceeding. */ gpio_pin_set_active(sc->cspins[cs & ~SPIBUS_CS_HIGH], pinactive); gpio_pin_is_active(sc->cspins[cs & ~SPIBUS_CS_HIGH], &pinactive); } static void spi_hw_setup(struct spi_softc *sc, u_int cs, u_int mode, u_int freq) { uint32_t reg; /* * Set up control register, and write it first to bring the device out * of reset. */ sc->ctlreg = CTLREG_EN | CTLREG_CMODES_MASTER | CTLREG_SMC; sc->ctlreg |= spi_calc_clockdiv(sc, freq); sc->ctlreg |= 7 << CTLREG_BLEN_SHIFT; /* XXX byte at a time */ WR4(sc, ECSPI_CTLREG, sc->ctlreg); /* * Set up the config register. Note that we do all transfers with the * SPI hardware's chip-select set to zero. The actual chip select is * handled with a gpio pin. */ reg = 0; if (cs & SPIBUS_CS_HIGH) reg |= 1u << CFGREG_SSPOL_SHIFT; if (mode & SPIBUS_MODE_CPHA) reg |= 1u << CFGREG_SCLKPHA_SHIFT; if (mode & SPIBUS_MODE_CPOL) { reg |= 1u << CFGREG_SCLKPOL_SHIFT; reg |= 1u << CFGREG_SCLKCTL_SHIFT; } WR4(sc, ECSPI_CFGREG, reg); /* * Set up the rx/tx FIFO interrupt thresholds. */ reg = (FIFO_RXTHRESH << DMA_RX_THRESH_SHIFT); reg |= (FIFO_TXTHRESH << DMA_TX_THRESH_SHIFT); WR4(sc, ECSPI_DMAREG, reg); /* * Do a dummy read, to make sure the preceding writes reach the spi * hardware before we assert any gpio chip select. */ (void)RD4(sc, ECSPI_CFGREG); } static void spi_empty_rxfifo(struct spi_softc *sc) { while (sc->rxidx < sc->rxlen && (RD4(sc, ECSPI_STATREG) & SREG_RR)) { sc->rxbuf[sc->rxidx++] = (uint8_t)RD4(sc, ECSPI_RXDATA); --sc->fifocnt; } } static void spi_fill_txfifo(struct spi_softc *sc) { while (sc->txidx < sc->txlen && sc->fifocnt < FIFO_SIZE) { WR4(sc, ECSPI_TXDATA, sc->txbuf[sc->txidx++]); ++sc->fifocnt; } /* * If we're out of data, disable tx data ready (threshold) interrupts, * and enable tx fifo empty interrupts. */ if (sc->txidx == sc->txlen) sc->intreg = (sc->intreg & ~INTREG_TDREN) | INTREG_TEEN; } static void spi_intr(void *arg) { struct spi_softc *sc = arg; uint32_t intreg, status; mtx_lock(&sc->mtx); sc = arg; intreg = sc->intreg; status = RD4(sc, ECSPI_STATREG); WR4(sc, ECSPI_STATREG, status); /* Clear w1c bits. */ /* * If we get an overflow error, just signal that the transfer is done * and wakeup the waiting thread, which will see that txidx != txlen and * return an IO error to the caller. */ if (__predict_false(status & SREG_RO)) { if (sc->debug || bootverbose) { device_printf(sc->dev, "rxoverflow rxidx %u txidx %u\n", sc->rxidx, sc->txidx); } sc->intreg = 0; wakeup(sc); mtx_unlock(&sc->mtx); return; } if (status & SREG_RR) spi_empty_rxfifo(sc); if (status & SREG_TDR) spi_fill_txfifo(sc); /* * If we're out of bytes to send... * - If Transfer Complete is set (shift register is empty) and we've * received everything we expect, we're all done. * - Else if Tx Fifo Empty is set, we need to stop waiting for that and * switch to waiting for Transfer Complete (wait for shift register * to empty out), and also for Receive Ready (last of incoming data). */ if (sc->txidx == sc->txlen) { if ((status & SREG_TC) && sc->fifocnt == 0) { sc->intreg = 0; wakeup(sc); } else if (status & SREG_TE) { sc->intreg &= ~(sc->intreg & ~INTREG_TEEN); sc->intreg |= INTREG_TCEN | INTREG_RREN; } } /* * If interrupt flags changed, write the new flags to the hardware and * do a dummy readback to ensure the changes reach the hardware before * we exit the isr. */ if (sc->intreg != intreg) { WR4(sc, ECSPI_INTREG, sc->intreg); (void)RD4(sc, ECSPI_INTREG); } if (sc->debug >= 3) { device_printf(sc->dev, "spi_intr, sreg 0x%08x intreg was 0x%08x now 0x%08x\n", status, intreg, sc->intreg); } mtx_unlock(&sc->mtx); } static int spi_xfer_buf(struct spi_softc *sc, void *rxbuf, void *txbuf, uint32_t len) { int err; if (sc->debug >= 1) { device_printf(sc->dev, "spi_xfer_buf, rxbuf %p txbuf %p len %u\n", rxbuf, txbuf, len); } if (len == 0) return (0); sc->rxbuf = rxbuf; sc->rxlen = len; sc->rxidx = 0; sc->txbuf = txbuf; sc->txlen = len; sc->txidx = 0; sc->intreg = INTREG_RDREN | INTREG_TDREN; spi_fill_txfifo(sc); /* Enable interrupts last; spi_fill_txfifo() can change sc->intreg */ WR4(sc, ECSPI_INTREG, sc->intreg); err = 0; while (err == 0 && sc->intreg != 0) err = msleep(sc, &sc->mtx, 0, "imxspi", 10 * hz); if (sc->rxidx != sc->rxlen || sc->txidx != sc->txlen) err = EIO; return (err); } static int spi_transfer(device_t dev, device_t child, struct spi_command *cmd) { struct spi_softc *sc = device_get_softc(dev); uint32_t cs, mode, clock; int err; spibus_get_cs(child, &cs); spibus_get_clock(child, &clock); spibus_get_mode(child, &mode); if (cs > CS_MAX || sc->cspins[cs] == NULL) { if (sc->debug || bootverbose) device_printf(sc->dev, "Invalid chip select %u\n", cs); return (EINVAL); } mtx_lock(&sc->mtx); + device_busy(sc->dev); if (sc->debug >= 1) { device_printf(sc->dev, "spi_transfer, cs 0x%x clock %u mode %u\n", cs, clock, mode); } /* Set up the hardware and select the device. */ spi_hw_setup(sc, cs, mode, clock); spi_set_chipsel(sc, cs, true); /* Transfer command then data bytes. */ err = 0; if (cmd->tx_cmd_sz > 0) err = spi_xfer_buf(sc, cmd->rx_cmd, cmd->tx_cmd, cmd->tx_cmd_sz); if (cmd->tx_data_sz > 0 && err == 0) err = spi_xfer_buf(sc, cmd->rx_data, cmd->tx_data, cmd->tx_data_sz); /* Deselect the device, turn off (and reset) hardware. */ spi_set_chipsel(sc, cs, false); WR4(sc, ECSPI_CTLREG, 0); + device_unbusy(sc->dev); mtx_unlock(&sc->mtx); return (err); } static phandle_t spi_get_node(device_t bus, device_t dev) { /* * Share our controller node with our spibus child; it instantiates * devices by walking the children contained within our node. */ return ofw_bus_get_node(bus); } static int spi_detach(device_t dev) { struct spi_softc *sc = device_get_softc(dev); - int idx; + int error, idx; - mtx_lock(&sc->mtx); + if ((error = bus_generic_detach(sc->dev)) != 0) + return (error); - bus_generic_detach(sc->dev); if (sc->spibus != NULL) device_delete_child(dev, sc->spibus); for (idx = 0; idx < nitems(sc->cspins); ++idx) { if (sc->cspins[idx] != NULL) gpio_pin_release(sc->cspins[idx]); } if (sc->inthandle != NULL) bus_teardown_intr(sc->dev, sc->intres, sc->inthandle); if (sc->intres != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->intres); if (sc->memres != NULL) bus_release_resource(sc->dev, SYS_RES_MEMORY, 0, sc->memres); - mtx_unlock(&sc->mtx); mtx_destroy(&sc->mtx); return (0); } static int spi_attach(device_t dev) { struct spi_softc *sc = device_get_softc(dev); phandle_t node; int err, idx, rid; sc->dev = dev; sc->basefreq = imx_ccm_ecspi_hz(); mtx_init(&sc->mtx, device_get_nameunit(dev), NULL, MTX_DEF); /* Set up debug-enable sysctl. */ SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->debug, 0, "Enable debug, higher values = more info"); /* Allocate mmio register access resources. */ rid = 0; sc->memres = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->memres == NULL) { device_printf(sc->dev, "could not allocate registers\n"); spi_detach(sc->dev); return (ENXIO); } /* Allocate interrupt resources and set up handler. */ rid = 0; sc->intres = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->intres == NULL) { device_printf(sc->dev, "could not allocate interrupt\n"); device_detach(sc->dev); return (ENXIO); } err = bus_setup_intr(sc->dev, sc->intres, INTR_TYPE_MISC | INTR_MPSAFE, NULL, spi_intr, sc, &sc->inthandle); if (err != 0) { device_printf(sc->dev, "could not setup interrupt handler"); device_detach(sc->dev); return (ENXIO); } /* Allocate gpio pins for configured chip selects. */ node = ofw_bus_get_node(sc->dev); for (idx = 0; idx < nitems(sc->cspins); ++idx) { err = gpio_pin_get_by_ofw_propidx(sc->dev, node, "cs-gpios", idx, &sc->cspins[idx]); if (err == 0) { gpio_pin_setflags(sc->cspins[idx], GPIO_PIN_OUTPUT); } else if (sc->debug >= 2) { device_printf(sc->dev, "cannot configure gpio for chip select %u\n", idx); } } /* * Hardware init: put all channels into Master mode, turn off the enable * bit (gates off clocks); we only enable the hardware while xfers run. */ WR4(sc, ECSPI_CTLREG, CTLREG_CMODES_MASTER); /* * Add the spibus driver as a child, and setup a one-shot intrhook to * attach it after interrupts are working. It will attach actual SPI * devices as its children, and those devices may need to do IO during * their attach. We can't do IO until timers and interrupts are working. */ sc->spibus = device_add_child(dev, "spibus", -1); config_intrhook_oneshot((ich_func_t)bus_generic_attach, dev); return (0); } static int spi_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "i.MX ECSPI Master"); return (BUS_PROBE_DEFAULT); } static device_method_t spi_methods[] = { DEVMETHOD(device_probe, spi_probe), DEVMETHOD(device_attach, spi_attach), DEVMETHOD(device_detach, spi_detach), /* spibus_if */ DEVMETHOD(spibus_transfer, spi_transfer), /* ofw_bus_if */ DEVMETHOD(ofw_bus_get_node, spi_get_node), DEVMETHOD_END }; static driver_t spi_driver = { "imx_spi", spi_methods, sizeof(struct spi_softc), }; static devclass_t spi_devclass; DRIVER_MODULE(imx_spi, simplebus, spi_driver, spi_devclass, 0, 0); DRIVER_MODULE(ofw_spibus, imx_spi, ofw_spibus_driver, ofw_spibus_devclass, 0, 0); MODULE_DEPEND(imx_spi, ofw_spibus, 1, 1, 1); SIMPLEBUS_PNP_INFO(compat_data); Index: projects/import-googletest-1.8.1/sys/cam/scsi/scsi_da.c =================================================================== --- projects/import-googletest-1.8.1/sys/cam/scsi/scsi_da.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/cam/scsi/scsi_da.c (revision 345026) @@ -1,6525 +1,6526 @@ /*- * Implementation of SCSI Direct Access Peripheral driver for CAM. * * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1997 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #ifdef _KERNEL #include "opt_da.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #endif /* _KERNEL */ #ifndef _KERNEL #include #include #endif /* _KERNEL */ #include #include #include #include #include #include #include #include #ifdef _KERNEL /* * Note that there are probe ordering dependencies here. The order isn't * controlled by this enumeration, but by explicit state transitions in * dastart() and dadone(). Here are some of the dependencies: * * 1. RC should come first, before RC16, unless there is evidence that RC16 * is supported. * 2. BDC needs to come before any of the ATA probes, or the ZONE probe. * 3. The ATA probes should go in this order: * ATA -> LOGDIR -> IDDIR -> SUP -> ATA_ZONE */ typedef enum { DA_STATE_PROBE_WP, DA_STATE_PROBE_RC, DA_STATE_PROBE_RC16, DA_STATE_PROBE_LBP, DA_STATE_PROBE_BLK_LIMITS, DA_STATE_PROBE_BDC, DA_STATE_PROBE_ATA, DA_STATE_PROBE_ATA_LOGDIR, DA_STATE_PROBE_ATA_IDDIR, DA_STATE_PROBE_ATA_SUP, DA_STATE_PROBE_ATA_ZONE, DA_STATE_PROBE_ZONE, DA_STATE_NORMAL } da_state; typedef enum { DA_FLAG_PACK_INVALID = 0x000001, DA_FLAG_NEW_PACK = 0x000002, DA_FLAG_PACK_LOCKED = 0x000004, DA_FLAG_PACK_REMOVABLE = 0x000008, DA_FLAG_NEED_OTAG = 0x000020, DA_FLAG_WAS_OTAG = 0x000040, DA_FLAG_RETRY_UA = 0x000080, DA_FLAG_OPEN = 0x000100, DA_FLAG_SCTX_INIT = 0x000200, DA_FLAG_CAN_RC16 = 0x000400, DA_FLAG_PROBED = 0x000800, DA_FLAG_DIRTY = 0x001000, DA_FLAG_ANNOUNCED = 0x002000, DA_FLAG_CAN_ATA_DMA = 0x004000, DA_FLAG_CAN_ATA_LOG = 0x008000, DA_FLAG_CAN_ATA_IDLOG = 0x010000, DA_FLAG_CAN_ATA_SUPCAP = 0x020000, DA_FLAG_CAN_ATA_ZONE = 0x040000, DA_FLAG_TUR_PENDING = 0x080000 } da_flags; typedef enum { DA_Q_NONE = 0x00, DA_Q_NO_SYNC_CACHE = 0x01, DA_Q_NO_6_BYTE = 0x02, DA_Q_NO_PREVENT = 0x04, DA_Q_4K = 0x08, DA_Q_NO_RC16 = 0x10, DA_Q_NO_UNMAP = 0x20, DA_Q_RETRY_BUSY = 0x40, DA_Q_SMR_DM = 0x80, DA_Q_STRICT_UNMAP = 0x100, DA_Q_128KB = 0x200 } da_quirks; #define DA_Q_BIT_STRING \ "\020" \ "\001NO_SYNC_CACHE" \ "\002NO_6_BYTE" \ "\003NO_PREVENT" \ "\0044K" \ "\005NO_RC16" \ "\006NO_UNMAP" \ "\007RETRY_BUSY" \ "\010SMR_DM" \ "\011STRICT_UNMAP" \ "\012128KB" typedef enum { DA_CCB_PROBE_RC = 0x01, DA_CCB_PROBE_RC16 = 0x02, DA_CCB_PROBE_LBP = 0x03, DA_CCB_PROBE_BLK_LIMITS = 0x04, DA_CCB_PROBE_BDC = 0x05, DA_CCB_PROBE_ATA = 0x06, DA_CCB_BUFFER_IO = 0x07, DA_CCB_DUMP = 0x0A, DA_CCB_DELETE = 0x0B, DA_CCB_TUR = 0x0C, DA_CCB_PROBE_ZONE = 0x0D, DA_CCB_PROBE_ATA_LOGDIR = 0x0E, DA_CCB_PROBE_ATA_IDDIR = 0x0F, DA_CCB_PROBE_ATA_SUP = 0x10, DA_CCB_PROBE_ATA_ZONE = 0x11, DA_CCB_PROBE_WP = 0x12, DA_CCB_TYPE_MASK = 0x1F, DA_CCB_RETRY_UA = 0x20 } da_ccb_state; /* * Order here is important for method choice * * We prefer ATA_TRIM as tests run against a Sandforce 2281 SSD attached to * LSI 2008 (mps) controller (FW: v12, Drv: v14) resulted 20% quicker deletes * using ATA_TRIM than the corresponding UNMAP results for a real world mysql * import taking 5mins. * */ typedef enum { DA_DELETE_NONE, DA_DELETE_DISABLE, DA_DELETE_ATA_TRIM, DA_DELETE_UNMAP, DA_DELETE_WS16, DA_DELETE_WS10, DA_DELETE_ZERO, DA_DELETE_MIN = DA_DELETE_ATA_TRIM, DA_DELETE_MAX = DA_DELETE_ZERO } da_delete_methods; /* * For SCSI, host managed drives show up as a separate device type. For * ATA, host managed drives also have a different device signature. * XXX KDM figure out the ATA host managed signature. */ typedef enum { DA_ZONE_NONE = 0x00, DA_ZONE_DRIVE_MANAGED = 0x01, DA_ZONE_HOST_AWARE = 0x02, DA_ZONE_HOST_MANAGED = 0x03 } da_zone_mode; /* * We distinguish between these interface cases in addition to the drive type: * o ATA drive behind a SCSI translation layer that knows about ZBC/ZAC * o ATA drive behind a SCSI translation layer that does not know about * ZBC/ZAC, and so needs to be managed via ATA passthrough. In this * case, we would need to share the ATA code with the ada(4) driver. * o SCSI drive. */ typedef enum { DA_ZONE_IF_SCSI, DA_ZONE_IF_ATA_PASS, DA_ZONE_IF_ATA_SAT, } da_zone_interface; typedef enum { DA_ZONE_FLAG_RZ_SUP = 0x0001, DA_ZONE_FLAG_OPEN_SUP = 0x0002, DA_ZONE_FLAG_CLOSE_SUP = 0x0004, DA_ZONE_FLAG_FINISH_SUP = 0x0008, DA_ZONE_FLAG_RWP_SUP = 0x0010, DA_ZONE_FLAG_SUP_MASK = (DA_ZONE_FLAG_RZ_SUP | DA_ZONE_FLAG_OPEN_SUP | DA_ZONE_FLAG_CLOSE_SUP | DA_ZONE_FLAG_FINISH_SUP | DA_ZONE_FLAG_RWP_SUP), DA_ZONE_FLAG_URSWRZ = 0x0020, DA_ZONE_FLAG_OPT_SEQ_SET = 0x0040, DA_ZONE_FLAG_OPT_NONSEQ_SET = 0x0080, DA_ZONE_FLAG_MAX_SEQ_SET = 0x0100, DA_ZONE_FLAG_SET_MASK = (DA_ZONE_FLAG_OPT_SEQ_SET | DA_ZONE_FLAG_OPT_NONSEQ_SET | DA_ZONE_FLAG_MAX_SEQ_SET) } da_zone_flags; static struct da_zone_desc { da_zone_flags value; const char *desc; } da_zone_desc_table[] = { {DA_ZONE_FLAG_RZ_SUP, "Report Zones" }, {DA_ZONE_FLAG_OPEN_SUP, "Open" }, {DA_ZONE_FLAG_CLOSE_SUP, "Close" }, {DA_ZONE_FLAG_FINISH_SUP, "Finish" }, {DA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" }, }; typedef void da_delete_func_t (struct cam_periph *periph, union ccb *ccb, struct bio *bp); static da_delete_func_t da_delete_trim; static da_delete_func_t da_delete_unmap; static da_delete_func_t da_delete_ws; static const void * da_delete_functions[] = { NULL, NULL, da_delete_trim, da_delete_unmap, da_delete_ws, da_delete_ws, da_delete_ws }; static const char *da_delete_method_names[] = { "NONE", "DISABLE", "ATA_TRIM", "UNMAP", "WS16", "WS10", "ZERO" }; static const char *da_delete_method_desc[] = { "NONE", "DISABLED", "ATA TRIM", "UNMAP", "WRITE SAME(16) with UNMAP", "WRITE SAME(10) with UNMAP", "ZERO" }; /* Offsets into our private area for storing information */ #define ccb_state ppriv_field0 #define ccb_bp ppriv_ptr1 struct disk_params { u_int8_t heads; u_int32_t cylinders; u_int8_t secs_per_track; u_int32_t secsize; /* Number of bytes/sector */ u_int64_t sectors; /* total number sectors */ u_int stripesize; u_int stripeoffset; }; #define UNMAP_RANGE_MAX 0xffffffff #define UNMAP_HEAD_SIZE 8 #define UNMAP_RANGE_SIZE 16 #define UNMAP_MAX_RANGES 2048 /* Protocol Max is 4095 */ #define UNMAP_BUF_SIZE ((UNMAP_MAX_RANGES * UNMAP_RANGE_SIZE) + \ UNMAP_HEAD_SIZE) #define WS10_MAX_BLKS 0xffff #define WS16_MAX_BLKS 0xffffffff #define ATA_TRIM_MAX_RANGES ((UNMAP_BUF_SIZE / \ (ATA_DSM_RANGE_SIZE * ATA_DSM_BLK_SIZE)) * ATA_DSM_BLK_SIZE) #define DA_WORK_TUR (1 << 16) typedef enum { DA_REF_OPEN = 1, DA_REF_OPEN_HOLD, DA_REF_CLOSE_HOLD, DA_REF_PROBE_HOLD, DA_REF_TUR, DA_REF_GEOM, DA_REF_SYSCTL, DA_REF_REPROBE, DA_REF_MAX /* KEEP LAST */ } da_ref_token; struct da_softc { struct cam_iosched_softc *cam_iosched; struct bio_queue_head delete_run_queue; LIST_HEAD(, ccb_hdr) pending_ccbs; int refcount; /* Active xpt_action() calls */ da_state state; da_flags flags; da_quirks quirks; int minimum_cmd_size; int error_inject; int trim_max_ranges; int delete_available; /* Delete methods possibly available */ da_zone_mode zone_mode; da_zone_interface zone_interface; da_zone_flags zone_flags; struct ata_gp_log_dir ata_logdir; int valid_logdir_len; struct ata_identify_log_pages ata_iddir; int valid_iddir_len; uint64_t optimal_seq_zones; uint64_t optimal_nonseq_zones; uint64_t max_seq_zones; u_int maxio; uint32_t unmap_max_ranges; uint32_t unmap_max_lba; /* Max LBAs in UNMAP req */ uint32_t unmap_gran; uint32_t unmap_gran_align; uint64_t ws_max_blks; uint64_t trim_count; uint64_t trim_ranges; uint64_t trim_lbas; da_delete_methods delete_method_pref; da_delete_methods delete_method; da_delete_func_t *delete_func; int unmappedio; int rotating; struct disk_params params; struct disk *disk; union ccb saved_ccb; struct task sysctl_task; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; struct callout sendordered_c; uint64_t wwpn; uint8_t unmap_buf[UNMAP_BUF_SIZE]; struct scsi_read_capacity_data_long rcaplong; struct callout mediapoll_c; int ref_flags[DA_REF_MAX]; #ifdef CAM_IO_STATS struct sysctl_ctx_list sysctl_stats_ctx; struct sysctl_oid *sysctl_stats_tree; u_int errors; u_int timeouts; u_int invalidations; #endif #define DA_ANNOUNCETMP_SZ 160 char announce_temp[DA_ANNOUNCETMP_SZ]; #define DA_ANNOUNCE_SZ 400 char announcebuf[DA_ANNOUNCE_SZ]; }; #define dadeleteflag(softc, delete_method, enable) \ if (enable) { \ softc->delete_available |= (1 << delete_method); \ } else { \ softc->delete_available &= ~(1 << delete_method); \ } struct da_quirk_entry { struct scsi_inquiry_pattern inq_pat; da_quirks quirks; }; static const char quantum[] = "QUANTUM"; static const char microp[] = "MICROP"; static struct da_quirk_entry da_quirk_table[] = { /* SPI, FC devices */ { /* * Fujitsu M2513A MO drives. * Tested devices: M2513A2 firmware versions 1200 & 1300. * (dip switch selects whether T_DIRECT or T_OPTICAL device) * Reported by: W.Scholten */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* See above. */ {T_OPTICAL, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * This particular Fujitsu drive doesn't like the * synchronize cache command. * Reported by: Tom Jackson */ {T_DIRECT, SIP_MEDIA_FIXED, "FUJITSU", "M2954*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * This drive doesn't like the synchronize cache command * either. Reported by: Matthew Jacob * in NetBSD PR kern/6027, August 24, 1998. */ {T_DIRECT, SIP_MEDIA_FIXED, microp, "2217*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * This drive doesn't like the synchronize cache command * either. Reported by: Hellmuth Michaelis (hm@kts.org) * (PR 8882). */ {T_DIRECT, SIP_MEDIA_FIXED, microp, "2112*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Doesn't like the synchronize cache command. * Reported by: Blaz Zupan */ {T_DIRECT, SIP_MEDIA_FIXED, "NEC", "D3847*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Doesn't like the synchronize cache command. * Reported by: Blaz Zupan */ {T_DIRECT, SIP_MEDIA_FIXED, quantum, "MAVERICK 540S", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Doesn't like the synchronize cache command. */ {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS525S", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Doesn't like the synchronize cache command. * Reported by: walter@pelissero.de */ {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS540S", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Doesn't work correctly with 6 byte reads/writes. * Returns illegal request, and points to byte 9 of the * 6-byte CDB. * Reported by: Adam McDougall */ {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 4*", "*"}, /*quirks*/ DA_Q_NO_6_BYTE }, { /* See above. */ {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 2*", "*"}, /*quirks*/ DA_Q_NO_6_BYTE }, { /* * Doesn't like the synchronize cache command. * Reported by: walter@pelissero.de */ {T_DIRECT, SIP_MEDIA_FIXED, "CONNER", "CP3500*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * The CISS RAID controllers do not support SYNC_CACHE */ {T_DIRECT, SIP_MEDIA_FIXED, "COMPAQ", "RAID*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * The STEC SSDs sometimes hang on UNMAP. */ {T_DIRECT, SIP_MEDIA_FIXED, "STEC", "*", "*"}, /*quirks*/ DA_Q_NO_UNMAP }, { /* * VMware returns BUSY status when storage has transient * connectivity problems, so better wait. * Also VMware returns odd errors on misaligned UNMAPs. */ {T_DIRECT, SIP_MEDIA_FIXED, "VMware*", "*", "*"}, /*quirks*/ DA_Q_RETRY_BUSY | DA_Q_STRICT_UNMAP }, /* USB mass storage devices supported by umass(4) */ { /* * EXATELECOM (Sigmatel) i-Bead 100/105 USB Flash MP3 Player * PR: kern/51675 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "EXATEL", "i-BEAD10*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Power Quotient Int. (PQI) USB flash key * PR: kern/53067 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "USB Flash Disk*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Creative Nomad MUVO mp3 player (USB) * PR: kern/53094 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "NOMAD_MUVO", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT }, { /* * Jungsoft NEXDISK USB flash key * PR: kern/54737 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "JUNGSOFT", "NEXDISK*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * FreeDik USB Mini Data Drive * PR: kern/54786 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "FreeDik*", "Mini Data Drive", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Sigmatel USB Flash MP3 Player * PR: kern/57046 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "SigmaTel", "MSCN", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT }, { /* * Neuros USB Digital Audio Computer * PR: kern/63645 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "NEUROS", "dig. audio comp.", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * SEAGRAND NP-900 MP3 Player * PR: kern/64563 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "SEAGRAND", "NP-900*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT }, { /* * iRiver iFP MP3 player (with UMS Firmware) * PR: kern/54881, i386/63941, kern/66124 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "iRiver", "iFP*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Frontier Labs NEX IA+ Digital Audio Player, rev 1.10/0.01 * PR: kern/70158 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "FL" , "Nex*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * ZICPlay USB MP3 Player with FM * PR: kern/75057 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "ACTIONS*" , "USB DISK*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * TEAC USB floppy mechanisms */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "TEAC" , "FD-05*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Kingston DataTraveler II+ USB Pen-Drive. * Reported by: Pawel Jakub Dawidek */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston" , "DataTraveler II+", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * USB DISK Pro PMAP * Reported by: jhs * PR: usb/96381 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, " ", "USB DISK Pro", "PMAP"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Motorola E398 Mobile Phone (TransFlash memory card). * Reported by: Wojciech A. Koszek * PR: usb/89889 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Motorola" , "Motorola Phone", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Qware BeatZkey! Pro * PR: usb/79164 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "GENERIC", "USB DISK DEVICE", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Time DPA20B 1GB MP3 Player * PR: usb/81846 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB2.0*", "(FS) FLASH DISK*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Samsung USB key 128Mb * PR: usb/90081 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB-DISK", "FreeDik-FlashUsb", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Kingston DataTraveler 2.0 USB Flash memory. * PR: usb/89196 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler 2.0", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Creative MUVO Slim mp3 player (USB) * PR: usb/86131 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "MuVo Slim", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT }, { /* * United MP5512 Portable MP3 Player (2-in-1 USB DISK/MP3) * PR: usb/80487 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "MUSIC DISK", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * SanDisk Micro Cruzer 128MB * PR: usb/75970 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "SanDisk" , "Micro Cruzer", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * TOSHIBA TransMemory USB sticks * PR: kern/94660 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "TOSHIBA", "TransMemory", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * PNY USB 3.0 Flash Drives */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "PNY", "USB 3.0 FD*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_RC16 }, { /* * PNY USB Flash keys * PR: usb/75578, usb/72344, usb/65436 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "*" , "USB DISK*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Genesys GL3224 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*", "120?"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_4K | DA_Q_NO_RC16 }, { /* * Genesys 6-in-1 Card Reader * PR: usb/94647 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Rekam Digital CAMERA * PR: usb/98713 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "CAMERA*", "4MP-9J6*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * iRiver H10 MP3 player * PR: usb/102547 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "H10*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * iRiver U10 MP3 player * PR: usb/92306 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "U10*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * X-Micro Flash Disk * PR: usb/96901 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "X-Micro", "Flash Disk", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * EasyMP3 EM732X USB 2.0 Flash MP3 Player * PR: usb/96546 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "EM732X", "MP3 Player*", "1.00"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Denver MP3 player * PR: usb/107101 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "DENVER", "MP3 PLAYER", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Philips USB Key Audio KEY013 * PR: usb/68412 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "PHILIPS", "Key*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT }, { /* * JNC MP3 Player * PR: usb/94439 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "JNC*" , "MP3 Player*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * SAMSUNG MP0402H * PR: usb/108427 */ {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "MP0402H", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * I/O Magic USB flash - Giga Bank * PR: usb/108810 */ {T_DIRECT, SIP_MEDIA_FIXED, "GS-Magic", "stor*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * JoyFly 128mb USB Flash Drive * PR: 96133 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "Flash Disk*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * ChipsBnk usb stick * PR: 103702 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "ChipsBnk", "USB*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Storcase (Kingston) InfoStation IFS FC2/SATA-R 201A * PR: 129858 */ {T_DIRECT, SIP_MEDIA_FIXED, "IFS", "FC2/SATA-R*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Samsung YP-U3 mp3-player * PR: 125398 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Samsung", "YP-U3", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { {T_DIRECT, SIP_MEDIA_REMOVABLE, "Netac", "OnlyDisk*", "2000"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Sony Cyber-Shot DSC cameras * PR: usb/137035 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "Sony", "Sony DSC", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT }, { {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler G3", "1.00"}, /*quirks*/ DA_Q_NO_PREVENT }, { /* At least several Transcent USB sticks lie on RC16. */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "JetFlash", "Transcend*", "*"}, /*quirks*/ DA_Q_NO_RC16 }, { /* * I-O Data USB Flash Disk * PR: usb/211716 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "I-O DATA", "USB Flash Disk*", "*"}, /*quirks*/ DA_Q_NO_RC16 }, { /* - * 16GB SLC CHIPFANCIER - * PR: usb/234503 + * SLC CHIPFANCIER USB drives + * PR: usb/234503 (RC10 right, RC16 wrong) + * 16GB, 32GB and 128GB confirmed to have same issue */ - {T_DIRECT, SIP_MEDIA_REMOVABLE, "16G SLC", "CHIPFANCIER", - "1.00"}, /*quirks*/ DA_Q_NO_RC16 + {T_DIRECT, SIP_MEDIA_REMOVABLE, "*SLC", "CHIPFANCIER", + "*"}, /*quirks*/ DA_Q_NO_RC16 }, /* ATA/SATA devices over SAS/USB/... */ { /* Sandisk X400 */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SanDisk SD8SB8U1*", "*" }, /*quirks*/DA_Q_128KB }, { /* Hitachi Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "Hitachi", "H??????????E3*", "*" }, /*quirks*/DA_Q_4K }, { /* Micron Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Micron 5100 MTFDDAK*", "*" }, /*quirks*/DA_Q_4K }, { /* Samsung Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD155UI*", "*" }, /*quirks*/DA_Q_4K }, { /* Samsung Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD155UI*", "*" }, /*quirks*/DA_Q_4K }, { /* Samsung Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD204UI*", "*" }, /*quirks*/DA_Q_4K }, { /* Samsung Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD204UI*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DL*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST????DL", "*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???DM*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST???DM*", "*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DM*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Barracuda Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST????DM", "*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500423AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "3AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500424AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "4AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640423AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "3AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640424AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "4AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750420AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "0AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750422AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "2AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750423AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "3AS*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Thin Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???LT*", "*" }, /*quirks*/DA_Q_4K }, { /* Seagate Momentus Thin Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ST???LT*", "*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RS*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RS*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RX*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RX*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RS*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RS*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RX*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Caviar Green Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RX*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Black Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PKT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Black Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PKT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Black Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PKT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Black Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PKT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Blue Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PVT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Blue Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PVT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Blue Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PVT*", "*" }, /*quirks*/DA_Q_4K }, { /* WDC Scorpio Blue Advanced Format (4k) drives */ { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PVT*", "*" }, /*quirks*/DA_Q_4K }, { /* * Olympus digital cameras (C-3040ZOOM, C-2040ZOOM, C-1) * PR: usb/97472 */ { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "C*", "*"}, /*quirks*/ DA_Q_NO_6_BYTE | DA_Q_NO_SYNC_CACHE }, { /* * Olympus digital cameras (D-370) * PR: usb/97472 */ { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "D*", "*"}, /*quirks*/ DA_Q_NO_6_BYTE }, { /* * Olympus digital cameras (E-100RS, E-10). * PR: usb/97472 */ { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "E*", "*"}, /*quirks*/ DA_Q_NO_6_BYTE | DA_Q_NO_SYNC_CACHE }, { /* * Olympus FE-210 camera */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "FE210*", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Pentax Digital Camera * PR: usb/93389 */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "PENTAX", "DIGITAL CAMERA", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * LG UP3S MP3 player */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "LG", "UP3S", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * Laser MP3-2GA13 MP3 player */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "(HS) Flash Disk", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, { /* * LaCie external 250GB Hard drive des by Porsche * Submitted by: Ben Stuyts * PR: 121474 */ {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HM250JI", "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE }, /* SATA SSDs */ { /* * Corsair Force 2 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair CSSD-F*", "*" }, /*quirks*/DA_Q_4K }, { /* * Corsair Force 3 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force 3*", "*" }, /*quirks*/DA_Q_4K }, { /* * Corsair Neutron GTX SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" }, /*quirks*/DA_Q_4K }, { /* * Corsair Force GT & GS SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force G*", "*" }, /*quirks*/DA_Q_4K }, { /* * Crucial M4 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "M4-CT???M4SSD2*", "*" }, /*quirks*/DA_Q_4K }, { /* * Crucial RealSSD C300 SSDs * 4k optimised */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "C300-CTFDDAC???MAG*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel 320 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2CW*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel 330 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2CT*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel 510 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2MH*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel 520 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BW*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel S3610 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BX*", "*" }, /*quirks*/DA_Q_4K }, { /* * Intel X25-M Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2M*", "*" }, /*quirks*/DA_Q_4K }, { /* * Kingston E100 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SE100S3*", "*" }, /*quirks*/DA_Q_4K }, { /* * Kingston HyperX 3k SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SH103S3*", "*" }, /*quirks*/DA_Q_4K }, { /* * Marvell SSDs (entry taken from OpenSolaris) * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MARVELL SD88SA02*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Agility 2 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Agility 3 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-AGILITY3*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Deneva R Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "DENRSTE251M45*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Vertex 2 SSDs (inc pro series) * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ?VERTEX2*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Vertex 3 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX3*", "*" }, /*quirks*/DA_Q_4K }, { /* * OCZ Vertex 4 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX4*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 750 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 750*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 830 Series SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG SSD 830 Series*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 840 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 840*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 845 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 845*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 850 SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 850*", "*" }, /*quirks*/DA_Q_4K }, { /* * Samsung 843T Series SSDs (MZ7WD*) * Samsung PM851 Series SSDs (MZ7TE*) * Samsung PM853T Series SSDs (MZ7GE*) * Samsung SM863 Series SSDs (MZ7KM*) * 4k optimised */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG MZ7*", "*" }, /*quirks*/DA_Q_4K }, { /* * Same as for SAMSUNG MZ7* but enable the quirks for SSD * starting with MZ7* too */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MZ7*", "*" }, /*quirks*/DA_Q_4K }, { /* * SuperTalent TeraDrive CT SSDs * 4k optimised & trim only works in 4k requests + 4k aligned */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "FTM??CT25H*", "*" }, /*quirks*/DA_Q_4K }, { /* * XceedIOPS SATA SSDs * 4k optimised */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SG9XCS2D*", "*" }, /*quirks*/DA_Q_4K }, { /* * Hama Innostor USB-Stick */ { T_DIRECT, SIP_MEDIA_REMOVABLE, "Innostor", "Innostor*", "*" }, /*quirks*/DA_Q_NO_RC16 }, { /* * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR) * Drive Managed SATA hard drive. This drive doesn't report * in firmware that it is a drive managed SMR drive. */ { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST8000AS000[23]*", "*" }, /*quirks*/DA_Q_SMR_DM }, { /* * MX-ES USB Drive by Mach Xtreme */ { T_DIRECT, SIP_MEDIA_REMOVABLE, "MX", "MXUB3*", "*"}, /*quirks*/DA_Q_NO_RC16 }, }; static disk_strategy_t dastrategy; static dumper_t dadump; static periph_init_t dainit; static void daasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg); static void dasysctlinit(void *context, int pending); static int dasysctlsofttimeout(SYSCTL_HANDLER_ARGS); static int dacmdsizesysctl(SYSCTL_HANDLER_ARGS); static int dadeletemethodsysctl(SYSCTL_HANDLER_ARGS); static int dazonemodesysctl(SYSCTL_HANDLER_ARGS); static int dazonesupsysctl(SYSCTL_HANDLER_ARGS); static int dadeletemaxsysctl(SYSCTL_HANDLER_ARGS); static void dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method); static off_t dadeletemaxsize(struct da_softc *softc, da_delete_methods delete_method); static void dadeletemethodchoose(struct da_softc *softc, da_delete_methods default_method); static void daprobedone(struct cam_periph *periph, union ccb *ccb); static periph_ctor_t daregister; static periph_dtor_t dacleanup; static periph_start_t dastart; static periph_oninv_t daoninvalidate; static void dazonedone(struct cam_periph *periph, union ccb *ccb); static void dadone(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probewp(struct cam_periph *periph, union ccb *done_ccb); static void dadone_proberc(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probelbp(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeblklimits(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probebdc(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeata(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeatalogdir(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeataiddir(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeatasup(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probeatazone(struct cam_periph *periph, union ccb *done_ccb); static void dadone_probezone(struct cam_periph *periph, union ccb *done_ccb); static void dadone_tur(struct cam_periph *periph, union ccb *done_ccb); static int daerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags); static void daprevent(struct cam_periph *periph, int action); static void dareprobe(struct cam_periph *periph); static void dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector, struct scsi_read_capacity_data_long *rcaplong, size_t rcap_size); static timeout_t dasendorderedtag; static void dashutdown(void *arg, int howto); static timeout_t damediapoll; #ifndef DA_DEFAULT_POLL_PERIOD #define DA_DEFAULT_POLL_PERIOD 3 #endif #ifndef DA_DEFAULT_TIMEOUT #define DA_DEFAULT_TIMEOUT 60 /* Timeout in seconds */ #endif #ifndef DA_DEFAULT_SOFTTIMEOUT #define DA_DEFAULT_SOFTTIMEOUT 0 #endif #ifndef DA_DEFAULT_RETRY #define DA_DEFAULT_RETRY 4 #endif #ifndef DA_DEFAULT_SEND_ORDERED #define DA_DEFAULT_SEND_ORDERED 1 #endif static int da_poll_period = DA_DEFAULT_POLL_PERIOD; static int da_retry_count = DA_DEFAULT_RETRY; static int da_default_timeout = DA_DEFAULT_TIMEOUT; static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT; static int da_send_ordered = DA_DEFAULT_SEND_ORDERED; static int da_disable_wp_detection = 0; static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0, "CAM Direct Access Disk driver"); SYSCTL_INT(_kern_cam_da, OID_AUTO, poll_period, CTLFLAG_RWTUN, &da_poll_period, 0, "Media polling period in seconds"); SYSCTL_INT(_kern_cam_da, OID_AUTO, retry_count, CTLFLAG_RWTUN, &da_retry_count, 0, "Normal I/O retry count"); SYSCTL_INT(_kern_cam_da, OID_AUTO, default_timeout, CTLFLAG_RWTUN, &da_default_timeout, 0, "Normal I/O timeout (in seconds)"); SYSCTL_INT(_kern_cam_da, OID_AUTO, send_ordered, CTLFLAG_RWTUN, &da_send_ordered, 0, "Send Ordered Tags"); SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN, &da_disable_wp_detection, 0, "Disable detection of write-protected disks"); SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout, CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, dasysctlsofttimeout, "I", "Soft I/O timeout (ms)"); TUNABLE_INT64("kern.cam.da.default_softtimeout", &da_default_softtimeout); /* * DA_ORDEREDTAG_INTERVAL determines how often, relative * to the default timeout, we check to see whether an ordered * tagged transaction is appropriate to prevent simple tag * starvation. Since we'd like to ensure that there is at least * 1/2 of the timeout length left for a starved transaction to * complete after we've sent an ordered tag, we must poll at least * four times in every timeout period. This takes care of the worst * case where a starved transaction starts during an interval that * meets the requirement "don't send an ordered tag" test so it takes * us two intervals to determine that a tag must be sent. */ #ifndef DA_ORDEREDTAG_INTERVAL #define DA_ORDEREDTAG_INTERVAL 4 #endif static struct periph_driver dadriver = { dainit, "da", TAILQ_HEAD_INITIALIZER(dadriver.units), /* generation */ 0 }; PERIPHDRIVER_DECLARE(da, dadriver); static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers"); /* * This driver takes out references / holds in well defined pairs, never * recursively. These macros / inline functions enforce those rules. They * are only enabled with DA_TRACK_REFS or INVARIANTS. If DA_TRACK_REFS is * defined to be 2 or larger, the tracking also includes debug printfs. */ #if defined(DA_TRACK_REFS) || defined(INVARIANTS) #ifndef DA_TRACK_REFS #define DA_TRACK_REFS 1 #endif #if DA_TRACK_REFS > 1 static const char *da_ref_text[] = { "bogus", "open", "open hold", "close hold", "reprobe hold", "Test Unit Ready", "Geom", "sysctl", "reprobe", "max -- also bogus" }; #define DA_PERIPH_PRINT(periph, msg, args...) \ CAM_PERIPH_PRINT(periph, msg, ##args) #else #define DA_PERIPH_PRINT(periph, msg, args...) #endif static inline void token_sanity(da_ref_token token) { if ((unsigned)token >= DA_REF_MAX) panic("Bad token value passed in %d\n", token); } static inline int da_periph_hold(struct cam_periph *periph, int priority, da_ref_token token) { int err = cam_periph_hold(periph, priority); token_sanity(token); DA_PERIPH_PRINT(periph, "Holding device %s (%d): %d\n", da_ref_text[token], token, err); if (err == 0) { int cnt; struct da_softc *softc = periph->softc; cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1); if (cnt != 0) panic("Re-holding for reason %d, cnt = %d", token, cnt); } return (err); } static inline void da_periph_unhold(struct cam_periph *periph, da_ref_token token) { int cnt; struct da_softc *softc = periph->softc; token_sanity(token); DA_PERIPH_PRINT(periph, "Unholding device %s (%d)\n", da_ref_text[token], token); cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1); if (cnt != 1) panic("Unholding %d with cnt = %d", token, cnt); cam_periph_unhold(periph); } static inline int da_periph_acquire(struct cam_periph *periph, da_ref_token token) { int err = cam_periph_acquire(periph); token_sanity(token); DA_PERIPH_PRINT(periph, "acquiring device %s (%d): %d\n", da_ref_text[token], token, err); if (err == 0) { int cnt; struct da_softc *softc = periph->softc; cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1); if (cnt != 0) panic("Re-refing for reason %d, cnt = %d", token, cnt); } return (err); } static inline void da_periph_release(struct cam_periph *periph, da_ref_token token) { int cnt; struct da_softc *softc = periph->softc; token_sanity(token); DA_PERIPH_PRINT(periph, "releasing device %s (%d)\n", da_ref_text[token], token); cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1); if (cnt != 1) panic("Releasing %d with cnt = %d", token, cnt); cam_periph_release(periph); } static inline void da_periph_release_locked(struct cam_periph *periph, da_ref_token token) { int cnt; struct da_softc *softc = periph->softc; token_sanity(token); DA_PERIPH_PRINT(periph, "releasing device (locked) %s (%d)\n", da_ref_text[token], token); cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1); if (cnt != 1) panic("Unholding %d with cnt = %d", token, cnt); cam_periph_release_locked(periph); } #define cam_periph_hold POISON #define cam_periph_unhold POISON #define cam_periph_acquire POISON #define cam_periph_release POISON #define cam_periph_release_locked POISON #else #define da_periph_hold(periph, prio, token) cam_periph_hold((periph), (prio)) #define da_periph_unhold(periph, token) cam_periph_unhold((periph)) #define da_periph_acquire(periph, token) cam_periph_acquire((periph)) #define da_periph_release(periph, token) cam_periph_release((periph)) #define da_periph_release_locked(periph, token) cam_periph_release_locked((periph)) #endif static int daopen(struct disk *dp) { struct cam_periph *periph; struct da_softc *softc; int error; periph = (struct cam_periph *)dp->d_drv1; if (da_periph_acquire(periph, DA_REF_OPEN) != 0) { return (ENXIO); } cam_periph_lock(periph); if ((error = da_periph_hold(periph, PRIBIO|PCATCH, DA_REF_OPEN_HOLD)) != 0) { cam_periph_unlock(periph); da_periph_release(periph, DA_REF_OPEN); return (error); } CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, ("daopen\n")); softc = (struct da_softc *)periph->softc; dareprobe(periph); /* Wait for the disk size update. */ error = cam_periph_sleep(periph, &softc->disk->d_mediasize, PRIBIO, "dareprobe", 0); if (error != 0) xpt_print(periph->path, "unable to retrieve capacity data\n"); if (periph->flags & CAM_PERIPH_INVALID) error = ENXIO; if (error == 0 && (softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 && (softc->quirks & DA_Q_NO_PREVENT) == 0) daprevent(periph, PR_PREVENT); if (error == 0) { softc->flags &= ~DA_FLAG_PACK_INVALID; softc->flags |= DA_FLAG_OPEN; } da_periph_unhold(periph, DA_REF_OPEN_HOLD); cam_periph_unlock(periph); if (error != 0) da_periph_release(periph, DA_REF_OPEN); return (error); } static int daclose(struct disk *dp) { struct cam_periph *periph; struct da_softc *softc; union ccb *ccb; periph = (struct cam_periph *)dp->d_drv1; softc = (struct da_softc *)periph->softc; cam_periph_lock(periph); CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, ("daclose\n")); if (da_periph_hold(periph, PRIBIO, DA_REF_CLOSE_HOLD) == 0) { /* Flush disk cache. */ if ((softc->flags & DA_FLAG_DIRTY) != 0 && (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 && (softc->flags & DA_FLAG_PACK_INVALID) == 0) { ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); scsi_synchronize_cache(&ccb->csio, /*retries*/1, /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG, /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE, 5 * 60 * 1000); cam_periph_runccb(ccb, daerror, /*cam_flags*/0, /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR, softc->disk->d_devstat); softc->flags &= ~DA_FLAG_DIRTY; xpt_release_ccb(ccb); } /* Allow medium removal. */ if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 && (softc->quirks & DA_Q_NO_PREVENT) == 0) daprevent(periph, PR_ALLOW); da_periph_unhold(periph, DA_REF_CLOSE_HOLD); } /* * If we've got removeable media, mark the blocksize as * unavailable, since it could change when new media is * inserted. */ if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0) softc->disk->d_devstat->flags |= DEVSTAT_BS_UNAVAILABLE; softc->flags &= ~DA_FLAG_OPEN; while (softc->refcount != 0) cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1); cam_periph_unlock(periph); da_periph_release(periph, DA_REF_OPEN); return (0); } static void daschedule(struct cam_periph *periph) { struct da_softc *softc = (struct da_softc *)periph->softc; if (softc->state != DA_STATE_NORMAL) return; cam_iosched_schedule(softc->cam_iosched, periph); } /* * Actually translate the requested transfer into one the physical driver * can understand. The transfer is described by a buf and will include * only one physical transfer. */ static void dastrategy(struct bio *bp) { struct cam_periph *periph; struct da_softc *softc; periph = (struct cam_periph *)bp->bio_disk->d_drv1; softc = (struct da_softc *)periph->softc; cam_periph_lock(periph); /* * If the device has been made invalid, error out */ if ((softc->flags & DA_FLAG_PACK_INVALID)) { cam_periph_unlock(periph); biofinish(bp, NULL, ENXIO); return; } CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastrategy(%p)\n", bp)); /* * Zone commands must be ordered, because they can depend on the * effects of previously issued commands, and they may affect * commands after them. */ if (bp->bio_cmd == BIO_ZONE) bp->bio_flags |= BIO_ORDERED; /* * Place it in the queue of disk activities for this disk */ cam_iosched_queue_work(softc->cam_iosched, bp); /* * Schedule ourselves for performing the work. */ daschedule(periph); cam_periph_unlock(periph); return; } static int dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length) { struct cam_periph *periph; struct da_softc *softc; u_int secsize; struct ccb_scsiio csio; struct disk *dp; int error = 0; dp = arg; periph = dp->d_drv1; softc = (struct da_softc *)periph->softc; secsize = softc->params.secsize; if ((softc->flags & DA_FLAG_PACK_INVALID) != 0) return (ENXIO); memset(&csio, 0, sizeof(csio)); if (length > 0) { xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); csio.ccb_h.ccb_state = DA_CCB_DUMP; scsi_read_write(&csio, /*retries*/0, /*cbfcnp*/NULL, MSG_ORDERED_Q_TAG, /*read*/SCSI_RW_WRITE, /*byte2*/0, /*minimum_cmd_size*/ softc->minimum_cmd_size, offset / secsize, length / secsize, /*data_ptr*/(u_int8_t *) virtual, /*dxfer_len*/length, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); error = cam_periph_runccb((union ccb *)&csio, cam_periph_error, 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) printf("Aborting dump due to I/O error.\n"); return (error); } /* * Sync the disk cache contents to the physical media. */ if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) { xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); csio.ccb_h.ccb_state = DA_CCB_DUMP; scsi_synchronize_cache(&csio, /*retries*/0, /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG, /*begin_lba*/0,/* Cover the whole disk */ /*lb_count*/0, SSD_FULL_SIZE, 5 * 1000); error = cam_periph_runccb((union ccb *)&csio, cam_periph_error, 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) xpt_print(periph->path, "Synchronize cache failed\n"); } return (error); } static int dagetattr(struct bio *bp) { int ret; struct cam_periph *periph; periph = (struct cam_periph *)bp->bio_disk->d_drv1; cam_periph_lock(periph); ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute, periph->path); cam_periph_unlock(periph); if (ret == 0) bp->bio_completed = bp->bio_length; return ret; } static void dainit(void) { cam_status status; /* * Install a global async callback. This callback will * receive async callbacks like "new device found". */ status = xpt_register_async(AC_FOUND_DEVICE, daasync, NULL, NULL); if (status != CAM_REQ_CMP) { printf("da: Failed to attach master async callback " "due to status 0x%x!\n", status); } else if (da_send_ordered) { /* Register our shutdown event handler */ if ((EVENTHANDLER_REGISTER(shutdown_post_sync, dashutdown, NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) printf("dainit: shutdown event registration failed!\n"); } } /* * Callback from GEOM, called when it has finished cleaning up its * resources. */ static void dadiskgonecb(struct disk *dp) { struct cam_periph *periph; periph = (struct cam_periph *)dp->d_drv1; da_periph_release(periph, DA_REF_GEOM); } static void daoninvalidate(struct cam_periph *periph) { struct da_softc *softc; cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; /* * De-register any async callbacks. */ xpt_register_async(0, daasync, periph, periph->path); softc->flags |= DA_FLAG_PACK_INVALID; #ifdef CAM_IO_STATS softc->invalidations++; #endif /* * Return all queued I/O with ENXIO. * XXX Handle any transactions queued to the card * with XPT_ABORT_CCB. */ cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); /* * Tell GEOM that we've gone away, we'll get a callback when it is * done cleaning up its resources. */ disk_gone(softc->disk); } static void dacleanup(struct cam_periph *periph) { struct da_softc *softc; softc = (struct da_softc *)periph->softc; cam_periph_unlock(periph); cam_iosched_fini(softc->cam_iosched); /* * If we can't free the sysctl tree, oh well... */ if ((softc->flags & DA_FLAG_SCTX_INIT) != 0) { #ifdef CAM_IO_STATS if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0) xpt_print(periph->path, "can't remove sysctl stats context\n"); #endif if (sysctl_ctx_free(&softc->sysctl_ctx) != 0) xpt_print(periph->path, "can't remove sysctl context\n"); } callout_drain(&softc->mediapoll_c); disk_destroy(softc->disk); callout_drain(&softc->sendordered_c); free(softc, M_DEVBUF); cam_periph_lock(periph); } static void daasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg) { struct cam_periph *periph; struct da_softc *softc; periph = (struct cam_periph *)callback_arg; switch (code) { case AC_FOUND_DEVICE: /* callback to create periph, no locking yet */ { struct ccb_getdev *cgd; cam_status status; cgd = (struct ccb_getdev *)arg; if (cgd == NULL) break; if (cgd->protocol != PROTO_SCSI) break; if (SID_QUAL(&cgd->inq_data) != SID_QUAL_LU_CONNECTED) break; if (SID_TYPE(&cgd->inq_data) != T_DIRECT && SID_TYPE(&cgd->inq_data) != T_RBC && SID_TYPE(&cgd->inq_data) != T_OPTICAL && SID_TYPE(&cgd->inq_data) != T_ZBC_HM) break; /* * Allocate a peripheral instance for * this device and start the probe * process. */ status = cam_periph_alloc(daregister, daoninvalidate, dacleanup, dastart, "da", CAM_PERIPH_BIO, path, daasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP && status != CAM_REQ_INPROG) printf("daasync: Unable to attach to new device " "due to status 0x%x\n", status); return; } case AC_ADVINFO_CHANGED: /* Doesn't touch periph */ { uintptr_t buftype; buftype = (uintptr_t)arg; if (buftype == CDAI_TYPE_PHYS_PATH) { struct da_softc *softc; softc = periph->softc; disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT); } break; } case AC_UNIT_ATTENTION: { union ccb *ccb; int error_code, sense_key, asc, ascq; softc = (struct da_softc *)periph->softc; ccb = (union ccb *)arg; /* * Handle all UNIT ATTENTIONs except our own, as they will be * handled by daerror(). Since this comes from a different periph, * that periph's lock is held, not ours, so we have to take it ours * out to touch softc flags. */ if (xpt_path_periph(ccb->ccb_h.path) != periph && scsi_extract_sense_ccb(ccb, &error_code, &sense_key, &asc, &ascq)) { if (asc == 0x2A && ascq == 0x09) { xpt_print(ccb->ccb_h.path, "Capacity data has changed\n"); cam_periph_lock(periph); softc->flags &= ~DA_FLAG_PROBED; cam_periph_unlock(periph); dareprobe(periph); } else if (asc == 0x28 && ascq == 0x00) { cam_periph_lock(periph); softc->flags &= ~DA_FLAG_PROBED; cam_periph_unlock(periph); disk_media_changed(softc->disk, M_NOWAIT); } else if (asc == 0x3F && ascq == 0x03) { xpt_print(ccb->ccb_h.path, "INQUIRY data has changed\n"); cam_periph_lock(periph); softc->flags &= ~DA_FLAG_PROBED; cam_periph_unlock(periph); dareprobe(periph); } } break; } case AC_SCSI_AEN: /* Called for this path: periph locked */ /* * Appears to be currently unused for SCSI devices, only ata SIMs * generate this. */ cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) && (softc->flags & DA_FLAG_TUR_PENDING) == 0) { if (da_periph_acquire(periph, DA_REF_TUR) == 0) { cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); daschedule(periph); } } /* FALLTHROUGH */ case AC_SENT_BDR: /* Called for this path: periph locked */ case AC_BUS_RESET: /* Called for this path: periph locked */ { struct ccb_hdr *ccbh; cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; /* * Don't fail on the expected unit attention * that will occur. */ softc->flags |= DA_FLAG_RETRY_UA; LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le) ccbh->ccb_state |= DA_CCB_RETRY_UA; break; } case AC_INQ_CHANGED: /* Called for this path: periph locked */ cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; softc->flags &= ~DA_FLAG_PROBED; dareprobe(periph); break; default: break; } cam_periph_async(periph, code, path, arg); } static void dasysctlinit(void *context, int pending) { struct cam_periph *periph; struct da_softc *softc; char tmpstr[32], tmpstr2[16]; struct ccb_trans_settings cts; periph = (struct cam_periph *)context; /* * periph was held for us when this task was enqueued */ if (periph->flags & CAM_PERIPH_INVALID) { da_periph_release(periph, DA_REF_SYSCTL); return; } softc = (struct da_softc *)periph->softc; snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number); snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number); sysctl_ctx_init(&softc->sysctl_ctx); cam_periph_lock(periph); softc->flags |= DA_FLAG_SCTX_INIT; cam_periph_unlock(periph); softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2, CTLFLAG_RD, 0, tmpstr, "device_index"); if (softc->sysctl_tree == NULL) { printf("dasysctlinit: unable to allocate sysctl tree\n"); da_periph_release(periph, DA_REF_SYSCTL); return; } /* * Now register the sysctl handler, so the user can change the value on * the fly. */ SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RWTUN, softc, 0, dadeletemethodsysctl, "A", "BIO_DELETE execution method"); SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "delete_max", CTLTYPE_U64 | CTLFLAG_RW, softc, 0, dadeletemaxsysctl, "Q", "Maximum BIO_DELETE size"); SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW, &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I", "Minimum CDB size"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "trim_count", CTLFLAG_RD, &softc->trim_count, "Total number of unmap/dsm commands sent"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "trim_ranges", CTLFLAG_RD, &softc->trim_ranges, "Total number of ranges in unmap/dsm commands"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "trim_lbas", CTLFLAG_RD, &softc->trim_lbas, "Total lbas in the unmap/dsm commands sent"); SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD, softc, 0, dazonemodesysctl, "A", "Zone Mode"); SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD, softc, 0, dazonesupsysctl, "A", "Zone Support"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones, "Optimal Number of Open Sequential Write Preferred Zones"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "optimal_nonseq_zones", CTLFLAG_RD, &softc->optimal_nonseq_zones, "Optimal Number of Non-Sequentially Written Sequential Write " "Preferred Zones"); SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones, "Maximum Number of Open Sequential Write Required Zones"); SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "error_inject", CTLFLAG_RW, &softc->error_inject, 0, "error_inject leaf"); SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "unmapped_io", CTLFLAG_RD, &softc->unmappedio, 0, "Unmapped I/O leaf"); SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "rotating", CTLFLAG_RD, &softc->rotating, 0, "Rotating media"); #ifdef CAM_TEST_FAILURE SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, periph, 0, cam_periph_invalidate_sysctl, "I", "Write 1 to invalidate the drive immediately"); #endif /* * Add some addressing info. */ memset(&cts, 0, sizeof (cts)); xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cam_periph_lock(periph); xpt_action((union ccb *)&cts); cam_periph_unlock(periph); if (cts.ccb_h.status != CAM_REQ_CMP) { da_periph_release(periph, DA_REF_SYSCTL); return; } if (cts.protocol == PROTO_SCSI && cts.transport == XPORT_FC) { struct ccb_trans_settings_fc *fc = &cts.xport_specific.fc; if (fc->valid & CTS_FC_VALID_WWPN) { softc->wwpn = fc->wwpn; SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "wwpn", CTLFLAG_RD, &softc->wwpn, "World Wide Port Name"); } } #ifdef CAM_IO_STATS /* * Now add some useful stats. * XXX These should live in cam_periph and be common to all periphs */ softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats", CTLFLAG_RD, 0, "Statistics"); SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, SYSCTL_CHILDREN(softc->sysctl_stats_tree), OID_AUTO, "errors", CTLFLAG_RD, &softc->errors, 0, "Transport errors reported by the SIM"); SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, SYSCTL_CHILDREN(softc->sysctl_stats_tree), OID_AUTO, "timeouts", CTLFLAG_RD, &softc->timeouts, 0, "Device timeouts reported by the SIM"); SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, SYSCTL_CHILDREN(softc->sysctl_stats_tree), OID_AUTO, "pack_invalidations", CTLFLAG_RD, &softc->invalidations, 0, "Device pack invalidations"); #endif cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, softc->sysctl_tree); da_periph_release(periph, DA_REF_SYSCTL); } static int dadeletemaxsysctl(SYSCTL_HANDLER_ARGS) { int error; uint64_t value; struct da_softc *softc; softc = (struct da_softc *)arg1; value = softc->disk->d_delmaxsize; error = sysctl_handle_64(oidp, &value, 0, req); if ((error != 0) || (req->newptr == NULL)) return (error); /* only accept values smaller than the calculated value */ if (value > dadeletemaxsize(softc, softc->delete_method)) { return (EINVAL); } softc->disk->d_delmaxsize = value; return (0); } static int dacmdsizesysctl(SYSCTL_HANDLER_ARGS) { int error, value; value = *(int *)arg1; error = sysctl_handle_int(oidp, &value, 0, req); if ((error != 0) || (req->newptr == NULL)) return (error); /* * Acceptable values here are 6, 10, 12 or 16. */ if (value < 6) value = 6; else if ((value > 6) && (value <= 10)) value = 10; else if ((value > 10) && (value <= 12)) value = 12; else if (value > 12) value = 16; *(int *)arg1 = value; return (0); } static int dasysctlsofttimeout(SYSCTL_HANDLER_ARGS) { sbintime_t value; int error; value = da_default_softtimeout / SBT_1MS; error = sysctl_handle_int(oidp, (int *)&value, 0, req); if ((error != 0) || (req->newptr == NULL)) return (error); /* XXX Should clip this to a reasonable level */ if (value > da_default_timeout * 1000) return (EINVAL); da_default_softtimeout = value * SBT_1MS; return (0); } static void dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method) { softc->delete_method = delete_method; softc->disk->d_delmaxsize = dadeletemaxsize(softc, delete_method); softc->delete_func = da_delete_functions[delete_method]; if (softc->delete_method > DA_DELETE_DISABLE) softc->disk->d_flags |= DISKFLAG_CANDELETE; else softc->disk->d_flags &= ~DISKFLAG_CANDELETE; } static off_t dadeletemaxsize(struct da_softc *softc, da_delete_methods delete_method) { off_t sectors; switch(delete_method) { case DA_DELETE_UNMAP: sectors = (off_t)softc->unmap_max_lba; break; case DA_DELETE_ATA_TRIM: sectors = (off_t)ATA_DSM_RANGE_MAX * softc->trim_max_ranges; break; case DA_DELETE_WS16: sectors = omin(softc->ws_max_blks, WS16_MAX_BLKS); break; case DA_DELETE_ZERO: case DA_DELETE_WS10: sectors = omin(softc->ws_max_blks, WS10_MAX_BLKS); break; default: return 0; } return (off_t)softc->params.secsize * omin(sectors, softc->params.sectors); } static void daprobedone(struct cam_periph *periph, union ccb *ccb) { struct da_softc *softc; softc = (struct da_softc *)periph->softc; cam_periph_assert(periph, MA_OWNED); dadeletemethodchoose(softc, DA_DELETE_NONE); if (bootverbose && (softc->flags & DA_FLAG_ANNOUNCED) == 0) { char buf[80]; int i, sep; snprintf(buf, sizeof(buf), "Delete methods: <"); sep = 0; for (i = 0; i <= DA_DELETE_MAX; i++) { if ((softc->delete_available & (1 << i)) == 0 && i != softc->delete_method) continue; if (sep) strlcat(buf, ",", sizeof(buf)); strlcat(buf, da_delete_method_names[i], sizeof(buf)); if (i == softc->delete_method) strlcat(buf, "(*)", sizeof(buf)); sep = 1; } strlcat(buf, ">", sizeof(buf)); printf("%s%d: %s\n", periph->periph_name, periph->unit_number, buf); } if ((softc->disk->d_flags & DISKFLAG_WRITE_PROTECT) != 0 && (softc->flags & DA_FLAG_ANNOUNCED) == 0) { printf("%s%d: Write Protected\n", periph->periph_name, periph->unit_number); } /* * Since our peripheral may be invalidated by an error * above or an external event, we must release our CCB * before releasing the probe lock on the peripheral. * The peripheral will only go away once the last lock * is removed, and we need it around for the CCB release * operation. */ xpt_release_ccb(ccb); softc->state = DA_STATE_NORMAL; softc->flags |= DA_FLAG_PROBED; daschedule(periph); wakeup(&softc->disk->d_mediasize); if ((softc->flags & DA_FLAG_ANNOUNCED) == 0) { softc->flags |= DA_FLAG_ANNOUNCED; da_periph_unhold(periph, DA_REF_PROBE_HOLD); } else da_periph_release_locked(periph, DA_REF_REPROBE); } static void dadeletemethodchoose(struct da_softc *softc, da_delete_methods default_method) { int i, methods; /* If available, prefer the method requested by user. */ i = softc->delete_method_pref; methods = softc->delete_available | (1 << DA_DELETE_DISABLE); if (methods & (1 << i)) { dadeletemethodset(softc, i); return; } /* Use the pre-defined order to choose the best performing delete. */ for (i = DA_DELETE_MIN; i <= DA_DELETE_MAX; i++) { if (i == DA_DELETE_ZERO) continue; if (softc->delete_available & (1 << i)) { dadeletemethodset(softc, i); return; } } /* Fallback to default. */ dadeletemethodset(softc, default_method); } static int dadeletemethodsysctl(SYSCTL_HANDLER_ARGS) { char buf[16]; const char *p; struct da_softc *softc; int i, error, value; softc = (struct da_softc *)arg1; value = softc->delete_method; if (value < 0 || value > DA_DELETE_MAX) p = "UNKNOWN"; else p = da_delete_method_names[value]; strncpy(buf, p, sizeof(buf)); error = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (error != 0 || req->newptr == NULL) return (error); for (i = 0; i <= DA_DELETE_MAX; i++) { if (strcmp(buf, da_delete_method_names[i]) == 0) break; } if (i > DA_DELETE_MAX) return (EINVAL); softc->delete_method_pref = i; dadeletemethodchoose(softc, DA_DELETE_NONE); return (0); } static int dazonemodesysctl(SYSCTL_HANDLER_ARGS) { char tmpbuf[40]; struct da_softc *softc; int error; softc = (struct da_softc *)arg1; switch (softc->zone_mode) { case DA_ZONE_DRIVE_MANAGED: snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed"); break; case DA_ZONE_HOST_AWARE: snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware"); break; case DA_ZONE_HOST_MANAGED: snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed"); break; case DA_ZONE_NONE: default: snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned"); break; } error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req); return (error); } static int dazonesupsysctl(SYSCTL_HANDLER_ARGS) { char tmpbuf[180]; struct da_softc *softc; struct sbuf sb; int error, first; unsigned int i; softc = (struct da_softc *)arg1; error = 0; first = 1; sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0); for (i = 0; i < sizeof(da_zone_desc_table) / sizeof(da_zone_desc_table[0]); i++) { if (softc->zone_flags & da_zone_desc_table[i].value) { if (first == 0) sbuf_printf(&sb, ", "); else first = 0; sbuf_cat(&sb, da_zone_desc_table[i].desc); } } if (first == 1) sbuf_printf(&sb, "None"); sbuf_finish(&sb); error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); return (error); } static cam_status daregister(struct cam_periph *periph, void *arg) { struct da_softc *softc; struct ccb_pathinq cpi; struct ccb_getdev *cgd; char tmpstr[80]; caddr_t match; cgd = (struct ccb_getdev *)arg; if (cgd == NULL) { printf("daregister: no getdev CCB, can't register device\n"); return(CAM_REQ_CMP_ERR); } softc = (struct da_softc *)malloc(sizeof(*softc), M_DEVBUF, M_NOWAIT|M_ZERO); if (softc == NULL) { printf("daregister: Unable to probe new device. " "Unable to allocate softc\n"); return(CAM_REQ_CMP_ERR); } if (cam_iosched_init(&softc->cam_iosched, periph) != 0) { printf("daregister: Unable to probe new device. " "Unable to allocate iosched memory\n"); free(softc, M_DEVBUF); return(CAM_REQ_CMP_ERR); } LIST_INIT(&softc->pending_ccbs); softc->state = DA_STATE_PROBE_WP; bioq_init(&softc->delete_run_queue); if (SID_IS_REMOVABLE(&cgd->inq_data)) softc->flags |= DA_FLAG_PACK_REMOVABLE; softc->unmap_max_ranges = UNMAP_MAX_RANGES; softc->unmap_max_lba = UNMAP_RANGE_MAX; softc->unmap_gran = 0; softc->unmap_gran_align = 0; softc->ws_max_blks = WS16_MAX_BLKS; softc->trim_max_ranges = ATA_TRIM_MAX_RANGES; softc->rotating = 1; periph->softc = softc; /* * See if this device has any quirks. */ match = cam_quirkmatch((caddr_t)&cgd->inq_data, (caddr_t)da_quirk_table, nitems(da_quirk_table), sizeof(*da_quirk_table), scsi_inquiry_match); if (match != NULL) softc->quirks = ((struct da_quirk_entry *)match)->quirks; else softc->quirks = DA_Q_NONE; /* Check if the SIM does not want 6 byte commands */ xpt_path_inq(&cpi, periph->path); if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE)) softc->quirks |= DA_Q_NO_6_BYTE; if (SID_TYPE(&cgd->inq_data) == T_ZBC_HM) softc->zone_mode = DA_ZONE_HOST_MANAGED; else if (softc->quirks & DA_Q_SMR_DM) softc->zone_mode = DA_ZONE_DRIVE_MANAGED; else softc->zone_mode = DA_ZONE_NONE; if (softc->zone_mode != DA_ZONE_NONE) { if (scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) { if (scsi_vpd_supported_page(periph, SVPD_ZONED_BDC)) softc->zone_interface = DA_ZONE_IF_ATA_SAT; else softc->zone_interface = DA_ZONE_IF_ATA_PASS; } else softc->zone_interface = DA_ZONE_IF_SCSI; } TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph); /* * Take an exclusive section lock qon the periph while dastart is called * to finish the probe. The lock will be dropped in dadone at the end * of probe. This locks out daopen and daclose from racing with the * probe. * * XXX if cam_periph_hold returns an error, we don't hold a refcount. */ (void)da_periph_hold(periph, PRIBIO, DA_REF_PROBE_HOLD); /* * Schedule a periodic event to occasionally send an * ordered tag to a device. */ callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0); callout_reset(&softc->sendordered_c, (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, dasendorderedtag, periph); cam_periph_unlock(periph); /* * RBC devices don't have to support READ(6), only READ(10). */ if (softc->quirks & DA_Q_NO_6_BYTE || SID_TYPE(&cgd->inq_data) == T_RBC) softc->minimum_cmd_size = 10; else softc->minimum_cmd_size = 6; /* * Load the user's default, if any. */ snprintf(tmpstr, sizeof(tmpstr), "kern.cam.da.%d.minimum_cmd_size", periph->unit_number); TUNABLE_INT_FETCH(tmpstr, &softc->minimum_cmd_size); /* * 6, 10, 12 and 16 are the currently permissible values. */ if (softc->minimum_cmd_size > 12) softc->minimum_cmd_size = 16; else if (softc->minimum_cmd_size > 10) softc->minimum_cmd_size = 12; else if (softc->minimum_cmd_size > 6) softc->minimum_cmd_size = 10; else softc->minimum_cmd_size = 6; /* Predict whether device may support READ CAPACITY(16). */ if (SID_ANSI_REV(&cgd->inq_data) >= SCSI_REV_SPC3 && (softc->quirks & DA_Q_NO_RC16) == 0) { softc->flags |= DA_FLAG_CAN_RC16; } /* * Register this media as a disk. */ softc->disk = disk_alloc(); softc->disk->d_devstat = devstat_new_entry(periph->periph_name, periph->unit_number, 0, DEVSTAT_BS_UNAVAILABLE, SID_TYPE(&cgd->inq_data) | XPORT_DEVSTAT_TYPE(cpi.transport), DEVSTAT_PRIORITY_DISK); softc->disk->d_open = daopen; softc->disk->d_close = daclose; softc->disk->d_strategy = dastrategy; softc->disk->d_dump = dadump; softc->disk->d_getattr = dagetattr; softc->disk->d_gone = dadiskgonecb; softc->disk->d_name = "da"; softc->disk->d_drv1 = periph; if (cpi.maxio == 0) softc->maxio = DFLTPHYS; /* traditional default */ else if (cpi.maxio > MAXPHYS) softc->maxio = MAXPHYS; /* for safety */ else softc->maxio = cpi.maxio; if (softc->quirks & DA_Q_128KB) softc->maxio = min(softc->maxio, 128 * 1024); softc->disk->d_maxsize = softc->maxio; softc->disk->d_unit = periph->unit_number; softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE; if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE; if ((cpi.hba_misc & PIM_UNMAPPED) != 0) { softc->unmappedio = 1; softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO; } cam_strvis(softc->disk->d_descr, cgd->inq_data.vendor, sizeof(cgd->inq_data.vendor), sizeof(softc->disk->d_descr)); strlcat(softc->disk->d_descr, " ", sizeof(softc->disk->d_descr)); cam_strvis(&softc->disk->d_descr[strlen(softc->disk->d_descr)], cgd->inq_data.product, sizeof(cgd->inq_data.product), sizeof(softc->disk->d_descr) - strlen(softc->disk->d_descr)); softc->disk->d_hba_vendor = cpi.hba_vendor; softc->disk->d_hba_device = cpi.hba_device; softc->disk->d_hba_subvendor = cpi.hba_subvendor; softc->disk->d_hba_subdevice = cpi.hba_subdevice; /* * Acquire a reference to the periph before we register with GEOM. * We'll release this reference once GEOM calls us back (via * dadiskgonecb()) telling us that our provider has been freed. */ if (da_periph_acquire(periph, DA_REF_GEOM) != 0) { xpt_print(periph->path, "%s: lost periph during " "registration!\n", __func__); cam_periph_lock(periph); return (CAM_REQ_CMP_ERR); } disk_create(softc->disk, DISK_VERSION); cam_periph_lock(periph); /* * Add async callbacks for events of interest. * I don't bother checking if this fails as, * in most cases, the system will function just * fine without them and the only alternative * would be to not attach the device on failure. */ xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE | AC_ADVINFO_CHANGED | AC_SCSI_AEN | AC_UNIT_ATTENTION | AC_INQ_CHANGED, daasync, periph, periph->path); /* * Emit an attribute changed notification just in case * physical path information arrived before our async * event handler was registered, but after anyone attaching * to our disk device polled it. */ disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT); /* * Schedule a periodic media polling events. */ callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0); if ((softc->flags & DA_FLAG_PACK_REMOVABLE) && (cgd->inq_flags & SID_AEN) == 0 && da_poll_period != 0) callout_reset(&softc->mediapoll_c, da_poll_period * hz, damediapoll, periph); xpt_schedule(periph, CAM_PRIORITY_DEV); return(CAM_REQ_CMP); } static int da_zone_bio_to_scsi(int disk_zone_cmd) { switch (disk_zone_cmd) { case DISK_ZONE_OPEN: return ZBC_OUT_SA_OPEN; case DISK_ZONE_CLOSE: return ZBC_OUT_SA_CLOSE; case DISK_ZONE_FINISH: return ZBC_OUT_SA_FINISH; case DISK_ZONE_RWP: return ZBC_OUT_SA_RWP; } return -1; } static int da_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp, int *queue_ccb) { struct da_softc *softc; int error; error = 0; if (bp->bio_cmd != BIO_ZONE) { error = EINVAL; goto bailout; } softc = periph->softc; switch (bp->bio_zone.zone_cmd) { case DISK_ZONE_OPEN: case DISK_ZONE_CLOSE: case DISK_ZONE_FINISH: case DISK_ZONE_RWP: { int zone_flags; int zone_sa; uint64_t lba; zone_sa = da_zone_bio_to_scsi(bp->bio_zone.zone_cmd); if (zone_sa == -1) { xpt_print(periph->path, "Cannot translate zone " "cmd %#x to SCSI\n", bp->bio_zone.zone_cmd); error = EINVAL; goto bailout; } zone_flags = 0; lba = bp->bio_zone.zone_params.rwp.id; if (bp->bio_zone.zone_params.rwp.flags & DISK_ZONE_RWP_FLAG_ALL) zone_flags |= ZBC_OUT_ALL; if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) { scsi_zbc_out(&ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*service_action*/ zone_sa, /*zone_id*/ lba, /*zone_flags*/ zone_flags, /*data_ptr*/ NULL, /*dxfer_len*/ 0, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); } else { /* * Note that in this case, even though we can * technically use NCQ, we don't bother for several * reasons: * 1. It hasn't been tested on a SAT layer that * supports it. This is new as of SAT-4. * 2. Even when there is a SAT layer that supports * it, that SAT layer will also probably support * ZBC -> ZAC translation, since they are both * in the SAT-4 spec. * 3. Translation will likely be preferable to ATA * passthrough. LSI / Avago at least single * steps ATA passthrough commands in the HBA, * regardless of protocol, so unless that * changes, there is a performance penalty for * doing ATA passthrough no matter whether * you're using NCQ/FPDMA, DMA or PIO. * 4. It requires a 32-byte CDB, which at least at * this point in CAM requires a CDB pointer, which * would require us to allocate an additional bit * of storage separate from the CCB. */ error = scsi_ata_zac_mgmt_out(&ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*use_ncq*/ 0, /*zm_action*/ zone_sa, /*zone_id*/ lba, /*zone_flags*/ zone_flags, /*data_ptr*/ NULL, /*dxfer_len*/ 0, /*cdb_storage*/ NULL, /*cdb_storage_len*/ 0, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (error != 0) { error = EINVAL; xpt_print(periph->path, "scsi_ata_zac_mgmt_out() returned an " "error!"); goto bailout; } } *queue_ccb = 1; break; } case DISK_ZONE_REPORT_ZONES: { uint8_t *rz_ptr; uint32_t num_entries, alloc_size; struct disk_zone_report *rep; rep = &bp->bio_zone.zone_params.report; num_entries = rep->entries_allocated; if (num_entries == 0) { xpt_print(periph->path, "No entries allocated for " "Report Zones request\n"); error = EINVAL; goto bailout; } alloc_size = sizeof(struct scsi_report_zones_hdr) + (sizeof(struct scsi_report_zones_desc) * num_entries); alloc_size = min(alloc_size, softc->disk->d_maxsize); rz_ptr = malloc(alloc_size, M_SCSIDA, M_NOWAIT | M_ZERO); if (rz_ptr == NULL) { xpt_print(periph->path, "Unable to allocate memory " "for Report Zones request\n"); error = ENOMEM; goto bailout; } if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) { scsi_zbc_in(&ccb->csio, /*retries*/ da_retry_count, /*cbcfnp*/ dadone, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*service_action*/ ZBC_IN_SA_REPORT_ZONES, /*zone_start_lba*/ rep->starting_id, /*zone_options*/ rep->rep_options, /*data_ptr*/ rz_ptr, /*dxfer_len*/ alloc_size, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); } else { /* * Note that in this case, even though we can * technically use NCQ, we don't bother for several * reasons: * 1. It hasn't been tested on a SAT layer that * supports it. This is new as of SAT-4. * 2. Even when there is a SAT layer that supports * it, that SAT layer will also probably support * ZBC -> ZAC translation, since they are both * in the SAT-4 spec. * 3. Translation will likely be preferable to ATA * passthrough. LSI / Avago at least single * steps ATA passthrough commands in the HBA, * regardless of protocol, so unless that * changes, there is a performance penalty for * doing ATA passthrough no matter whether * you're using NCQ/FPDMA, DMA or PIO. * 4. It requires a 32-byte CDB, which at least at * this point in CAM requires a CDB pointer, which * would require us to allocate an additional bit * of storage separate from the CCB. */ error = scsi_ata_zac_mgmt_in(&ccb->csio, /*retries*/ da_retry_count, /*cbcfnp*/ dadone, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*use_ncq*/ 0, /*zm_action*/ ATA_ZM_REPORT_ZONES, /*zone_id*/ rep->starting_id, /*zone_flags*/ rep->rep_options, /*data_ptr*/ rz_ptr, /*dxfer_len*/ alloc_size, /*cdb_storage*/ NULL, /*cdb_storage_len*/ 0, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (error != 0) { error = EINVAL; xpt_print(periph->path, "scsi_ata_zac_mgmt_in() returned an " "error!"); goto bailout; } } /* * For BIO_ZONE, this isn't normally needed. However, it * is used by devstat_end_transaction_bio() to determine * how much data was transferred. */ /* * XXX KDM we have a problem. But I'm not sure how to fix * it. devstat uses bio_bcount - bio_resid to calculate * the amount of data transferred. The GEOM disk code * uses bio_length - bio_resid to calculate the amount of * data in bio_completed. We have different structure * sizes above and below the ada(4) driver. So, if we * use the sizes above, the amount transferred won't be * quite accurate for devstat. If we use different sizes * for bio_bcount and bio_length (above and below * respectively), then the residual needs to match one or * the other. Everything is calculated after the bio * leaves the driver, so changing the values around isn't * really an option. For now, just set the count to the * passed in length. This means that the calculations * above (e.g. bio_completed) will be correct, but the * amount of data reported to devstat will be slightly * under or overstated. */ bp->bio_bcount = bp->bio_length; *queue_ccb = 1; break; } case DISK_ZONE_GET_PARAMS: { struct disk_zone_disk_params *params; params = &bp->bio_zone.zone_params.disk_params; bzero(params, sizeof(*params)); switch (softc->zone_mode) { case DA_ZONE_DRIVE_MANAGED: params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED; break; case DA_ZONE_HOST_AWARE: params->zone_mode = DISK_ZONE_MODE_HOST_AWARE; break; case DA_ZONE_HOST_MANAGED: params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED; break; default: case DA_ZONE_NONE: params->zone_mode = DISK_ZONE_MODE_NONE; break; } if (softc->zone_flags & DA_ZONE_FLAG_URSWRZ) params->flags |= DISK_ZONE_DISK_URSWRZ; if (softc->zone_flags & DA_ZONE_FLAG_OPT_SEQ_SET) { params->optimal_seq_zones = softc->optimal_seq_zones; params->flags |= DISK_ZONE_OPT_SEQ_SET; } if (softc->zone_flags & DA_ZONE_FLAG_OPT_NONSEQ_SET) { params->optimal_nonseq_zones = softc->optimal_nonseq_zones; params->flags |= DISK_ZONE_OPT_NONSEQ_SET; } if (softc->zone_flags & DA_ZONE_FLAG_MAX_SEQ_SET) { params->max_seq_zones = softc->max_seq_zones; params->flags |= DISK_ZONE_MAX_SEQ_SET; } if (softc->zone_flags & DA_ZONE_FLAG_RZ_SUP) params->flags |= DISK_ZONE_RZ_SUP; if (softc->zone_flags & DA_ZONE_FLAG_OPEN_SUP) params->flags |= DISK_ZONE_OPEN_SUP; if (softc->zone_flags & DA_ZONE_FLAG_CLOSE_SUP) params->flags |= DISK_ZONE_CLOSE_SUP; if (softc->zone_flags & DA_ZONE_FLAG_FINISH_SUP) params->flags |= DISK_ZONE_FINISH_SUP; if (softc->zone_flags & DA_ZONE_FLAG_RWP_SUP) params->flags |= DISK_ZONE_RWP_SUP; break; } default: break; } bailout: return (error); } static void dastart(struct cam_periph *periph, union ccb *start_ccb) { struct da_softc *softc; cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastart\n")); skipstate: switch (softc->state) { case DA_STATE_NORMAL: { struct bio *bp; uint8_t tag_code; more: bp = cam_iosched_next_bio(softc->cam_iosched); if (bp == NULL) { if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { softc->flags |= DA_FLAG_TUR_PENDING; cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); scsi_test_unit_ready(&start_ccb->csio, /*retries*/ da_retry_count, dadone_tur, MSG_SIMPLE_Q_TAG, SSD_FULL_SIZE, da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_TUR; xpt_action(start_ccb); } else xpt_release_ccb(start_ccb); break; } if (bp->bio_cmd == BIO_DELETE) { if (softc->delete_func != NULL) { softc->delete_func(periph, start_ccb, bp); goto out; } else { /* * Not sure this is possible, but failsafe by * lying and saying "sure, done." */ biofinish(bp, NULL, 0); goto more; } } if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); da_periph_release_locked(periph, DA_REF_TUR); } if ((bp->bio_flags & BIO_ORDERED) != 0 || (softc->flags & DA_FLAG_NEED_OTAG) != 0) { softc->flags &= ~DA_FLAG_NEED_OTAG; softc->flags |= DA_FLAG_WAS_OTAG; tag_code = MSG_ORDERED_Q_TAG; } else { tag_code = MSG_SIMPLE_Q_TAG; } switch (bp->bio_cmd) { case BIO_WRITE: case BIO_READ: { void *data_ptr; int rw_op; biotrack(bp, __func__); if (bp->bio_cmd == BIO_WRITE) { softc->flags |= DA_FLAG_DIRTY; rw_op = SCSI_RW_WRITE; } else { rw_op = SCSI_RW_READ; } data_ptr = bp->bio_data; if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) { rw_op |= SCSI_RW_BIO; data_ptr = bp; } scsi_read_write(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone, /*tag_action*/tag_code, rw_op, /*byte2*/0, softc->minimum_cmd_size, /*lba*/bp->bio_pblkno, /*block_count*/bp->bio_bcount / softc->params.secsize, data_ptr, /*dxfer_len*/ bp->bio_bcount, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) start_ccb->csio.bio = bp; #endif break; } case BIO_FLUSH: /* * If we don't support sync cache, or the disk * isn't dirty, FLUSH is a no-op. Use the * allocated CCB for the next bio if one is * available. */ if ((softc->quirks & DA_Q_NO_SYNC_CACHE) != 0 || (softc->flags & DA_FLAG_DIRTY) == 0) { biodone(bp); goto skipstate; } /* * BIO_FLUSH doesn't currently communicate * range data, so we synchronize the cache * over the whole disk. */ scsi_synchronize_cache(&start_ccb->csio, /*retries*/1, /*cbfcnp*/dadone, /*tag_action*/tag_code, /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE, da_default_timeout*1000); /* * Clear the dirty flag before sending the command. * Either this sync cache will be successful, or it * will fail after a retry. If it fails, it is * unlikely to be successful if retried later, so * we'll save ourselves time by just marking the * device clean. */ softc->flags &= ~DA_FLAG_DIRTY; break; case BIO_ZONE: { int error, queue_ccb; queue_ccb = 0; error = da_zone_cmd(periph, start_ccb, bp,&queue_ccb); if ((error != 0) || (queue_ccb == 0)) { biofinish(bp, NULL, error); xpt_release_ccb(start_ccb); return; } break; } } start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO; start_ccb->ccb_h.flags |= CAM_UNLOCKED; start_ccb->ccb_h.softtimeout = sbttotv(da_default_softtimeout); out: LIST_INSERT_HEAD(&softc->pending_ccbs, &start_ccb->ccb_h, periph_links.le); /* We expect a unit attention from this device */ if ((softc->flags & DA_FLAG_RETRY_UA) != 0) { start_ccb->ccb_h.ccb_state |= DA_CCB_RETRY_UA; softc->flags &= ~DA_FLAG_RETRY_UA; } start_ccb->ccb_h.ccb_bp = bp; softc->refcount++; cam_periph_unlock(periph); xpt_action(start_ccb); cam_periph_lock(periph); /* May have more work to do, so ensure we stay scheduled */ daschedule(periph); break; } case DA_STATE_PROBE_WP: { void *mode_buf; int mode_buf_len; if (da_disable_wp_detection) { if ((softc->flags & DA_FLAG_CAN_RC16) != 0) softc->state = DA_STATE_PROBE_RC16; else softc->state = DA_STATE_PROBE_RC; goto skipstate; } mode_buf_len = 192; mode_buf = malloc(mode_buf_len, M_SCSIDA, M_NOWAIT); if (mode_buf == NULL) { xpt_print(periph->path, "Unable to send mode sense - " "malloc failure\n"); if ((softc->flags & DA_FLAG_CAN_RC16) != 0) softc->state = DA_STATE_PROBE_RC16; else softc->state = DA_STATE_PROBE_RC; goto skipstate; } scsi_mode_sense_len(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_probewp, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*dbd*/ FALSE, /*pc*/ SMS_PAGE_CTRL_CURRENT, /*page*/ SMS_ALL_PAGES_PAGE, /*param_buf*/ mode_buf, /*param_len*/ mode_buf_len, /*minimum_cmd_size*/ softc->minimum_cmd_size, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_WP; xpt_action(start_ccb); break; } case DA_STATE_PROBE_RC: { struct scsi_read_capacity_data *rcap; rcap = (struct scsi_read_capacity_data *) malloc(sizeof(*rcap), M_SCSIDA, M_NOWAIT|M_ZERO); if (rcap == NULL) { printf("dastart: Couldn't malloc read_capacity data\n"); /* da_free_periph??? */ break; } scsi_read_capacity(&start_ccb->csio, /*retries*/da_retry_count, dadone_proberc, MSG_SIMPLE_Q_TAG, rcap, SSD_FULL_SIZE, /*timeout*/5000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC; xpt_action(start_ccb); break; } case DA_STATE_PROBE_RC16: { struct scsi_read_capacity_data_long *rcaplong; rcaplong = (struct scsi_read_capacity_data_long *) malloc(sizeof(*rcaplong), M_SCSIDA, M_NOWAIT|M_ZERO); if (rcaplong == NULL) { printf("dastart: Couldn't malloc read_capacity data\n"); /* da_free_periph??? */ break; } scsi_read_capacity_16(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_proberc, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*lba*/ 0, /*reladr*/ 0, /*pmi*/ 0, /*rcap_buf*/ (uint8_t *)rcaplong, /*rcap_buf_len*/ sizeof(*rcaplong), /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC16; xpt_action(start_ccb); break; } case DA_STATE_PROBE_LBP: { struct scsi_vpd_logical_block_prov *lbp; if (!scsi_vpd_supported_page(periph, SVPD_LBP)) { /* * If we get here we don't support any SBC-3 delete * methods with UNMAP as the Logical Block Provisioning * VPD page support is required for devices which * support it according to T10/1799-D Revision 31 * however older revisions of the spec don't mandate * this so we currently don't remove these methods * from the available set. */ softc->state = DA_STATE_PROBE_BLK_LIMITS; goto skipstate; } lbp = (struct scsi_vpd_logical_block_prov *) malloc(sizeof(*lbp), M_SCSIDA, M_NOWAIT|M_ZERO); if (lbp == NULL) { printf("dastart: Couldn't malloc lbp data\n"); /* da_free_periph??? */ break; } scsi_inquiry(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone_probelbp, /*tag_action*/MSG_SIMPLE_Q_TAG, /*inq_buf*/(u_int8_t *)lbp, /*inq_len*/sizeof(*lbp), /*evpd*/TRUE, /*page_code*/SVPD_LBP, /*sense_len*/SSD_MIN_SIZE, /*timeout*/da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_LBP; xpt_action(start_ccb); break; } case DA_STATE_PROBE_BLK_LIMITS: { struct scsi_vpd_block_limits *block_limits; if (!scsi_vpd_supported_page(periph, SVPD_BLOCK_LIMITS)) { /* Not supported skip to next probe */ softc->state = DA_STATE_PROBE_BDC; goto skipstate; } block_limits = (struct scsi_vpd_block_limits *) malloc(sizeof(*block_limits), M_SCSIDA, M_NOWAIT|M_ZERO); if (block_limits == NULL) { printf("dastart: Couldn't malloc block_limits data\n"); /* da_free_periph??? */ break; } scsi_inquiry(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone_probeblklimits, /*tag_action*/MSG_SIMPLE_Q_TAG, /*inq_buf*/(u_int8_t *)block_limits, /*inq_len*/sizeof(*block_limits), /*evpd*/TRUE, /*page_code*/SVPD_BLOCK_LIMITS, /*sense_len*/SSD_MIN_SIZE, /*timeout*/da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BLK_LIMITS; xpt_action(start_ccb); break; } case DA_STATE_PROBE_BDC: { struct scsi_vpd_block_characteristics *bdc; if (!scsi_vpd_supported_page(periph, SVPD_BDC)) { softc->state = DA_STATE_PROBE_ATA; goto skipstate; } bdc = (struct scsi_vpd_block_characteristics *) malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO); if (bdc == NULL) { printf("dastart: Couldn't malloc bdc data\n"); /* da_free_periph??? */ break; } scsi_inquiry(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone_probebdc, /*tag_action*/MSG_SIMPLE_Q_TAG, /*inq_buf*/(u_int8_t *)bdc, /*inq_len*/sizeof(*bdc), /*evpd*/TRUE, /*page_code*/SVPD_BDC, /*sense_len*/SSD_MIN_SIZE, /*timeout*/da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BDC; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ATA: { struct ata_params *ata_params; if (!scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) { if ((softc->zone_mode == DA_ZONE_HOST_AWARE) || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) { /* * Note that if the ATA VPD page isn't * supported, we aren't talking to an ATA * device anyway. Support for that VPD * page is mandatory for SCSI to ATA (SAT) * translation layers. */ softc->state = DA_STATE_PROBE_ZONE; goto skipstate; } daprobedone(periph, start_ccb); break; } ata_params = (struct ata_params*) malloc(sizeof(*ata_params), M_SCSIDA,M_NOWAIT|M_ZERO); if (ata_params == NULL) { xpt_print(periph->path, "Couldn't malloc ata_params " "data\n"); /* da_free_periph??? */ break; } scsi_ata_identify(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone_probeata, /*tag_action*/MSG_SIMPLE_Q_TAG, /*data_ptr*/(u_int8_t *)ata_params, /*dxfer_len*/sizeof(*ata_params), /*sense_len*/SSD_FULL_SIZE, /*timeout*/da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ATA_LOGDIR: { struct ata_gp_log_dir *log_dir; int retval; retval = 0; if ((softc->flags & DA_FLAG_CAN_ATA_LOG) == 0) { /* * If we don't have log support, not much point in * trying to probe zone support. */ daprobedone(periph, start_ccb); break; } /* * If we have an ATA device (the SCSI ATA Information VPD * page should be present and the ATA identify should have * succeeded) and it supports logs, ask for the log directory. */ log_dir = malloc(sizeof(*log_dir), M_SCSIDA, M_NOWAIT|M_ZERO); if (log_dir == NULL) { xpt_print(periph->path, "Couldn't malloc log_dir " "data\n"); daprobedone(periph, start_ccb); break; } retval = scsi_ata_read_log(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_probeatalogdir, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*log_address*/ ATA_LOG_DIRECTORY, /*page_number*/ 0, /*block_count*/ 1, /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ? AP_PROTO_DMA : AP_PROTO_PIO_IN, /*data_ptr*/ (uint8_t *)log_dir, /*dxfer_len*/ sizeof(*log_dir), /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (retval != 0) { xpt_print(periph->path, "scsi_ata_read_log() failed!"); free(log_dir, M_SCSIDA); daprobedone(periph, start_ccb); break; } start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_LOGDIR; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ATA_IDDIR: { struct ata_identify_log_pages *id_dir; int retval; retval = 0; /* * Check here to see whether the Identify Device log is * supported in the directory of logs. If so, continue * with requesting the log of identify device pages. */ if ((softc->flags & DA_FLAG_CAN_ATA_IDLOG) == 0) { daprobedone(periph, start_ccb); break; } id_dir = malloc(sizeof(*id_dir), M_SCSIDA, M_NOWAIT | M_ZERO); if (id_dir == NULL) { xpt_print(periph->path, "Couldn't malloc id_dir " "data\n"); daprobedone(periph, start_ccb); break; } retval = scsi_ata_read_log(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_probeataiddir, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*log_address*/ ATA_IDENTIFY_DATA_LOG, /*page_number*/ ATA_IDL_PAGE_LIST, /*block_count*/ 1, /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ? AP_PROTO_DMA : AP_PROTO_PIO_IN, /*data_ptr*/ (uint8_t *)id_dir, /*dxfer_len*/ sizeof(*id_dir), /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (retval != 0) { xpt_print(periph->path, "scsi_ata_read_log() failed!"); free(id_dir, M_SCSIDA); daprobedone(periph, start_ccb); break; } start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_IDDIR; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ATA_SUP: { struct ata_identify_log_sup_cap *sup_cap; int retval; retval = 0; /* * Check here to see whether the Supported Capabilities log * is in the list of Identify Device logs. */ if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP) == 0) { daprobedone(periph, start_ccb); break; } sup_cap = malloc(sizeof(*sup_cap), M_SCSIDA, M_NOWAIT|M_ZERO); if (sup_cap == NULL) { xpt_print(periph->path, "Couldn't malloc sup_cap " "data\n"); daprobedone(periph, start_ccb); break; } retval = scsi_ata_read_log(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_probeatasup, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*log_address*/ ATA_IDENTIFY_DATA_LOG, /*page_number*/ ATA_IDL_SUP_CAP, /*block_count*/ 1, /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ? AP_PROTO_DMA : AP_PROTO_PIO_IN, /*data_ptr*/ (uint8_t *)sup_cap, /*dxfer_len*/ sizeof(*sup_cap), /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (retval != 0) { xpt_print(periph->path, "scsi_ata_read_log() failed!"); free(sup_cap, M_SCSIDA); daprobedone(periph, start_ccb); break; } start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_SUP; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ATA_ZONE: { struct ata_zoned_info_log *ata_zone; int retval; retval = 0; /* * Check here to see whether the zoned device information * page is supported. If so, continue on to request it. * If not, skip to DA_STATE_PROBE_LOG or done. */ if ((softc->flags & DA_FLAG_CAN_ATA_ZONE) == 0) { daprobedone(periph, start_ccb); break; } ata_zone = malloc(sizeof(*ata_zone), M_SCSIDA, M_NOWAIT|M_ZERO); if (ata_zone == NULL) { xpt_print(periph->path, "Couldn't malloc ata_zone " "data\n"); daprobedone(periph, start_ccb); break; } retval = scsi_ata_read_log(&start_ccb->csio, /*retries*/ da_retry_count, /*cbfcnp*/ dadone_probeatazone, /*tag_action*/ MSG_SIMPLE_Q_TAG, /*log_address*/ ATA_IDENTIFY_DATA_LOG, /*page_number*/ ATA_IDL_ZDI, /*block_count*/ 1, /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ? AP_PROTO_DMA : AP_PROTO_PIO_IN, /*data_ptr*/ (uint8_t *)ata_zone, /*dxfer_len*/ sizeof(*ata_zone), /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ da_default_timeout * 1000); if (retval != 0) { xpt_print(periph->path, "scsi_ata_read_log() failed!"); free(ata_zone, M_SCSIDA); daprobedone(periph, start_ccb); break; } start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_ZONE; xpt_action(start_ccb); break; } case DA_STATE_PROBE_ZONE: { struct scsi_vpd_zoned_bdc *bdc; /* * Note that this page will be supported for SCSI protocol * devices that support ZBC (SMR devices), as well as ATA * protocol devices that are behind a SAT (SCSI to ATA * Translation) layer that supports converting ZBC commands * to their ZAC equivalents. */ if (!scsi_vpd_supported_page(periph, SVPD_ZONED_BDC)) { daprobedone(periph, start_ccb); break; } bdc = (struct scsi_vpd_zoned_bdc *) malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO); if (bdc == NULL) { xpt_release_ccb(start_ccb); xpt_print(periph->path, "Couldn't malloc zone VPD " "data\n"); break; } scsi_inquiry(&start_ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone_probezone, /*tag_action*/MSG_SIMPLE_Q_TAG, /*inq_buf*/(u_int8_t *)bdc, /*inq_len*/sizeof(*bdc), /*evpd*/TRUE, /*page_code*/SVPD_ZONED_BDC, /*sense_len*/SSD_FULL_SIZE, /*timeout*/da_default_timeout * 1000); start_ccb->ccb_h.ccb_bp = NULL; start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ZONE; xpt_action(start_ccb); break; } } } /* * In each of the methods below, while its the caller's * responsibility to ensure the request will fit into a * single device request, we might have changed the delete * method due to the device incorrectly advertising either * its supported methods or limits. * * To prevent this causing further issues we validate the * against the methods limits, and warn which would * otherwise be unnecessary. */ static void da_delete_unmap(struct cam_periph *periph, union ccb *ccb, struct bio *bp) { struct da_softc *softc = (struct da_softc *)periph->softc;; struct bio *bp1; uint8_t *buf = softc->unmap_buf; struct scsi_unmap_desc *d = (void *)&buf[UNMAP_HEAD_SIZE]; uint64_t lba, lastlba = (uint64_t)-1; uint64_t totalcount = 0; uint64_t count; uint32_t c, lastcount = 0, ranges = 0; /* * Currently this doesn't take the UNMAP * Granularity and Granularity Alignment * fields into account. * * This could result in both unoptimal unmap * requests as as well as UNMAP calls unmapping * fewer LBA's than requested. */ bzero(softc->unmap_buf, sizeof(softc->unmap_buf)); bp1 = bp; do { /* * Note: ada and da are different in how they store the * pending bp's in a trim. ada stores all of them in the * trim_req.bps. da stores all but the first one in the * delete_run_queue. ada then completes all the bps in * its adadone() loop. da completes all the bps in the * delete_run_queue in dadone, and relies on the biodone * after to complete. This should be reconciled since there's * no real reason to do it differently. XXX */ if (bp1 != bp) bioq_insert_tail(&softc->delete_run_queue, bp1); lba = bp1->bio_pblkno; count = bp1->bio_bcount / softc->params.secsize; /* Try to extend the previous range. */ if (lba == lastlba) { c = omin(count, UNMAP_RANGE_MAX - lastcount); lastlba += c; lastcount += c; scsi_ulto4b(lastcount, d[ranges - 1].length); count -= c; lba += c; totalcount += c; } else if ((softc->quirks & DA_Q_STRICT_UNMAP) && softc->unmap_gran != 0) { /* Align length of the previous range. */ if ((c = lastcount % softc->unmap_gran) != 0) { if (lastcount <= c) { totalcount -= lastcount; lastlba = (uint64_t)-1; lastcount = 0; ranges--; } else { totalcount -= c; lastlba -= c; lastcount -= c; scsi_ulto4b(lastcount, d[ranges - 1].length); } } /* Align beginning of the new range. */ c = (lba - softc->unmap_gran_align) % softc->unmap_gran; if (c != 0) { c = softc->unmap_gran - c; if (count <= c) { count = 0; } else { lba += c; count -= c; } } } while (count > 0) { c = omin(count, UNMAP_RANGE_MAX); if (totalcount + c > softc->unmap_max_lba || ranges >= softc->unmap_max_ranges) { xpt_print(periph->path, "%s issuing short delete %ld > %ld" "|| %d >= %d", da_delete_method_desc[softc->delete_method], totalcount + c, softc->unmap_max_lba, ranges, softc->unmap_max_ranges); break; } scsi_u64to8b(lba, d[ranges].lba); scsi_ulto4b(c, d[ranges].length); lba += c; totalcount += c; ranges++; count -= c; lastlba = lba; lastcount = c; } bp1 = cam_iosched_next_trim(softc->cam_iosched); if (bp1 == NULL) break; if (ranges >= softc->unmap_max_ranges || totalcount + bp1->bio_bcount / softc->params.secsize > softc->unmap_max_lba) { cam_iosched_put_back_trim(softc->cam_iosched, bp1); break; } } while (1); /* Align length of the last range. */ if ((softc->quirks & DA_Q_STRICT_UNMAP) && softc->unmap_gran != 0 && (c = lastcount % softc->unmap_gran) != 0) { if (lastcount <= c) ranges--; else scsi_ulto4b(lastcount - c, d[ranges - 1].length); } scsi_ulto2b(ranges * 16 + 6, &buf[0]); scsi_ulto2b(ranges * 16, &buf[2]); scsi_unmap(&ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone, /*tag_action*/MSG_SIMPLE_Q_TAG, /*byte2*/0, /*data_ptr*/ buf, /*dxfer_len*/ ranges * 16 + 8, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; ccb->ccb_h.flags |= CAM_UNLOCKED; softc->trim_count++; softc->trim_ranges += ranges; softc->trim_lbas += totalcount; cam_iosched_submit_trim(softc->cam_iosched); } static void da_delete_trim(struct cam_periph *periph, union ccb *ccb, struct bio *bp) { struct da_softc *softc = (struct da_softc *)periph->softc; struct bio *bp1; uint8_t *buf = softc->unmap_buf; uint64_t lastlba = (uint64_t)-1; uint64_t count; uint64_t lba; uint32_t lastcount = 0, c, requestcount; int ranges = 0, off, block_count; bzero(softc->unmap_buf, sizeof(softc->unmap_buf)); bp1 = bp; do { if (bp1 != bp)//XXX imp XXX bioq_insert_tail(&softc->delete_run_queue, bp1); lba = bp1->bio_pblkno; count = bp1->bio_bcount / softc->params.secsize; requestcount = count; /* Try to extend the previous range. */ if (lba == lastlba) { c = omin(count, ATA_DSM_RANGE_MAX - lastcount); lastcount += c; off = (ranges - 1) * 8; buf[off + 6] = lastcount & 0xff; buf[off + 7] = (lastcount >> 8) & 0xff; count -= c; lba += c; } while (count > 0) { c = omin(count, ATA_DSM_RANGE_MAX); off = ranges * 8; buf[off + 0] = lba & 0xff; buf[off + 1] = (lba >> 8) & 0xff; buf[off + 2] = (lba >> 16) & 0xff; buf[off + 3] = (lba >> 24) & 0xff; buf[off + 4] = (lba >> 32) & 0xff; buf[off + 5] = (lba >> 40) & 0xff; buf[off + 6] = c & 0xff; buf[off + 7] = (c >> 8) & 0xff; lba += c; ranges++; count -= c; lastcount = c; if (count != 0 && ranges == softc->trim_max_ranges) { xpt_print(periph->path, "%s issuing short delete %ld > %ld\n", da_delete_method_desc[softc->delete_method], requestcount, (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX); break; } } lastlba = lba; bp1 = cam_iosched_next_trim(softc->cam_iosched); if (bp1 == NULL) break; if (bp1->bio_bcount / softc->params.secsize > (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) { cam_iosched_put_back_trim(softc->cam_iosched, bp1); break; } } while (1); block_count = howmany(ranges, ATA_DSM_BLK_RANGES); scsi_ata_trim(&ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone, /*tag_action*/MSG_SIMPLE_Q_TAG, block_count, /*data_ptr*/buf, /*dxfer_len*/block_count * ATA_DSM_BLK_SIZE, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; ccb->ccb_h.flags |= CAM_UNLOCKED; cam_iosched_submit_trim(softc->cam_iosched); } /* * We calculate ws_max_blks here based off d_delmaxsize instead * of using softc->ws_max_blks as it is absolute max for the * device not the protocol max which may well be lower. */ static void da_delete_ws(struct cam_periph *periph, union ccb *ccb, struct bio *bp) { struct da_softc *softc; struct bio *bp1; uint64_t ws_max_blks; uint64_t lba; uint64_t count; /* forward compat with WS32 */ softc = (struct da_softc *)periph->softc; ws_max_blks = softc->disk->d_delmaxsize / softc->params.secsize; lba = bp->bio_pblkno; count = 0; bp1 = bp; do { if (bp1 != bp)//XXX imp XXX bioq_insert_tail(&softc->delete_run_queue, bp1); count += bp1->bio_bcount / softc->params.secsize; if (count > ws_max_blks) { xpt_print(periph->path, "%s issuing short delete %ld > %ld\n", da_delete_method_desc[softc->delete_method], count, ws_max_blks); count = omin(count, ws_max_blks); break; } bp1 = cam_iosched_next_trim(softc->cam_iosched); if (bp1 == NULL) break; if (lba + count != bp1->bio_pblkno || count + bp1->bio_bcount / softc->params.secsize > ws_max_blks) { cam_iosched_put_back_trim(softc->cam_iosched, bp1); break; } } while (1); scsi_write_same(&ccb->csio, /*retries*/da_retry_count, /*cbfcnp*/dadone, /*tag_action*/MSG_SIMPLE_Q_TAG, /*byte2*/softc->delete_method == DA_DELETE_ZERO ? 0 : SWS_UNMAP, softc->delete_method == DA_DELETE_WS16 ? 16 : 10, /*lba*/lba, /*block_count*/count, /*data_ptr*/ __DECONST(void *, zero_region), /*dxfer_len*/ softc->params.secsize, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; ccb->ccb_h.flags |= CAM_UNLOCKED; cam_iosched_submit_trim(softc->cam_iosched); } static int cmd6workaround(union ccb *ccb) { struct scsi_rw_6 cmd6; struct scsi_rw_10 *cmd10; struct da_softc *softc; u_int8_t *cdb; struct bio *bp; int frozen; cdb = ccb->csio.cdb_io.cdb_bytes; softc = (struct da_softc *)xpt_path_periph(ccb->ccb_h.path)->softc; if (ccb->ccb_h.ccb_state == DA_CCB_DELETE) { da_delete_methods old_method = softc->delete_method; /* * Typically there are two reasons for failure here * 1. Delete method was detected as supported but isn't * 2. Delete failed due to invalid params e.g. too big * * While we will attempt to choose an alternative delete method * this may result in short deletes if the existing delete * requests from geom are big for the new method chosen. * * This method assumes that the error which triggered this * will not retry the io otherwise a panic will occur */ dadeleteflag(softc, old_method, 0); dadeletemethodchoose(softc, DA_DELETE_DISABLE); if (softc->delete_method == DA_DELETE_DISABLE) xpt_print(ccb->ccb_h.path, "%s failed, disabling BIO_DELETE\n", da_delete_method_desc[old_method]); else xpt_print(ccb->ccb_h.path, "%s failed, switching to %s BIO_DELETE\n", da_delete_method_desc[old_method], da_delete_method_desc[softc->delete_method]); while ((bp = bioq_takefirst(&softc->delete_run_queue)) != NULL) cam_iosched_queue_work(softc->cam_iosched, bp); cam_iosched_queue_work(softc->cam_iosched, (struct bio *)ccb->ccb_h.ccb_bp); ccb->ccb_h.ccb_bp = NULL; return (0); } /* Detect unsupported PREVENT ALLOW MEDIUM REMOVAL. */ if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 && (*cdb == PREVENT_ALLOW) && (softc->quirks & DA_Q_NO_PREVENT) == 0) { if (bootverbose) xpt_print(ccb->ccb_h.path, "PREVENT ALLOW MEDIUM REMOVAL not supported.\n"); softc->quirks |= DA_Q_NO_PREVENT; return (0); } /* Detect unsupported SYNCHRONIZE CACHE(10). */ if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 && (*cdb == SYNCHRONIZE_CACHE) && (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) { if (bootverbose) xpt_print(ccb->ccb_h.path, "SYNCHRONIZE CACHE(10) not supported.\n"); softc->quirks |= DA_Q_NO_SYNC_CACHE; softc->disk->d_flags &= ~DISKFLAG_CANFLUSHCACHE; return (0); } /* Translation only possible if CDB is an array and cmd is R/W6 */ if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0 || (*cdb != READ_6 && *cdb != WRITE_6)) return 0; xpt_print(ccb->ccb_h.path, "READ(6)/WRITE(6) not supported, " "increasing minimum_cmd_size to 10.\n"); softc->minimum_cmd_size = 10; bcopy(cdb, &cmd6, sizeof(struct scsi_rw_6)); cmd10 = (struct scsi_rw_10 *)cdb; cmd10->opcode = (cmd6.opcode == READ_6) ? READ_10 : WRITE_10; cmd10->byte2 = 0; scsi_ulto4b(scsi_3btoul(cmd6.addr), cmd10->addr); cmd10->reserved = 0; scsi_ulto2b(cmd6.length, cmd10->length); cmd10->control = cmd6.control; ccb->csio.cdb_len = sizeof(*cmd10); /* Requeue request, unfreezing queue if necessary */ frozen = (ccb->ccb_h.status & CAM_DEV_QFRZN) != 0; ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_action(ccb); if (frozen) { cam_release_devq(ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } return (ERESTART); } static void dazonedone(struct cam_periph *periph, union ccb *ccb) { struct da_softc *softc; struct bio *bp; softc = periph->softc; bp = (struct bio *)ccb->ccb_h.ccb_bp; switch (bp->bio_zone.zone_cmd) { case DISK_ZONE_OPEN: case DISK_ZONE_CLOSE: case DISK_ZONE_FINISH: case DISK_ZONE_RWP: break; case DISK_ZONE_REPORT_ZONES: { uint32_t avail_len; struct disk_zone_report *rep; struct scsi_report_zones_hdr *hdr; struct scsi_report_zones_desc *desc; struct disk_zone_rep_entry *entry; uint32_t hdr_len, num_avail; uint32_t num_to_fill, i; int ata; rep = &bp->bio_zone.zone_params.report; avail_len = ccb->csio.dxfer_len - ccb->csio.resid; /* * Note that bio_resid isn't normally used for zone * commands, but it is used by devstat_end_transaction_bio() * to determine how much data was transferred. Because * the size of the SCSI/ATA data structures is different * than the size of the BIO interface structures, the * amount of data actually transferred from the drive will * be different than the amount of data transferred to * the user. */ bp->bio_resid = ccb->csio.resid; hdr = (struct scsi_report_zones_hdr *)ccb->csio.data_ptr; if (avail_len < sizeof(*hdr)) { /* * Is there a better error than EIO here? We asked * for at least the header, and we got less than * that. */ bp->bio_error = EIO; bp->bio_flags |= BIO_ERROR; bp->bio_resid = bp->bio_bcount; break; } if (softc->zone_interface == DA_ZONE_IF_ATA_PASS) ata = 1; else ata = 0; hdr_len = ata ? le32dec(hdr->length) : scsi_4btoul(hdr->length); if (hdr_len > 0) rep->entries_available = hdr_len / sizeof(*desc); else rep->entries_available = 0; /* * NOTE: using the same values for the BIO version of the * same field as the SCSI/ATA values. This means we could * get some additional values that aren't defined in bio.h * if more values of the same field are defined later. */ rep->header.same = hdr->byte4 & SRZ_SAME_MASK; rep->header.maximum_lba = ata ? le64dec(hdr->maximum_lba) : scsi_8btou64(hdr->maximum_lba); /* * If the drive reports no entries that match the query, * we're done. */ if (hdr_len == 0) { rep->entries_filled = 0; break; } num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc), hdr_len / sizeof(*desc)); /* * If the drive didn't return any data, then we're done. */ if (num_avail == 0) { rep->entries_filled = 0; break; } num_to_fill = min(num_avail, rep->entries_allocated); /* * If the user didn't allocate any entries for us to fill, * we're done. */ if (num_to_fill == 0) { rep->entries_filled = 0; break; } for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0]; i < num_to_fill; i++, desc++, entry++) { /* * NOTE: we're mapping the values here directly * from the SCSI/ATA bit definitions to the bio.h * definitons. There is also a warning in * disk_zone.h, but the impact is that if * additional values are added in the SCSI/ATA * specs these will be visible to consumers of * this interface. */ entry->zone_type = desc->zone_type & SRZ_TYPE_MASK; entry->zone_condition = (desc->zone_flags & SRZ_ZONE_COND_MASK) >> SRZ_ZONE_COND_SHIFT; entry->zone_flags |= desc->zone_flags & (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET); entry->zone_length = ata ? le64dec(desc->zone_length) : scsi_8btou64(desc->zone_length); entry->zone_start_lba = ata ? le64dec(desc->zone_start_lba) : scsi_8btou64(desc->zone_start_lba); entry->write_pointer_lba = ata ? le64dec(desc->write_pointer_lba) : scsi_8btou64(desc->write_pointer_lba); } rep->entries_filled = num_to_fill; break; } case DISK_ZONE_GET_PARAMS: default: /* * In theory we should not get a GET_PARAMS bio, since it * should be handled without queueing the command to the * drive. */ panic("%s: Invalid zone command %d", __func__, bp->bio_zone.zone_cmd); break; } if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES) free(ccb->csio.data_ptr, M_SCSIDA); } static void dadone(struct cam_periph *periph, union ccb *done_ccb) { struct bio *bp, *bp1; struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; da_ccb_state state; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) if (csio->bio != NULL) biotrack(csio->bio, __func__); #endif state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK; cam_periph_lock(periph); bp = (struct bio *)done_ccb->ccb_h.ccb_bp; if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { int error; int sf; if ((csio->ccb_h.ccb_state & DA_CCB_RETRY_UA) != 0) sf = SF_RETRY_UA; else sf = 0; error = daerror(done_ccb, CAM_RETRY_SELTO, sf); if (error == ERESTART) { /* A retry was scheduled, so just return. */ cam_periph_unlock(periph); return; } bp = (struct bio *)done_ccb->ccb_h.ccb_bp; if (error != 0) { int queued_error; /* * return all queued I/O with EIO, so that * the client can retry these I/Os in the * proper order should it attempt to recover. */ queued_error = EIO; if (error == ENXIO && (softc->flags & DA_FLAG_PACK_INVALID)== 0) { /* * Catastrophic error. Mark our pack as * invalid. * * XXX See if this is really a media * XXX change first? */ xpt_print(periph->path, "Invalidating pack\n"); softc->flags |= DA_FLAG_PACK_INVALID; #ifdef CAM_IO_STATS softc->invalidations++; #endif queued_error = ENXIO; } cam_iosched_flush(softc->cam_iosched, NULL, queued_error); if (bp != NULL) { bp->bio_error = error; bp->bio_resid = bp->bio_bcount; bp->bio_flags |= BIO_ERROR; } } else if (bp != NULL) { if (state == DA_CCB_DELETE) bp->bio_resid = 0; else bp->bio_resid = csio->resid; bp->bio_error = 0; if (bp->bio_resid != 0) bp->bio_flags |= BIO_ERROR; } if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } else if (bp != NULL) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) panic("REQ_CMP with QFRZN"); if (bp->bio_cmd == BIO_ZONE) dazonedone(periph, done_ccb); else if (state == DA_CCB_DELETE) bp->bio_resid = 0; else bp->bio_resid = csio->resid; if ((csio->resid > 0) && (bp->bio_cmd != BIO_ZONE)) bp->bio_flags |= BIO_ERROR; if (softc->error_inject != 0) { bp->bio_error = softc->error_inject; bp->bio_resid = bp->bio_bcount; bp->bio_flags |= BIO_ERROR; softc->error_inject = 0; } } if (bp != NULL) biotrack(bp, __func__); LIST_REMOVE(&done_ccb->ccb_h, periph_links.le); if (LIST_EMPTY(&softc->pending_ccbs)) softc->flags |= DA_FLAG_WAS_OTAG; /* * We need to call cam_iosched before we call biodone so that we don't * measure any activity that happens in the completion routine, which in * the case of sendfile can be quite extensive. Release the periph * refcount taken in dastart() for each CCB. */ cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb); xpt_release_ccb(done_ccb); KASSERT(softc->refcount >= 1, ("dadone softc %p refcount %d", softc, softc->refcount)); softc->refcount--; if (state == DA_CCB_DELETE) { TAILQ_HEAD(, bio) queue; TAILQ_INIT(&queue); TAILQ_CONCAT(&queue, &softc->delete_run_queue.queue, bio_queue); softc->delete_run_queue.insert_point = NULL; /* * Normally, the xpt_release_ccb() above would make sure * that when we have more work to do, that work would * get kicked off. However, we specifically keep * delete_running set to 0 before the call above to * allow other I/O to progress when many BIO_DELETE * requests are pushed down. We set delete_running to 0 * and call daschedule again so that we don't stall if * there are no other I/Os pending apart from BIO_DELETEs. */ cam_iosched_trim_done(softc->cam_iosched); daschedule(periph); cam_periph_unlock(periph); while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { TAILQ_REMOVE(&queue, bp1, bio_queue); bp1->bio_error = bp->bio_error; if (bp->bio_flags & BIO_ERROR) { bp1->bio_flags |= BIO_ERROR; bp1->bio_resid = bp1->bio_bcount; } else bp1->bio_resid = 0; biodone(bp1); } } else { daschedule(periph); cam_periph_unlock(periph); } if (bp != NULL) biodone(bp); return; } static void dadone_probewp(struct cam_periph *periph, union ccb *done_ccb) { struct scsi_mode_header_6 *mode_hdr6; struct scsi_mode_header_10 *mode_hdr10; struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; uint8_t dev_spec; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probewp\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if (softc->minimum_cmd_size > 6) { mode_hdr10 = (struct scsi_mode_header_10 *)csio->data_ptr; dev_spec = mode_hdr10->dev_spec; } else { mode_hdr6 = (struct scsi_mode_header_6 *)csio->data_ptr; dev_spec = mode_hdr6->dev_spec; } if (cam_ccb_status(done_ccb) == CAM_REQ_CMP) { if ((dev_spec & 0x80) != 0) softc->disk->d_flags |= DISKFLAG_WRITE_PROTECT; else softc->disk->d_flags &= ~DISKFLAG_WRITE_PROTECT; } else { int error; error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); xpt_release_ccb(done_ccb); if ((softc->flags & DA_FLAG_CAN_RC16) != 0) softc->state = DA_STATE_PROBE_RC16; else softc->state = DA_STATE_PROBE_RC; xpt_schedule(periph, priority); return; } static void dadone_proberc(struct cam_periph *periph, union ccb *done_ccb) { struct scsi_read_capacity_data *rdcap; struct scsi_read_capacity_data_long *rcaplong; struct da_softc *softc; struct ccb_scsiio *csio; da_ccb_state state; char *announce_buf; u_int32_t priority; int lbp; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_proberc\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK; lbp = 0; rdcap = NULL; rcaplong = NULL; /* XXX TODO: can this be a malloc? */ announce_buf = softc->announce_temp; bzero(announce_buf, DA_ANNOUNCETMP_SZ); if (state == DA_CCB_PROBE_RC) rdcap =(struct scsi_read_capacity_data *)csio->data_ptr; else rcaplong = (struct scsi_read_capacity_data_long *) csio->data_ptr; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { struct disk_params *dp; uint32_t block_size; uint64_t maxsector; u_int lalba; /* Lowest aligned LBA. */ if (state == DA_CCB_PROBE_RC) { block_size = scsi_4btoul(rdcap->length); maxsector = scsi_4btoul(rdcap->addr); lalba = 0; /* * According to SBC-2, if the standard 10 * byte READ CAPACITY command returns 2^32, * we should issue the 16 byte version of * the command, since the device in question * has more sectors than can be represented * with the short version of the command. */ if (maxsector == 0xffffffff) { free(rdcap, M_SCSIDA); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_RC16; xpt_schedule(periph, priority); return; } } else { block_size = scsi_4btoul(rcaplong->length); maxsector = scsi_8btou64(rcaplong->addr); lalba = scsi_2btoul(rcaplong->lalba_lbp); } /* * Because GEOM code just will panic us if we * give them an 'illegal' value we'll avoid that * here. */ if (block_size == 0) { block_size = 512; if (maxsector == 0) maxsector = -1; } if (block_size >= MAXPHYS) { xpt_print(periph->path, "unsupportable block size %ju\n", (uintmax_t) block_size); announce_buf = NULL; cam_periph_invalidate(periph); } else { /* * We pass rcaplong into dasetgeom(), * because it will only use it if it is * non-NULL. */ dasetgeom(periph, block_size, maxsector, rcaplong, sizeof(*rcaplong)); lbp = (lalba & SRC16_LBPME_A); dp = &softc->params; snprintf(announce_buf, DA_ANNOUNCETMP_SZ, "%juMB (%ju %u byte sectors)", ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024), (uintmax_t)dp->sectors, dp->secsize); } } else { int error; /* * Retry any UNIT ATTENTION type errors. They * are expected at boot. */ error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) { /* * A retry was scheuled, so * just return. */ return; } else if (error != 0) { int asc, ascq; int sense_key, error_code; int have_sense; cam_status status; struct ccb_getdev cgd; /* Don't wedge this device's queue */ status = done_ccb->ccb_h.status; if ((status & CAM_DEV_QFRZN) != 0) cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); xpt_setup_ccb(&cgd.ccb_h, done_ccb->ccb_h.path, CAM_PRIORITY_NORMAL); cgd.ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)&cgd); if (scsi_extract_sense_ccb(done_ccb, &error_code, &sense_key, &asc, &ascq)) have_sense = TRUE; else have_sense = FALSE; /* * If we tried READ CAPACITY(16) and failed, * fallback to READ CAPACITY(10). */ if ((state == DA_CCB_PROBE_RC16) && (softc->flags & DA_FLAG_CAN_RC16) && (((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INVALID) || ((have_sense) && (error_code == SSD_CURRENT_ERROR || error_code == SSD_DESC_CURRENT_ERROR) && (sense_key == SSD_KEY_ILLEGAL_REQUEST)))) { cam_periph_assert(periph, MA_OWNED); softc->flags &= ~DA_FLAG_CAN_RC16; free(rdcap, M_SCSIDA); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_RC; xpt_schedule(periph, priority); return; } /* * Attach to anything that claims to be a * direct access or optical disk device, * as long as it doesn't return a "Logical * unit not supported" (0x25) error. * "Internal Target Failure" (0x44) is also * special and typically means that the * device is a SATA drive behind a SATL * translation that's fallen into a * terminally fatal state. */ if ((have_sense) && (asc != 0x25) && (asc != 0x44) && (error_code == SSD_CURRENT_ERROR || error_code == SSD_DESC_CURRENT_ERROR)) { const char *sense_key_desc; const char *asc_desc; dasetgeom(periph, 512, -1, NULL, 0); scsi_sense_desc(sense_key, asc, ascq, &cgd.inq_data, &sense_key_desc, &asc_desc); snprintf(announce_buf, DA_ANNOUNCETMP_SZ, "Attempt to query device " "size failed: %s, %s", sense_key_desc, asc_desc); } else { if (have_sense) scsi_sense_print(&done_ccb->csio); else { xpt_print(periph->path, "got CAM status %#x\n", done_ccb->ccb_h.status); } xpt_print(periph->path, "fatal error, " "failed to attach to device\n"); announce_buf = NULL; /* * Free up resources. */ cam_periph_invalidate(periph); } } } free(csio->data_ptr, M_SCSIDA); if (announce_buf != NULL && ((softc->flags & DA_FLAG_ANNOUNCED) == 0)) { struct sbuf sb; sbuf_new(&sb, softc->announcebuf, DA_ANNOUNCE_SZ, SBUF_FIXEDLEN); xpt_announce_periph_sbuf(periph, &sb, announce_buf); xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, DA_Q_BIT_STRING); sbuf_finish(&sb); sbuf_putbuf(&sb); /* * Create our sysctl variables, now that we know * we have successfully attached. */ /* increase the refcount */ if (da_periph_acquire(periph, DA_REF_SYSCTL) == 0) { taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); } else { /* XXX This message is useless! */ xpt_print(periph->path, "fatal error, " "could not acquire reference count\n"); } } /* We already probed the device. */ if (softc->flags & DA_FLAG_PROBED) { daprobedone(periph, done_ccb); return; } /* Ensure re-probe doesn't see old delete. */ softc->delete_available = 0; dadeleteflag(softc, DA_DELETE_ZERO, 1); if (lbp && (softc->quirks & DA_Q_NO_UNMAP) == 0) { /* * Based on older SBC-3 spec revisions * any of the UNMAP methods "may" be * available via LBP given this flag so * we flag all of them as available and * then remove those which further * probes confirm aren't available * later. * * We could also check readcap(16) p_type * flag to exclude one or more invalid * write same (X) types here */ dadeleteflag(softc, DA_DELETE_WS16, 1); dadeleteflag(softc, DA_DELETE_WS10, 1); dadeleteflag(softc, DA_DELETE_UNMAP, 1); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_LBP; xpt_schedule(periph, priority); return; } xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_BDC; xpt_schedule(periph, priority); return; } static void dadone_probelbp(struct cam_periph *periph, union ccb *done_ccb) { struct scsi_vpd_logical_block_prov *lbp; struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probelbp\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; lbp = (struct scsi_vpd_logical_block_prov *)csio->data_ptr; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { /* * T10/1799-D Revision 31 states at least one of these * must be supported but we don't currently enforce this. */ dadeleteflag(softc, DA_DELETE_WS16, (lbp->flags & SVPD_LBP_WS16)); dadeleteflag(softc, DA_DELETE_WS10, (lbp->flags & SVPD_LBP_WS10)); dadeleteflag(softc, DA_DELETE_UNMAP, (lbp->flags & SVPD_LBP_UNMAP)); } else { int error; error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } /* * Failure indicates we don't support any SBC-3 * delete methods with UNMAP */ } } free(lbp, M_SCSIDA); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_BLK_LIMITS; xpt_schedule(periph, priority); return; } static void dadone_probeblklimits(struct cam_periph *periph, union ccb *done_ccb) { struct scsi_vpd_block_limits *block_limits; struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeblklimits\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; block_limits = (struct scsi_vpd_block_limits *)csio->data_ptr; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { uint32_t max_txfer_len = scsi_4btoul( block_limits->max_txfer_len); uint32_t max_unmap_lba_cnt = scsi_4btoul( block_limits->max_unmap_lba_cnt); uint32_t max_unmap_blk_cnt = scsi_4btoul( block_limits->max_unmap_blk_cnt); uint32_t unmap_gran = scsi_4btoul( block_limits->opt_unmap_grain); uint32_t unmap_gran_align = scsi_4btoul( block_limits->unmap_grain_align); uint64_t ws_max_blks = scsi_8btou64( block_limits->max_write_same_length); if (max_txfer_len != 0) { softc->disk->d_maxsize = MIN(softc->maxio, (off_t)max_txfer_len * softc->params.secsize); } /* * We should already support UNMAP but we check lba * and block count to be sure */ if (max_unmap_lba_cnt != 0x00L && max_unmap_blk_cnt != 0x00L) { softc->unmap_max_lba = max_unmap_lba_cnt; softc->unmap_max_ranges = min(max_unmap_blk_cnt, UNMAP_MAX_RANGES); if (unmap_gran > 1) { softc->unmap_gran = unmap_gran; if (unmap_gran_align & 0x80000000) { softc->unmap_gran_align = unmap_gran_align & 0x7fffffff; } } } else { /* * Unexpected UNMAP limits which means the * device doesn't actually support UNMAP */ dadeleteflag(softc, DA_DELETE_UNMAP, 0); } if (ws_max_blks != 0x00L) softc->ws_max_blks = ws_max_blks; } else { int error; error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } /* * Failure here doesn't mean UNMAP is not * supported as this is an optional page. */ softc->unmap_max_lba = 1; softc->unmap_max_ranges = 1; } } free(block_limits, M_SCSIDA); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_BDC; xpt_schedule(periph, priority); return; } static void dadone_probebdc(struct cam_periph *periph, union ccb *done_ccb) { struct scsi_vpd_block_device_characteristics *bdc; struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probebdc\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; bdc = (struct scsi_vpd_block_device_characteristics *)csio->data_ptr; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { uint32_t valid_len; /* * Disable queue sorting for non-rotational media * by default. */ u_int16_t old_rate = softc->disk->d_rotation_rate; valid_len = csio->dxfer_len - csio->resid; if (SBDC_IS_PRESENT(bdc, valid_len, medium_rotation_rate)) { softc->disk->d_rotation_rate = scsi_2btoul(bdc->medium_rotation_rate); if (softc->disk->d_rotation_rate == SVPD_BDC_RATE_NON_ROTATING) { cam_iosched_set_sort_queue( softc->cam_iosched, 0); softc->rotating = 0; } if (softc->disk->d_rotation_rate != old_rate) { disk_attr_changed(softc->disk, "GEOM::rotation_rate", M_NOWAIT); } } if ((SBDC_IS_PRESENT(bdc, valid_len, flags)) && (softc->zone_mode == DA_ZONE_NONE)) { int ata_proto; if (scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) ata_proto = 1; else ata_proto = 0; /* * The Zoned field will only be set for * Drive Managed and Host Aware drives. If * they are Host Managed, the device type * in the standard INQUIRY data should be * set to T_ZBC_HM (0x14). */ if ((bdc->flags & SVPD_ZBC_MASK) == SVPD_HAW_ZBC) { softc->zone_mode = DA_ZONE_HOST_AWARE; softc->zone_interface = (ata_proto) ? DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI; } else if ((bdc->flags & SVPD_ZBC_MASK) == SVPD_DM_ZBC) { softc->zone_mode =DA_ZONE_DRIVE_MANAGED; softc->zone_interface = (ata_proto) ? DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI; } else if ((bdc->flags & SVPD_ZBC_MASK) != SVPD_ZBC_NR) { xpt_print(periph->path, "Unknown zoned " "type %#x", bdc->flags & SVPD_ZBC_MASK); } } } else { int error; error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(bdc, M_SCSIDA); xpt_release_ccb(done_ccb); softc->state = DA_STATE_PROBE_ATA; xpt_schedule(periph, priority); return; } static void dadone_probeata(struct cam_periph *periph, union ccb *done_ccb) { struct ata_params *ata_params; struct ccb_scsiio *csio; struct da_softc *softc; u_int32_t priority; int continue_probe; int error, i; int16_t *ptr; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeata\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; ata_params = (struct ata_params *)csio->data_ptr; ptr = (uint16_t *)ata_params; continue_probe = 0; error = 0; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { uint16_t old_rate; for (i = 0; i < sizeof(*ata_params) / 2; i++) ptr[i] = le16toh(ptr[i]); if (ata_params->support_dsm & ATA_SUPPORT_DSM_TRIM && (softc->quirks & DA_Q_NO_UNMAP) == 0) { dadeleteflag(softc, DA_DELETE_ATA_TRIM, 1); if (ata_params->max_dsm_blocks != 0) softc->trim_max_ranges = min( softc->trim_max_ranges, ata_params->max_dsm_blocks * ATA_DSM_BLK_RANGES); } /* * Disable queue sorting for non-rotational media * by default. */ old_rate = softc->disk->d_rotation_rate; softc->disk->d_rotation_rate = ata_params->media_rotation_rate; if (softc->disk->d_rotation_rate == ATA_RATE_NON_ROTATING) { cam_iosched_set_sort_queue(softc->cam_iosched, 0); softc->rotating = 0; } if (softc->disk->d_rotation_rate != old_rate) { disk_attr_changed(softc->disk, "GEOM::rotation_rate", M_NOWAIT); } cam_periph_assert(periph, MA_OWNED); if (ata_params->capabilities1 & ATA_SUPPORT_DMA) softc->flags |= DA_FLAG_CAN_ATA_DMA; if (ata_params->support.extension & ATA_SUPPORT_GENLOG) softc->flags |= DA_FLAG_CAN_ATA_LOG; /* * At this point, if we have a SATA host aware drive, * we communicate via ATA passthrough unless the * SAT layer supports ZBC -> ZAC translation. In * that case, * * XXX KDM figure out how to detect a host managed * SATA drive. */ if (softc->zone_mode == DA_ZONE_NONE) { /* * Note that we don't override the zone * mode or interface if it has already been * set. This is because it has either been * set as a quirk, or when we probed the * SCSI Block Device Characteristics page, * the zoned field was set. The latter * means that the SAT layer supports ZBC to * ZAC translation, and we would prefer to * use that if it is available. */ if ((ata_params->support3 & ATA_SUPPORT_ZONE_MASK) == ATA_SUPPORT_ZONE_HOST_AWARE) { softc->zone_mode = DA_ZONE_HOST_AWARE; softc->zone_interface = DA_ZONE_IF_ATA_PASS; } else if ((ata_params->support3 & ATA_SUPPORT_ZONE_MASK) == ATA_SUPPORT_ZONE_DEV_MANAGED) { softc->zone_mode =DA_ZONE_DRIVE_MANAGED; softc->zone_interface = DA_ZONE_IF_ATA_PASS; } } } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(ata_params, M_SCSIDA); if ((softc->zone_mode == DA_ZONE_HOST_AWARE) || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) { /* * If the ATA IDENTIFY failed, we could be talking * to a SCSI drive, although that seems unlikely, * since the drive did report that it supported the * ATA Information VPD page. If the ATA IDENTIFY * succeeded, and the SAT layer doesn't support * ZBC -> ZAC translation, continue on to get the * directory of ATA logs, and complete the rest of * the ZAC probe. If the SAT layer does support * ZBC -> ZAC translation, we want to use that, * and we'll probe the SCSI Zoned Block Device * Characteristics VPD page next. */ if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_LOG) && (softc->zone_interface == DA_ZONE_IF_ATA_PASS)) softc->state = DA_STATE_PROBE_ATA_LOGDIR; else softc->state = DA_STATE_PROBE_ZONE; continue_probe = 1; } if (continue_probe != 0) { xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); return; } else daprobedone(periph, done_ccb); return; } static void dadone_probeatalogdir(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; int error; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatalogdir\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { error = 0; softc->valid_logdir_len = 0; bzero(&softc->ata_logdir, sizeof(softc->ata_logdir)); softc->valid_logdir_len = csio->dxfer_len - csio->resid; if (softc->valid_logdir_len > 0) bcopy(csio->data_ptr, &softc->ata_logdir, min(softc->valid_logdir_len, sizeof(softc->ata_logdir))); /* * Figure out whether the Identify Device log is * supported. The General Purpose log directory * has a header, and lists the number of pages * available for each GP log identified by the * offset into the list. */ if ((softc->valid_logdir_len >= ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t))) && (le16dec(softc->ata_logdir.header) == ATA_GP_LOG_DIR_VERSION) && (le16dec(&softc->ata_logdir.num_pages[ (ATA_IDENTIFY_DATA_LOG * sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){ softc->flags |= DA_FLAG_CAN_ATA_IDLOG; } else { softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG; } } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { /* * If we can't get the ATA log directory, * then ATA logs are effectively not * supported even if the bit is set in the * identify data. */ softc->flags &= ~(DA_FLAG_CAN_ATA_LOG | DA_FLAG_CAN_ATA_IDLOG); if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_IDLOG)) { softc->state = DA_STATE_PROBE_ATA_IDDIR; xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); return; } daprobedone(periph, done_ccb); return; } static void dadone_probeataiddir(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; int error; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeataiddir\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { off_t entries_offset, max_entries; error = 0; softc->valid_iddir_len = 0; bzero(&softc->ata_iddir, sizeof(softc->ata_iddir)); softc->flags &= ~(DA_FLAG_CAN_ATA_SUPCAP | DA_FLAG_CAN_ATA_ZONE); softc->valid_iddir_len = csio->dxfer_len - csio->resid; if (softc->valid_iddir_len > 0) bcopy(csio->data_ptr, &softc->ata_iddir, min(softc->valid_iddir_len, sizeof(softc->ata_iddir))); entries_offset = __offsetof(struct ata_identify_log_pages,entries); max_entries = softc->valid_iddir_len - entries_offset; if ((softc->valid_iddir_len > (entries_offset + 1)) && (le64dec(softc->ata_iddir.header) == ATA_IDLOG_REVISION) && (softc->ata_iddir.entry_count > 0)) { int num_entries, i; num_entries = softc->ata_iddir.entry_count; num_entries = min(num_entries, softc->valid_iddir_len - entries_offset); for (i = 0; i < num_entries && i < max_entries; i++) { if (softc->ata_iddir.entries[i] == ATA_IDL_SUP_CAP) softc->flags |= DA_FLAG_CAN_ATA_SUPCAP; else if (softc->ata_iddir.entries[i] == ATA_IDL_ZDI) softc->flags |= DA_FLAG_CAN_ATA_ZONE; if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP) && (softc->flags & DA_FLAG_CAN_ATA_ZONE)) break; } } } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { /* * If we can't get the ATA Identify Data log * directory, then it effectively isn't * supported even if the ATA Log directory * a non-zero number of pages present for * this log. */ softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG; if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_SUPCAP)) { softc->state = DA_STATE_PROBE_ATA_SUP; xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); return; } daprobedone(periph, done_ccb); return; } static void dadone_probeatasup(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; u_int32_t priority; int error; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatasup\n")); softc = (struct da_softc *)periph->softc; priority = done_ccb->ccb_h.pinfo.priority; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { uint32_t valid_len; size_t needed_size; struct ata_identify_log_sup_cap *sup_cap; error = 0; sup_cap = (struct ata_identify_log_sup_cap *)csio->data_ptr; valid_len = csio->dxfer_len - csio->resid; needed_size = __offsetof(struct ata_identify_log_sup_cap, sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap); if (valid_len >= needed_size) { uint64_t zoned, zac_cap; zoned = le64dec(sup_cap->zoned_cap); if (zoned & ATA_ZONED_VALID) { /* * This should have already been * set, because this is also in the * ATA identify data. */ if ((zoned & ATA_ZONED_MASK) == ATA_SUPPORT_ZONE_HOST_AWARE) softc->zone_mode = DA_ZONE_HOST_AWARE; else if ((zoned & ATA_ZONED_MASK) == ATA_SUPPORT_ZONE_DEV_MANAGED) softc->zone_mode = DA_ZONE_DRIVE_MANAGED; } zac_cap = le64dec(sup_cap->sup_zac_cap); if (zac_cap & ATA_SUP_ZAC_CAP_VALID) { if (zac_cap & ATA_REPORT_ZONES_SUP) softc->zone_flags |= DA_ZONE_FLAG_RZ_SUP; if (zac_cap & ATA_ND_OPEN_ZONE_SUP) softc->zone_flags |= DA_ZONE_FLAG_OPEN_SUP; if (zac_cap & ATA_ND_CLOSE_ZONE_SUP) softc->zone_flags |= DA_ZONE_FLAG_CLOSE_SUP; if (zac_cap & ATA_ND_FINISH_ZONE_SUP) softc->zone_flags |= DA_ZONE_FLAG_FINISH_SUP; if (zac_cap & ATA_ND_RWP_SUP) softc->zone_flags |= DA_ZONE_FLAG_RWP_SUP; } else { /* * This field was introduced in * ACS-4, r08 on April 28th, 2015. * If the drive firmware was written * to an earlier spec, it won't have * the field. So, assume all * commands are supported. */ softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK; } } } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { /* * If we can't get the ATA Identify Data * Supported Capabilities page, clear the * flag... */ softc->flags &= ~DA_FLAG_CAN_ATA_SUPCAP; /* * And clear zone capabilities. */ softc->zone_flags &= ~DA_ZONE_FLAG_SUP_MASK; if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_ZONE)) { softc->state = DA_STATE_PROBE_ATA_ZONE; xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); return; } daprobedone(periph, done_ccb); return; } static void dadone_probeatazone(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; int error; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatazone\n")); softc = (struct da_softc *)periph->softc; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { struct ata_zoned_info_log *zi_log; uint32_t valid_len; size_t needed_size; zi_log = (struct ata_zoned_info_log *)csio->data_ptr; valid_len = csio->dxfer_len - csio->resid; needed_size = __offsetof(struct ata_zoned_info_log, version_info) + 1 + sizeof(zi_log->version_info); if (valid_len >= needed_size) { uint64_t tmpvar; tmpvar = le64dec(zi_log->zoned_cap); if (tmpvar & ATA_ZDI_CAP_VALID) { if (tmpvar & ATA_ZDI_CAP_URSWRZ) softc->zone_flags |= DA_ZONE_FLAG_URSWRZ; else softc->zone_flags &= ~DA_ZONE_FLAG_URSWRZ; } tmpvar = le64dec(zi_log->optimal_seq_zones); if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) { softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET; softc->optimal_seq_zones = (tmpvar & ATA_ZDI_OPT_SEQ_MASK); } else { softc->zone_flags &= ~DA_ZONE_FLAG_OPT_SEQ_SET; softc->optimal_seq_zones = 0; } tmpvar =le64dec(zi_log->optimal_nonseq_zones); if (tmpvar & ATA_ZDI_OPT_NS_VALID) { softc->zone_flags |= DA_ZONE_FLAG_OPT_NONSEQ_SET; softc->optimal_nonseq_zones = (tmpvar & ATA_ZDI_OPT_NS_MASK); } else { softc->zone_flags &= ~DA_ZONE_FLAG_OPT_NONSEQ_SET; softc->optimal_nonseq_zones = 0; } tmpvar = le64dec(zi_log->max_seq_req_zones); if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) { softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET; softc->max_seq_zones = (tmpvar & ATA_ZDI_MAX_SEQ_MASK); } else { softc->zone_flags &= ~DA_ZONE_FLAG_MAX_SEQ_SET; softc->max_seq_zones = 0; } } } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { softc->flags &= ~DA_FLAG_CAN_ATA_ZONE; softc->flags &= ~DA_ZONE_FLAG_SET_MASK; if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); daprobedone(periph, done_ccb); return; } static void dadone_probezone(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; int error; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probezone\n")); softc = (struct da_softc *)periph->softc; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { uint32_t valid_len; size_t needed_len; struct scsi_vpd_zoned_bdc *zoned_bdc; error = 0; zoned_bdc = (struct scsi_vpd_zoned_bdc *)csio->data_ptr; valid_len = csio->dxfer_len - csio->resid; needed_len = __offsetof(struct scsi_vpd_zoned_bdc, max_seq_req_zones) + 1 + sizeof(zoned_bdc->max_seq_req_zones); if ((valid_len >= needed_len) && (scsi_2btoul(zoned_bdc->page_length) >= SVPD_ZBDC_PL)) { if (zoned_bdc->flags & SVPD_ZBDC_URSWRZ) softc->zone_flags |= DA_ZONE_FLAG_URSWRZ; else softc->zone_flags &= ~DA_ZONE_FLAG_URSWRZ; softc->optimal_seq_zones = scsi_4btoul(zoned_bdc->optimal_seq_zones); softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET; softc->optimal_nonseq_zones = scsi_4btoul( zoned_bdc->optimal_nonseq_zones); softc->zone_flags |= DA_ZONE_FLAG_OPT_NONSEQ_SET; softc->max_seq_zones = scsi_4btoul(zoned_bdc->max_seq_req_zones); softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET; } /* * All of the zone commands are mandatory for SCSI * devices. * * XXX KDM this is valid as of September 2015. * Re-check this assumption once the SAT spec is * updated to support SCSI ZBC to ATA ZAC mapping. * Since ATA allows zone commands to be reported * as supported or not, this may not necessarily * be true for an ATA device behind a SAT (SCSI to * ATA Translation) layer. */ softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK; } else { error = daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA|SF_NO_PRINT); if (error == ERESTART) return; else if (error != 0) { if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge this device's queue */ cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } } } free(csio->data_ptr, M_SCSIDA); daprobedone(periph, done_ccb); return; } static void dadone_tur(struct cam_periph *periph, union ccb *done_ccb) { struct da_softc *softc; struct ccb_scsiio *csio; CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_tur\n")); softc = (struct da_softc *)periph->softc; csio = &done_ccb->csio; cam_periph_assert(periph, MA_OWNED); if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { if (daerror(done_ccb, CAM_RETRY_SELTO, SF_RETRY_UA | SF_NO_RECOVERY | SF_NO_PRINT) == ERESTART) return; /* Will complete again, keep reference */ if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) cam_release_devq(done_ccb->ccb_h.path, /*relsim_flags*/0, /*reduction*/0, /*timeout*/0, /*getcount_only*/0); } xpt_release_ccb(done_ccb); softc->flags &= ~DA_FLAG_TUR_PENDING; da_periph_release_locked(periph, DA_REF_TUR); return; } static void dareprobe(struct cam_periph *periph) { struct da_softc *softc; int status; softc = (struct da_softc *)periph->softc; /* Probe in progress; don't interfere. */ if (softc->state != DA_STATE_NORMAL) return; status = da_periph_acquire(periph, DA_REF_REPROBE); KASSERT(status == 0, ("dareprobe: cam_periph_acquire failed")); softc->state = DA_STATE_PROBE_WP; xpt_schedule(periph, CAM_PRIORITY_DEV); } static int daerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags) { struct da_softc *softc; struct cam_periph *periph; int error, error_code, sense_key, asc, ascq; #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) if (ccb->csio.bio != NULL) biotrack(ccb->csio.bio, __func__); #endif periph = xpt_path_periph(ccb->ccb_h.path); softc = (struct da_softc *)periph->softc; cam_periph_assert(periph, MA_OWNED); /* * Automatically detect devices that do not support * READ(6)/WRITE(6) and upgrade to using 10 byte cdbs. */ error = 0; if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INVALID) { error = cmd6workaround(ccb); } else if (scsi_extract_sense_ccb(ccb, &error_code, &sense_key, &asc, &ascq)) { if (sense_key == SSD_KEY_ILLEGAL_REQUEST) error = cmd6workaround(ccb); /* * If the target replied with CAPACITY DATA HAS CHANGED UA, * query the capacity and notify upper layers. */ else if (sense_key == SSD_KEY_UNIT_ATTENTION && asc == 0x2A && ascq == 0x09) { xpt_print(periph->path, "Capacity data has changed\n"); softc->flags &= ~DA_FLAG_PROBED; dareprobe(periph); sense_flags |= SF_NO_PRINT; } else if (sense_key == SSD_KEY_UNIT_ATTENTION && asc == 0x28 && ascq == 0x00) { softc->flags &= ~DA_FLAG_PROBED; disk_media_changed(softc->disk, M_NOWAIT); } else if (sense_key == SSD_KEY_UNIT_ATTENTION && asc == 0x3F && ascq == 0x03) { xpt_print(periph->path, "INQUIRY data has changed\n"); softc->flags &= ~DA_FLAG_PROBED; dareprobe(periph); sense_flags |= SF_NO_PRINT; } else if (sense_key == SSD_KEY_NOT_READY && asc == 0x3a && (softc->flags & DA_FLAG_PACK_INVALID) == 0) { softc->flags |= DA_FLAG_PACK_INVALID; disk_media_gone(softc->disk, M_NOWAIT); } } if (error == ERESTART) return (ERESTART); #ifdef CAM_IO_STATS switch (ccb->ccb_h.status & CAM_STATUS_MASK) { case CAM_CMD_TIMEOUT: softc->timeouts++; break; case CAM_REQ_ABORTED: case CAM_REQ_CMP_ERR: case CAM_REQ_TERMIO: case CAM_UNREC_HBA_ERROR: case CAM_DATA_RUN_ERR: softc->errors++; break; default: break; } #endif /* * XXX * Until we have a better way of doing pack validation, * don't treat UAs as errors. */ sense_flags |= SF_RETRY_UA; if (softc->quirks & DA_Q_RETRY_BUSY) sense_flags |= SF_RETRY_BUSY; return(cam_periph_error(ccb, cam_flags, sense_flags)); } static void damediapoll(void *arg) { struct cam_periph *periph = arg; struct da_softc *softc = periph->softc; if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) && (softc->flags & DA_FLAG_TUR_PENDING) == 0 && LIST_EMPTY(&softc->pending_ccbs)) { if (da_periph_acquire(periph, DA_REF_TUR) == 0) { cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); daschedule(periph); } } /* Queue us up again */ if (da_poll_period != 0) callout_schedule(&softc->mediapoll_c, da_poll_period * hz); } static void daprevent(struct cam_periph *periph, int action) { struct da_softc *softc; union ccb *ccb; int error; cam_periph_assert(periph, MA_OWNED); softc = (struct da_softc *)periph->softc; if (((action == PR_ALLOW) && (softc->flags & DA_FLAG_PACK_LOCKED) == 0) || ((action == PR_PREVENT) && (softc->flags & DA_FLAG_PACK_LOCKED) != 0)) { return; } ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); scsi_prevent(&ccb->csio, /*retries*/1, /*cbcfp*/NULL, MSG_SIMPLE_Q_TAG, action, SSD_FULL_SIZE, 5000); error = cam_periph_runccb(ccb, daerror, CAM_RETRY_SELTO, SF_RETRY_UA | SF_NO_PRINT, softc->disk->d_devstat); if (error == 0) { if (action == PR_ALLOW) softc->flags &= ~DA_FLAG_PACK_LOCKED; else softc->flags |= DA_FLAG_PACK_LOCKED; } xpt_release_ccb(ccb); } static void dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector, struct scsi_read_capacity_data_long *rcaplong, size_t rcap_len) { struct ccb_calc_geometry ccg; struct da_softc *softc; struct disk_params *dp; u_int lbppbe, lalba; int error; softc = (struct da_softc *)periph->softc; dp = &softc->params; dp->secsize = block_len; dp->sectors = maxsector + 1; if (rcaplong != NULL) { lbppbe = rcaplong->prot_lbppbe & SRC16_LBPPBE; lalba = scsi_2btoul(rcaplong->lalba_lbp); lalba &= SRC16_LALBA_A; } else { lbppbe = 0; lalba = 0; } if (lbppbe > 0) { dp->stripesize = block_len << lbppbe; dp->stripeoffset = (dp->stripesize - block_len * lalba) % dp->stripesize; } else if (softc->quirks & DA_Q_4K) { dp->stripesize = 4096; dp->stripeoffset = 0; } else if (softc->unmap_gran != 0) { dp->stripesize = block_len * softc->unmap_gran; dp->stripeoffset = (dp->stripesize - block_len * softc->unmap_gran_align) % dp->stripesize; } else { dp->stripesize = 0; dp->stripeoffset = 0; } /* * Have the controller provide us with a geometry * for this disk. The only time the geometry * matters is when we boot and the controller * is the only one knowledgeable enough to come * up with something that will make this a bootable * device. */ xpt_setup_ccb(&ccg.ccb_h, periph->path, CAM_PRIORITY_NORMAL); ccg.ccb_h.func_code = XPT_CALC_GEOMETRY; ccg.block_size = dp->secsize; ccg.volume_size = dp->sectors; ccg.heads = 0; ccg.secs_per_track = 0; ccg.cylinders = 0; xpt_action((union ccb*)&ccg); if ((ccg.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { /* * We don't know what went wrong here- but just pick * a geometry so we don't have nasty things like divide * by zero. */ dp->heads = 255; dp->secs_per_track = 255; dp->cylinders = dp->sectors / (255 * 255); if (dp->cylinders == 0) { dp->cylinders = 1; } } else { dp->heads = ccg.heads; dp->secs_per_track = ccg.secs_per_track; dp->cylinders = ccg.cylinders; } /* * If the user supplied a read capacity buffer, and if it is * different than the previous buffer, update the data in the EDT. * If it's the same, we don't bother. This avoids sending an * update every time someone opens this device. */ if ((rcaplong != NULL) && (bcmp(rcaplong, &softc->rcaplong, min(sizeof(softc->rcaplong), rcap_len)) != 0)) { struct ccb_dev_advinfo cdai; xpt_setup_ccb(&cdai.ccb_h, periph->path, CAM_PRIORITY_NORMAL); cdai.ccb_h.func_code = XPT_DEV_ADVINFO; cdai.buftype = CDAI_TYPE_RCAPLONG; cdai.flags = CDAI_FLAG_STORE; cdai.bufsiz = rcap_len; cdai.buf = (uint8_t *)rcaplong; xpt_action((union ccb *)&cdai); if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0) cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE); if (cdai.ccb_h.status != CAM_REQ_CMP) { xpt_print(periph->path, "%s: failed to set read " "capacity advinfo\n", __func__); /* Use cam_error_print() to decode the status */ cam_error_print((union ccb *)&cdai, CAM_ESF_CAM_STATUS, CAM_EPF_ALL); } else { bcopy(rcaplong, &softc->rcaplong, min(sizeof(softc->rcaplong), rcap_len)); } } softc->disk->d_sectorsize = softc->params.secsize; softc->disk->d_mediasize = softc->params.secsize * (off_t)softc->params.sectors; softc->disk->d_stripesize = softc->params.stripesize; softc->disk->d_stripeoffset = softc->params.stripeoffset; /* XXX: these are not actually "firmware" values, so they may be wrong */ softc->disk->d_fwsectors = softc->params.secs_per_track; softc->disk->d_fwheads = softc->params.heads; softc->disk->d_devstat->block_size = softc->params.secsize; softc->disk->d_devstat->flags &= ~DEVSTAT_BS_UNAVAILABLE; error = disk_resize(softc->disk, M_NOWAIT); if (error != 0) xpt_print(periph->path, "disk_resize(9) failed, error = %d\n", error); } static void dasendorderedtag(void *arg) { struct cam_periph *periph = arg; struct da_softc *softc = periph->softc; cam_periph_assert(periph, MA_OWNED); if (da_send_ordered) { if (!LIST_EMPTY(&softc->pending_ccbs)) { if ((softc->flags & DA_FLAG_WAS_OTAG) == 0) softc->flags |= DA_FLAG_NEED_OTAG; softc->flags &= ~DA_FLAG_WAS_OTAG; } } /* Queue us up again */ callout_reset(&softc->sendordered_c, (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, dasendorderedtag, periph); } /* * Step through all DA peripheral drivers, and if the device is still open, * sync the disk cache to physical media. */ static void dashutdown(void * arg, int howto) { struct cam_periph *periph; struct da_softc *softc; union ccb *ccb; int error; CAM_PERIPH_FOREACH(periph, &dadriver) { softc = (struct da_softc *)periph->softc; if (SCHEDULER_STOPPED()) { /* If we paniced with the lock held, do not recurse. */ if (!cam_periph_owned(periph) && (softc->flags & DA_FLAG_OPEN)) { dadump(softc->disk, NULL, 0, 0, 0); } continue; } cam_periph_lock(periph); /* * We only sync the cache if the drive is still open, and * if the drive is capable of it.. */ if (((softc->flags & DA_FLAG_OPEN) == 0) || (softc->quirks & DA_Q_NO_SYNC_CACHE)) { cam_periph_unlock(periph); continue; } ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); scsi_synchronize_cache(&ccb->csio, /*retries*/0, /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG, /*begin_lba*/0, /* whole disk */ /*lb_count*/0, SSD_FULL_SIZE, 60 * 60 * 1000); error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY | SF_QUIET_IR, softc->disk->d_devstat); if (error != 0) xpt_print(periph->path, "Synchronize cache failed\n"); xpt_release_ccb(ccb); cam_periph_unlock(periph); } } #else /* !_KERNEL */ /* * XXX These are only left out of the kernel build to silence warnings. If, * for some reason these functions are used in the kernel, the ifdefs should * be moved so they are included both in the kernel and userland. */ void scsi_format_unit(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t byte2, u_int16_t ileave, u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_format_unit *scsi_cmd; scsi_cmd = (struct scsi_format_unit *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = FORMAT_UNIT; scsi_cmd->byte2 = byte2; scsi_ulto2b(ileave, scsi_cmd->interleave); cam_fill_csio(csio, retries, cbfcnp, /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_read_defects(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint8_t list_format, uint32_t addr_desc_index, uint8_t *data_ptr, uint32_t dxfer_len, int minimum_cmd_size, uint8_t sense_len, uint32_t timeout) { uint8_t cdb_len; /* * These conditions allow using the 10 byte command. Otherwise we * need to use the 12 byte command. */ if ((minimum_cmd_size <= 10) && (addr_desc_index == 0) && (dxfer_len <= SRDD10_MAX_LENGTH)) { struct scsi_read_defect_data_10 *cdb10; cdb10 = (struct scsi_read_defect_data_10 *) &csio->cdb_io.cdb_bytes; cdb_len = sizeof(*cdb10); bzero(cdb10, cdb_len); cdb10->opcode = READ_DEFECT_DATA_10; cdb10->format = list_format; scsi_ulto2b(dxfer_len, cdb10->alloc_length); } else { struct scsi_read_defect_data_12 *cdb12; cdb12 = (struct scsi_read_defect_data_12 *) &csio->cdb_io.cdb_bytes; cdb_len = sizeof(*cdb12); bzero(cdb12, cdb_len); cdb12->opcode = READ_DEFECT_DATA_12; cdb12->format = list_format; scsi_ulto4b(dxfer_len, cdb12->alloc_length); scsi_ulto4b(addr_desc_index, cdb12->address_descriptor_index); } cam_fill_csio(csio, retries, cbfcnp, /*flags*/ CAM_DIR_IN, tag_action, data_ptr, dxfer_len, sense_len, cdb_len, timeout); } void scsi_sanitize(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t byte2, u_int16_t control, u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_sanitize *scsi_cmd; scsi_cmd = (struct scsi_sanitize *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = SANITIZE; scsi_cmd->byte2 = byte2; scsi_cmd->control = control; scsi_ulto2b(dxfer_len, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } #endif /* _KERNEL */ void scsi_zbc_out(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint8_t service_action, uint64_t zone_id, uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len, uint8_t sense_len, uint32_t timeout) { struct scsi_zbc_out *scsi_cmd; scsi_cmd = (struct scsi_zbc_out *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = ZBC_OUT; scsi_cmd->service_action = service_action; scsi_u64to8b(zone_id, scsi_cmd->zone_id); scsi_cmd->zone_flags = zone_flags; cam_fill_csio(csio, retries, cbfcnp, /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_zbc_in(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint8_t service_action, uint64_t zone_start_lba, uint8_t zone_options, uint8_t *data_ptr, uint32_t dxfer_len, uint8_t sense_len, uint32_t timeout) { struct scsi_zbc_in *scsi_cmd; scsi_cmd = (struct scsi_zbc_in *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = ZBC_IN; scsi_cmd->service_action = service_action; scsi_ulto4b(dxfer_len, scsi_cmd->length); scsi_u64to8b(zone_start_lba, scsi_cmd->zone_start_lba); scsi_cmd->zone_options = zone_options; cam_fill_csio(csio, retries, cbfcnp, /*flags*/ (dxfer_len > 0) ? CAM_DIR_IN : CAM_DIR_NONE, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } int scsi_ata_zac_mgmt_out(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int use_ncq, uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len, uint8_t *cdb_storage, size_t cdb_storage_len, uint8_t sense_len, uint32_t timeout) { uint8_t command_out, protocol, ata_flags; uint16_t features_out; uint32_t sectors_out, auxiliary; int retval; retval = 0; if (use_ncq == 0) { command_out = ATA_ZAC_MANAGEMENT_OUT; features_out = (zm_action & 0xf) | (zone_flags << 8); ata_flags = AP_FLAG_BYT_BLOK_BLOCKS; if (dxfer_len == 0) { protocol = AP_PROTO_NON_DATA; ata_flags |= AP_FLAG_TLEN_NO_DATA; sectors_out = 0; } else { protocol = AP_PROTO_DMA; ata_flags |= AP_FLAG_TLEN_SECT_CNT | AP_FLAG_TDIR_TO_DEV; sectors_out = ((dxfer_len >> 9) & 0xffff); } auxiliary = 0; } else { ata_flags = AP_FLAG_BYT_BLOK_BLOCKS; if (dxfer_len == 0) { command_out = ATA_NCQ_NON_DATA; features_out = ATA_NCQ_ZAC_MGMT_OUT; /* * We're assuming the SCSI to ATA translation layer * will set the NCQ tag number in the tag field. * That isn't clear from the SAT-4 spec (as of rev 05). */ sectors_out = 0; ata_flags |= AP_FLAG_TLEN_NO_DATA; } else { command_out = ATA_SEND_FPDMA_QUEUED; /* * Note that we're defaulting to normal priority, * and assuming that the SCSI to ATA translation * layer will insert the NCQ tag number in the tag * field. That isn't clear in the SAT-4 spec (as * of rev 05). */ sectors_out = ATA_SFPDMA_ZAC_MGMT_OUT << 8; ata_flags |= AP_FLAG_TLEN_FEAT | AP_FLAG_TDIR_TO_DEV; /* * For SEND FPDMA QUEUED, the transfer length is * encoded in the FEATURE register, and 0 means * that 65536 512 byte blocks are to be tranferred. * In practice, it seems unlikely that we'll see * a transfer that large, and it may confuse the * the SAT layer, because generally that means that * 0 bytes should be transferred. */ if (dxfer_len == (65536 * 512)) { features_out = 0; } else if (dxfer_len <= (65535 * 512)) { features_out = ((dxfer_len >> 9) & 0xffff); } else { /* The transfer is too big. */ retval = 1; goto bailout; } } auxiliary = (zm_action & 0xf) | (zone_flags << 8); protocol = AP_PROTO_FPDMA; } protocol |= AP_EXTEND; retval = scsi_ata_pass(csio, retries, cbfcnp, /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, /*protocol*/ protocol, /*ata_flags*/ ata_flags, /*features*/ features_out, /*sector_count*/ sectors_out, /*lba*/ zone_id, /*command*/ command_out, /*device*/ 0, /*icc*/ 0, /*auxiliary*/ auxiliary, /*control*/ 0, /*data_ptr*/ data_ptr, /*dxfer_len*/ dxfer_len, /*cdb_storage*/ cdb_storage, /*cdb_storage_len*/ cdb_storage_len, /*minimum_cmd_size*/ 0, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ timeout); bailout: return (retval); } int scsi_ata_zac_mgmt_in(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int use_ncq, uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len, uint8_t *cdb_storage, size_t cdb_storage_len, uint8_t sense_len, uint32_t timeout) { uint8_t command_out, protocol; uint16_t features_out, sectors_out; uint32_t auxiliary; int ata_flags; int retval; retval = 0; ata_flags = AP_FLAG_TDIR_FROM_DEV | AP_FLAG_BYT_BLOK_BLOCKS; if (use_ncq == 0) { command_out = ATA_ZAC_MANAGEMENT_IN; /* XXX KDM put a macro here */ features_out = (zm_action & 0xf) | (zone_flags << 8); sectors_out = dxfer_len >> 9; /* XXX KDM macro */ protocol = AP_PROTO_DMA; ata_flags |= AP_FLAG_TLEN_SECT_CNT; auxiliary = 0; } else { ata_flags |= AP_FLAG_TLEN_FEAT; command_out = ATA_RECV_FPDMA_QUEUED; sectors_out = ATA_RFPDMA_ZAC_MGMT_IN << 8; /* * For RECEIVE FPDMA QUEUED, the transfer length is * encoded in the FEATURE register, and 0 means * that 65536 512 byte blocks are to be tranferred. * In practice, it seems unlikely that we'll see * a transfer that large, and it may confuse the * the SAT layer, because generally that means that * 0 bytes should be transferred. */ if (dxfer_len == (65536 * 512)) { features_out = 0; } else if (dxfer_len <= (65535 * 512)) { features_out = ((dxfer_len >> 9) & 0xffff); } else { /* The transfer is too big. */ retval = 1; goto bailout; } auxiliary = (zm_action & 0xf) | (zone_flags << 8), protocol = AP_PROTO_FPDMA; } protocol |= AP_EXTEND; retval = scsi_ata_pass(csio, retries, cbfcnp, /*flags*/ CAM_DIR_IN, tag_action, /*protocol*/ protocol, /*ata_flags*/ ata_flags, /*features*/ features_out, /*sector_count*/ sectors_out, /*lba*/ zone_id, /*command*/ command_out, /*device*/ 0, /*icc*/ 0, /*auxiliary*/ auxiliary, /*control*/ 0, /*data_ptr*/ data_ptr, /*dxfer_len*/ (dxfer_len >> 9) * 512, /* XXX KDM */ /*cdb_storage*/ cdb_storage, /*cdb_storage_len*/ cdb_storage_len, /*minimum_cmd_size*/ 0, /*sense_len*/ SSD_FULL_SIZE, /*timeout*/ timeout); bailout: return (retval); } Index: projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c =================================================================== --- projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c (revision 345026) @@ -1,1055 +1,1055 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2012, 2018 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] */ #include #include #include #include #include #include #include #include /* * ZFS I/O Scheduler * --------------- * * ZFS issues I/O operations to leaf vdevs to satisfy and complete zios. The * I/O scheduler determines when and in what order those operations are * issued. The I/O scheduler divides operations into six I/O classes * prioritized in the following order: sync read, sync write, async read, * async write, scrub/resilver and trim. Each queue defines the minimum and * maximum number of concurrent operations that may be issued to the device. * In addition, the device has an aggregate maximum. Note that the sum of the * per-queue minimums must not exceed the aggregate maximum, and if the * aggregate maximum is equal to or greater than the sum of the per-queue * maximums, the per-queue minimum has no effect. * * For many physical devices, throughput increases with the number of * concurrent operations, but latency typically suffers. Further, physical * devices typically have a limit at which more concurrent operations have no * effect on throughput or can actually cause it to decrease. * * The scheduler selects the next operation to issue by first looking for an * I/O class whose minimum has not been satisfied. Once all are satisfied and * the aggregate maximum has not been hit, the scheduler looks for classes * whose maximum has not been satisfied. Iteration through the I/O classes is * done in the order specified above. No further operations are issued if the * aggregate maximum number of concurrent operations has been hit or if there * are no operations queued for an I/O class that has not hit its maximum. * Every time an I/O is queued or an operation completes, the I/O scheduler * looks for new operations to issue. * * All I/O classes have a fixed maximum number of outstanding operations * except for the async write class. Asynchronous writes represent the data * that is committed to stable storage during the syncing stage for * transaction groups (see txg.c). Transaction groups enter the syncing state * periodically so the number of queued async writes will quickly burst up and * then bleed down to zero. Rather than servicing them as quickly as possible, * the I/O scheduler changes the maximum number of active async write I/Os * according to the amount of dirty data in the pool (see dsl_pool.c). Since * both throughput and latency typically increase with the number of * concurrent operations issued to physical devices, reducing the burstiness * in the number of concurrent operations also stabilizes the response time of * operations from other -- and in particular synchronous -- queues. In broad * strokes, the I/O scheduler will issue more concurrent operations from the * async write queue as there's more dirty data in the pool. * * Async Writes * * The number of concurrent operations issued for the async write I/O class * follows a piece-wise linear function defined by a few adjustable points. * * | o---------| <-- zfs_vdev_async_write_max_active * ^ | /^ | * | | / | | * active | / | | * I/O | / | | * count | / | | * | / | | * |------------o | | <-- zfs_vdev_async_write_min_active * 0|____________^______|_________| * 0% | | 100% of zfs_dirty_data_max * | | * | `-- zfs_vdev_async_write_active_max_dirty_percent * `--------- zfs_vdev_async_write_active_min_dirty_percent * * Until the amount of dirty data exceeds a minimum percentage of the dirty * data allowed in the pool, the I/O scheduler will limit the number of * concurrent operations to the minimum. As that threshold is crossed, the * number of concurrent operations issued increases linearly to the maximum at * the specified maximum percentage of the dirty data allowed in the pool. * * Ideally, the amount of dirty data on a busy pool will stay in the sloped * part of the function between zfs_vdev_async_write_active_min_dirty_percent * and zfs_vdev_async_write_active_max_dirty_percent. If it exceeds the * maximum percentage, this indicates that the rate of incoming data is * greater than the rate that the backend storage can handle. In this case, we * must further throttle incoming writes (see dmu_tx_delay() for details). */ /* * The maximum number of I/Os active to each device. Ideally, this will be >= * the sum of each queue's max_active. It must be at least the sum of each * queue's min_active. */ uint32_t zfs_vdev_max_active = 1000; /* * Per-queue limits on the number of I/Os active to each device. If the * sum of the queue's max_active is < zfs_vdev_max_active, then the * min_active comes into play. We will send min_active from each queue, * and then select from queues in the order defined by zio_priority_t. * * In general, smaller max_active's will lead to lower latency of synchronous * operations. Larger max_active's may lead to higher overall throughput, * depending on underlying storage. * * The ratio of the queues' max_actives determines the balance of performance * between reads, writes, and scrubs. E.g., increasing * zfs_vdev_scrub_max_active will cause the scrub or resilver to complete * more quickly, but reads and writes to have higher latency and lower * throughput. */ uint32_t zfs_vdev_sync_read_min_active = 10; uint32_t zfs_vdev_sync_read_max_active = 10; uint32_t zfs_vdev_sync_write_min_active = 10; uint32_t zfs_vdev_sync_write_max_active = 10; uint32_t zfs_vdev_async_read_min_active = 1; uint32_t zfs_vdev_async_read_max_active = 3; uint32_t zfs_vdev_async_write_min_active = 1; uint32_t zfs_vdev_async_write_max_active = 10; uint32_t zfs_vdev_scrub_min_active = 1; uint32_t zfs_vdev_scrub_max_active = 2; uint32_t zfs_vdev_trim_min_active = 1; /* * TRIM max active is large in comparison to the other values due to the fact * that TRIM IOs are coalesced at the device layer. This value is set such * that a typical SSD can process the queued IOs in a single request. */ uint32_t zfs_vdev_trim_max_active = 64; uint32_t zfs_vdev_removal_min_active = 1; uint32_t zfs_vdev_removal_max_active = 2; uint32_t zfs_vdev_initializing_min_active = 1; uint32_t zfs_vdev_initializing_max_active = 1; /* * When the pool has less than zfs_vdev_async_write_active_min_dirty_percent * dirty data, use zfs_vdev_async_write_min_active. When it has more than * zfs_vdev_async_write_active_max_dirty_percent, use * zfs_vdev_async_write_max_active. The value is linearly interpolated * between min and max. */ int zfs_vdev_async_write_active_min_dirty_percent = 30; int zfs_vdev_async_write_active_max_dirty_percent = 60; /* * To reduce IOPs, we aggregate small adjacent I/Os into one large I/O. * For read I/Os, we also aggregate across small adjacency gaps; for writes * we include spans of optional I/Os to aid aggregation at the disk even when * they aren't able to help us aggregate at this level. */ int zfs_vdev_aggregation_limit = 1 << 20; int zfs_vdev_aggregation_limit_non_rotating = SPA_OLD_MAXBLOCKSIZE; int zfs_vdev_read_gap_limit = 32 << 10; int zfs_vdev_write_gap_limit = 4 << 10; /* * Define the queue depth percentage for each top-level. This percentage is * used in conjunction with zfs_vdev_async_max_active to determine how many * allocations a specific top-level vdev should handle. Once the queue depth * reaches zfs_vdev_queue_depth_pct * zfs_vdev_async_write_max_active / 100 * then allocator will stop allocating blocks on that top-level device. * The default kernel setting is 1000% which will yield 100 allocations per * device. For userland testing, the default setting is 300% which equates * to 30 allocations per device. */ #ifdef _KERNEL int zfs_vdev_queue_depth_pct = 1000; #else int zfs_vdev_queue_depth_pct = 300; #endif /* * When performing allocations for a given metaslab, we want to make sure that * there are enough IOs to aggregate together to improve throughput. We want to * ensure that there are at least 128k worth of IOs that can be aggregated, and * we assume that the average allocation size is 4k, so we need the queue depth * to be 32 per allocator to get good aggregation of sequential writes. */ int zfs_vdev_def_queue_depth = 32; #ifdef __FreeBSD__ #ifdef _KERNEL SYSCTL_DECL(_vfs_zfs_vdev); static int sysctl_zfs_async_write_active_min_dirty_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs_vdev, OID_AUTO, async_write_active_min_dirty_percent, CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), sysctl_zfs_async_write_active_min_dirty_percent, "I", "Percentage of async write dirty data below which " "async_write_min_active is used."); static int sysctl_zfs_async_write_active_max_dirty_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs_vdev, OID_AUTO, async_write_active_max_dirty_percent, CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), sysctl_zfs_async_write_active_max_dirty_percent, "I", "Percentage of async write dirty data above which " "async_write_max_active is used."); SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, max_active, CTLFLAG_RWTUN, &zfs_vdev_max_active, 0, "The maximum number of I/Os of all types active for each device."); #define ZFS_VDEV_QUEUE_KNOB_MIN(name) \ SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, name ## _min_active, CTLFLAG_RWTUN,\ &zfs_vdev_ ## name ## _min_active, 0, \ "Initial number of I/O requests of type " #name \ " active for each device"); #define ZFS_VDEV_QUEUE_KNOB_MAX(name) \ SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, name ## _max_active, CTLFLAG_RWTUN,\ &zfs_vdev_ ## name ## _max_active, 0, \ "Maximum number of I/O requests of type " #name \ " active for each device"); ZFS_VDEV_QUEUE_KNOB_MIN(sync_read); ZFS_VDEV_QUEUE_KNOB_MAX(sync_read); ZFS_VDEV_QUEUE_KNOB_MIN(sync_write); ZFS_VDEV_QUEUE_KNOB_MAX(sync_write); ZFS_VDEV_QUEUE_KNOB_MIN(async_read); ZFS_VDEV_QUEUE_KNOB_MAX(async_read); ZFS_VDEV_QUEUE_KNOB_MIN(async_write); ZFS_VDEV_QUEUE_KNOB_MAX(async_write); ZFS_VDEV_QUEUE_KNOB_MIN(scrub); ZFS_VDEV_QUEUE_KNOB_MAX(scrub); ZFS_VDEV_QUEUE_KNOB_MIN(trim); ZFS_VDEV_QUEUE_KNOB_MAX(trim); ZFS_VDEV_QUEUE_KNOB_MIN(removal); ZFS_VDEV_QUEUE_KNOB_MAX(removal); ZFS_VDEV_QUEUE_KNOB_MIN(initializing); ZFS_VDEV_QUEUE_KNOB_MAX(initializing); #undef ZFS_VDEV_QUEUE_KNOB SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, aggregation_limit, CTLFLAG_RWTUN, &zfs_vdev_aggregation_limit, 0, "I/O requests are aggregated up to this size"); SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, aggregation_limit_non_rotating, CTLFLAG_RWTUN, &zfs_vdev_aggregation_limit_non_rotating, 0, "I/O requests are aggregated up to this size for non-rotating media"); SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, read_gap_limit, CTLFLAG_RWTUN, &zfs_vdev_read_gap_limit, 0, "Acceptable gap between two reads being aggregated"); SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, write_gap_limit, CTLFLAG_RWTUN, &zfs_vdev_write_gap_limit, 0, "Acceptable gap between two writes being aggregated"); SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, queue_depth_pct, CTLFLAG_RWTUN, &zfs_vdev_queue_depth_pct, 0, "Queue depth percentage for each top-level"); SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, def_queue_depth, CTLFLAG_RWTUN, &zfs_vdev_def_queue_depth, 0, "Default queue depth for each allocator"); static int sysctl_zfs_async_write_active_min_dirty_percent(SYSCTL_HANDLER_ARGS) { int val, err; val = zfs_vdev_async_write_active_min_dirty_percent; err = sysctl_handle_int(oidp, &val, 0, req); if (err != 0 || req->newptr == NULL) return (err); if (val < 0 || val > 100 || val >= zfs_vdev_async_write_active_max_dirty_percent) return (EINVAL); zfs_vdev_async_write_active_min_dirty_percent = val; return (0); } static int sysctl_zfs_async_write_active_max_dirty_percent(SYSCTL_HANDLER_ARGS) { int val, err; val = zfs_vdev_async_write_active_max_dirty_percent; err = sysctl_handle_int(oidp, &val, 0, req); if (err != 0 || req->newptr == NULL) return (err); if (val < 0 || val > 100 || val <= zfs_vdev_async_write_active_min_dirty_percent) return (EINVAL); zfs_vdev_async_write_active_max_dirty_percent = val; return (0); } #endif #endif int vdev_queue_offset_compare(const void *x1, const void *x2) { const zio_t *z1 = (const zio_t *)x1; const zio_t *z2 = (const zio_t *)x2; int cmp = AVL_CMP(z1->io_offset, z2->io_offset); if (likely(cmp)) return (cmp); return (AVL_PCMP(z1, z2)); } static inline avl_tree_t * vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p) { return (&vq->vq_class[p].vqc_queued_tree); } static inline avl_tree_t * vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t) { if (t == ZIO_TYPE_READ) return (&vq->vq_read_offset_tree); else if (t == ZIO_TYPE_WRITE) return (&vq->vq_write_offset_tree); else return (NULL); } int vdev_queue_timestamp_compare(const void *x1, const void *x2) { const zio_t *z1 = x1; const zio_t *z2 = x2; if (z1->io_timestamp < z2->io_timestamp) return (-1); if (z1->io_timestamp > z2->io_timestamp) return (1); if (z1->io_offset < z2->io_offset) return (-1); if (z1->io_offset > z2->io_offset) return (1); if (z1 < z2) return (-1); if (z1 > z2) return (1); return (0); } void vdev_queue_init(vdev_t *vd) { vdev_queue_t *vq = &vd->vdev_queue; mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL); vq->vq_vdev = vd; avl_create(&vq->vq_active_tree, vdev_queue_offset_compare, sizeof (zio_t), offsetof(struct zio, io_queue_node)); avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ), vdev_queue_offset_compare, sizeof (zio_t), offsetof(struct zio, io_offset_node)); avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE), vdev_queue_offset_compare, sizeof (zio_t), offsetof(struct zio, io_offset_node)); for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { int (*compfn) (const void *, const void *); /* * The synchronous i/o queues are dispatched in FIFO rather * than LBA order. This provides more consistent latency for * these i/os. */ if (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE) compfn = vdev_queue_timestamp_compare; else compfn = vdev_queue_offset_compare; avl_create(vdev_queue_class_tree(vq, p), compfn, sizeof (zio_t), offsetof(struct zio, io_queue_node)); } vq->vq_lastoffset = 0; } void vdev_queue_fini(vdev_t *vd) { vdev_queue_t *vq = &vd->vdev_queue; for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) avl_destroy(vdev_queue_class_tree(vq, p)); avl_destroy(&vq->vq_active_tree); avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ)); avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE)); mutex_destroy(&vq->vq_lock); } static void vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio) { spa_t *spa = zio->io_spa; avl_tree_t *qtt; ASSERT(MUTEX_HELD(&vq->vq_lock)); ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio); qtt = vdev_queue_type_tree(vq, zio->io_type); if (qtt) avl_add(qtt, zio); #ifdef illumos mutex_enter(&spa->spa_iokstat_lock); spa->spa_queue_stats[zio->io_priority].spa_queued++; if (spa->spa_iokstat != NULL) kstat_waitq_enter(spa->spa_iokstat->ks_data); mutex_exit(&spa->spa_iokstat_lock); #endif } static void vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio) { spa_t *spa = zio->io_spa; avl_tree_t *qtt; ASSERT(MUTEX_HELD(&vq->vq_lock)); ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio); qtt = vdev_queue_type_tree(vq, zio->io_type); if (qtt) avl_remove(qtt, zio); #ifdef illumos mutex_enter(&spa->spa_iokstat_lock); ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_queued, >, 0); spa->spa_queue_stats[zio->io_priority].spa_queued--; if (spa->spa_iokstat != NULL) kstat_waitq_exit(spa->spa_iokstat->ks_data); mutex_exit(&spa->spa_iokstat_lock); #endif } static void vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio) { spa_t *spa = zio->io_spa; ASSERT(MUTEX_HELD(&vq->vq_lock)); ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); vq->vq_class[zio->io_priority].vqc_active++; avl_add(&vq->vq_active_tree, zio); #ifdef illumos mutex_enter(&spa->spa_iokstat_lock); spa->spa_queue_stats[zio->io_priority].spa_active++; if (spa->spa_iokstat != NULL) kstat_runq_enter(spa->spa_iokstat->ks_data); mutex_exit(&spa->spa_iokstat_lock); #endif } static void vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio) { spa_t *spa = zio->io_spa; ASSERT(MUTEX_HELD(&vq->vq_lock)); ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); vq->vq_class[zio->io_priority].vqc_active--; avl_remove(&vq->vq_active_tree, zio); #ifdef illumos mutex_enter(&spa->spa_iokstat_lock); ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_active, >, 0); spa->spa_queue_stats[zio->io_priority].spa_active--; if (spa->spa_iokstat != NULL) { kstat_io_t *ksio = spa->spa_iokstat->ks_data; kstat_runq_exit(spa->spa_iokstat->ks_data); if (zio->io_type == ZIO_TYPE_READ) { ksio->reads++; ksio->nread += zio->io_size; } else if (zio->io_type == ZIO_TYPE_WRITE) { ksio->writes++; ksio->nwritten += zio->io_size; } } mutex_exit(&spa->spa_iokstat_lock); #endif } static void vdev_queue_agg_io_done(zio_t *aio) { if (aio->io_type == ZIO_TYPE_READ) { zio_t *pio; zio_link_t *zl = NULL; while ((pio = zio_walk_parents(aio, &zl)) != NULL) { abd_copy_off(pio->io_abd, aio->io_abd, 0, pio->io_offset - aio->io_offset, pio->io_size); } } abd_free(aio->io_abd); } static int vdev_queue_class_min_active(zio_priority_t p) { switch (p) { case ZIO_PRIORITY_SYNC_READ: return (zfs_vdev_sync_read_min_active); case ZIO_PRIORITY_SYNC_WRITE: return (zfs_vdev_sync_write_min_active); case ZIO_PRIORITY_ASYNC_READ: return (zfs_vdev_async_read_min_active); case ZIO_PRIORITY_ASYNC_WRITE: return (zfs_vdev_async_write_min_active); case ZIO_PRIORITY_SCRUB: return (zfs_vdev_scrub_min_active); case ZIO_PRIORITY_TRIM: return (zfs_vdev_trim_min_active); case ZIO_PRIORITY_REMOVAL: return (zfs_vdev_removal_min_active); case ZIO_PRIORITY_INITIALIZING: return (zfs_vdev_initializing_min_active); default: panic("invalid priority %u", p); return (0); } } static __noinline int vdev_queue_max_async_writes(spa_t *spa) { int writes; uint64_t dirty = spa->spa_dsl_pool->dp_dirty_total; uint64_t min_bytes = zfs_dirty_data_max * zfs_vdev_async_write_active_min_dirty_percent / 100; uint64_t max_bytes = zfs_dirty_data_max * zfs_vdev_async_write_active_max_dirty_percent / 100; /* * Sync tasks correspond to interactive user actions. To reduce the * execution time of those actions we push data out as fast as possible. */ if (spa_has_pending_synctask(spa)) { return (zfs_vdev_async_write_max_active); } if (dirty < min_bytes) return (zfs_vdev_async_write_min_active); if (dirty > max_bytes) return (zfs_vdev_async_write_max_active); /* * linear interpolation: * slope = (max_writes - min_writes) / (max_bytes - min_bytes) * move right by min_bytes * move up by min_writes */ writes = (dirty - min_bytes) * (zfs_vdev_async_write_max_active - zfs_vdev_async_write_min_active) / (max_bytes - min_bytes) + zfs_vdev_async_write_min_active; ASSERT3U(writes, >=, zfs_vdev_async_write_min_active); ASSERT3U(writes, <=, zfs_vdev_async_write_max_active); return (writes); } static int vdev_queue_class_max_active(spa_t *spa, zio_priority_t p) { switch (p) { case ZIO_PRIORITY_SYNC_READ: return (zfs_vdev_sync_read_max_active); case ZIO_PRIORITY_SYNC_WRITE: return (zfs_vdev_sync_write_max_active); case ZIO_PRIORITY_ASYNC_READ: return (zfs_vdev_async_read_max_active); case ZIO_PRIORITY_ASYNC_WRITE: return (vdev_queue_max_async_writes(spa)); case ZIO_PRIORITY_SCRUB: return (zfs_vdev_scrub_max_active); case ZIO_PRIORITY_TRIM: return (zfs_vdev_trim_max_active); case ZIO_PRIORITY_REMOVAL: return (zfs_vdev_removal_max_active); case ZIO_PRIORITY_INITIALIZING: return (zfs_vdev_initializing_max_active); default: panic("invalid priority %u", p); return (0); } } /* * Return the i/o class to issue from, or ZIO_PRIORITY_MAX_QUEUEABLE if * there is no eligible class. */ static zio_priority_t vdev_queue_class_to_issue(vdev_queue_t *vq) { spa_t *spa = vq->vq_vdev->vdev_spa; zio_priority_t p; ASSERT(MUTEX_HELD(&vq->vq_lock)); if (avl_numnodes(&vq->vq_active_tree) >= zfs_vdev_max_active) return (ZIO_PRIORITY_NUM_QUEUEABLE); /* find a queue that has not reached its minimum # outstanding i/os */ for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 && vq->vq_class[p].vqc_active < vdev_queue_class_min_active(p)) return (p); } /* * If we haven't found a queue, look for one that hasn't reached its * maximum # outstanding i/os. */ for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) { if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 && vq->vq_class[p].vqc_active < vdev_queue_class_max_active(spa, p)) return (p); } /* No eligible queued i/os */ return (ZIO_PRIORITY_NUM_QUEUEABLE); } /* * Compute the range spanned by two i/os, which is the endpoint of the last * (lio->io_offset + lio->io_size) minus start of the first (fio->io_offset). * Conveniently, the gap between fio and lio is given by -IO_SPAN(lio, fio); * thus fio and lio are adjacent if and only if IO_SPAN(lio, fio) == 0. */ #define IO_SPAN(fio, lio) ((lio)->io_offset + (lio)->io_size - (fio)->io_offset) #define IO_GAP(fio, lio) (-IO_SPAN(lio, fio)) static zio_t * vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio) { zio_t *first, *last, *aio, *dio, *mandatory, *nio; zio_link_t *zl = NULL; uint64_t maxgap = 0; uint64_t size; uint64_t limit; int maxblocksize; boolean_t stretch; avl_tree_t *t; enum zio_flag flags; ASSERT(MUTEX_HELD(&vq->vq_lock)); maxblocksize = spa_maxblocksize(vq->vq_vdev->vdev_spa); if (vq->vq_vdev->vdev_nonrot) limit = zfs_vdev_aggregation_limit_non_rotating; else limit = zfs_vdev_aggregation_limit; limit = MAX(MIN(limit, maxblocksize), 0); - if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE || zio->io_size >= limit) + if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE || limit == 0) return (NULL); first = last = zio; if (zio->io_type == ZIO_TYPE_READ) maxgap = zfs_vdev_read_gap_limit; /* * We can aggregate I/Os that are sufficiently adjacent and of * the same flavor, as expressed by the AGG_INHERIT flags. * The latter requirement is necessary so that certain * attributes of the I/O, such as whether it's a normal I/O * or a scrub/resilver, can be preserved in the aggregate. * We can include optional I/Os, but don't allow them * to begin a range as they add no benefit in that situation. */ /* * We keep track of the last non-optional I/O. */ mandatory = (first->io_flags & ZIO_FLAG_OPTIONAL) ? NULL : first; /* * Walk backwards through sufficiently contiguous I/Os * recording the last non-optional I/O. */ flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT; t = vdev_queue_type_tree(vq, zio->io_type); while (t != NULL && (dio = AVL_PREV(t, first)) != NULL && (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && IO_SPAN(dio, last) <= limit && IO_GAP(dio, first) <= maxgap && dio->io_type == zio->io_type) { first = dio; if (mandatory == NULL && !(first->io_flags & ZIO_FLAG_OPTIONAL)) mandatory = first; } /* * Skip any initial optional I/Os. */ while ((first->io_flags & ZIO_FLAG_OPTIONAL) && first != last) { first = AVL_NEXT(t, first); ASSERT(first != NULL); } /* * Walk forward through sufficiently contiguous I/Os. * The aggregation limit does not apply to optional i/os, so that * we can issue contiguous writes even if they are larger than the * aggregation limit. */ while ((dio = AVL_NEXT(t, last)) != NULL && (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags && (IO_SPAN(first, dio) <= limit || (dio->io_flags & ZIO_FLAG_OPTIONAL)) && IO_SPAN(first, dio) <= maxblocksize && IO_GAP(last, dio) <= maxgap && dio->io_type == zio->io_type) { last = dio; if (!(last->io_flags & ZIO_FLAG_OPTIONAL)) mandatory = last; } /* * Now that we've established the range of the I/O aggregation * we must decide what to do with trailing optional I/Os. * For reads, there's nothing to do. While we are unable to * aggregate further, it's possible that a trailing optional * I/O would allow the underlying device to aggregate with * subsequent I/Os. We must therefore determine if the next * non-optional I/O is close enough to make aggregation * worthwhile. */ stretch = B_FALSE; if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) { zio_t *nio = last; while ((dio = AVL_NEXT(t, nio)) != NULL && IO_GAP(nio, dio) == 0 && IO_GAP(mandatory, dio) <= zfs_vdev_write_gap_limit) { nio = dio; if (!(nio->io_flags & ZIO_FLAG_OPTIONAL)) { stretch = B_TRUE; break; } } } if (stretch) { /* * We are going to include an optional io in our aggregated * span, thus closing the write gap. Only mandatory i/os can * start aggregated spans, so make sure that the next i/o * after our span is mandatory. */ dio = AVL_NEXT(t, last); dio->io_flags &= ~ZIO_FLAG_OPTIONAL; } else { /* do not include the optional i/o */ while (last != mandatory && last != first) { ASSERT(last->io_flags & ZIO_FLAG_OPTIONAL); last = AVL_PREV(t, last); ASSERT(last != NULL); } } if (first == last) return (NULL); size = IO_SPAN(first, last); ASSERT3U(size, <=, maxblocksize); aio = zio_vdev_delegated_io(first->io_vd, first->io_offset, abd_alloc_for_io(size, B_TRUE), size, first->io_type, zio->io_priority, flags | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE, vdev_queue_agg_io_done, NULL); aio->io_timestamp = first->io_timestamp; nio = first; do { dio = nio; nio = AVL_NEXT(t, dio); ASSERT3U(dio->io_type, ==, aio->io_type); if (dio->io_flags & ZIO_FLAG_NODATA) { ASSERT3U(dio->io_type, ==, ZIO_TYPE_WRITE); abd_zero_off(aio->io_abd, dio->io_offset - aio->io_offset, dio->io_size); } else if (dio->io_type == ZIO_TYPE_WRITE) { abd_copy_off(aio->io_abd, dio->io_abd, dio->io_offset - aio->io_offset, 0, dio->io_size); } zio_add_child(dio, aio); vdev_queue_io_remove(vq, dio); } while (dio != last); /* * We need to drop the vdev queue's lock to avoid a deadlock that we * could encounter since this I/O will complete immediately. */ mutex_exit(&vq->vq_lock); while ((dio = zio_walk_parents(aio, &zl)) != NULL) { zio_vdev_io_bypass(dio); zio_execute(dio); } mutex_enter(&vq->vq_lock); return (aio); } static zio_t * vdev_queue_io_to_issue(vdev_queue_t *vq) { zio_t *zio, *aio; zio_priority_t p; avl_index_t idx; avl_tree_t *tree; zio_t search; again: ASSERT(MUTEX_HELD(&vq->vq_lock)); p = vdev_queue_class_to_issue(vq); if (p == ZIO_PRIORITY_NUM_QUEUEABLE) { /* No eligible queued i/os */ return (NULL); } /* * For LBA-ordered queues (async / scrub / initializing), issue the * i/o which follows the most recently issued i/o in LBA (offset) order. * * For FIFO queues (sync), issue the i/o with the lowest timestamp. */ tree = vdev_queue_class_tree(vq, p); search.io_timestamp = 0; search.io_offset = vq->vq_last_offset + 1; VERIFY3P(avl_find(tree, &search, &idx), ==, NULL); zio = avl_nearest(tree, idx, AVL_AFTER); if (zio == NULL) zio = avl_first(tree); ASSERT3U(zio->io_priority, ==, p); aio = vdev_queue_aggregate(vq, zio); if (aio != NULL) zio = aio; else vdev_queue_io_remove(vq, zio); /* * If the I/O is or was optional and therefore has no data, we need to * simply discard it. We need to drop the vdev queue's lock to avoid a * deadlock that we could encounter since this I/O will complete * immediately. */ if (zio->io_flags & ZIO_FLAG_NODATA) { mutex_exit(&vq->vq_lock); zio_vdev_io_bypass(zio); zio_execute(zio); mutex_enter(&vq->vq_lock); goto again; } vdev_queue_pending_add(vq, zio); vq->vq_last_offset = zio->io_offset; return (zio); } zio_t * vdev_queue_io(zio_t *zio) { vdev_queue_t *vq = &zio->io_vd->vdev_queue; zio_t *nio; if (zio->io_flags & ZIO_FLAG_DONT_QUEUE) return (zio); /* * Children i/os inherent their parent's priority, which might * not match the child's i/o type. Fix it up here. */ if (zio->io_type == ZIO_TYPE_READ) { if (zio->io_priority != ZIO_PRIORITY_SYNC_READ && zio->io_priority != ZIO_PRIORITY_ASYNC_READ && zio->io_priority != ZIO_PRIORITY_SCRUB && zio->io_priority != ZIO_PRIORITY_REMOVAL && zio->io_priority != ZIO_PRIORITY_INITIALIZING) zio->io_priority = ZIO_PRIORITY_ASYNC_READ; } else if (zio->io_type == ZIO_TYPE_WRITE) { if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE && zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE && zio->io_priority != ZIO_PRIORITY_REMOVAL && zio->io_priority != ZIO_PRIORITY_INITIALIZING) zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE; } else { ASSERT(zio->io_type == ZIO_TYPE_FREE); zio->io_priority = ZIO_PRIORITY_TRIM; } zio->io_flags |= ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE; mutex_enter(&vq->vq_lock); zio->io_timestamp = gethrtime(); vdev_queue_io_add(vq, zio); nio = vdev_queue_io_to_issue(vq); mutex_exit(&vq->vq_lock); if (nio == NULL) return (NULL); if (nio->io_done == vdev_queue_agg_io_done) { zio_nowait(nio); return (NULL); } return (nio); } void vdev_queue_io_done(zio_t *zio) { vdev_queue_t *vq = &zio->io_vd->vdev_queue; zio_t *nio; mutex_enter(&vq->vq_lock); vdev_queue_pending_remove(vq, zio); vq->vq_io_complete_ts = gethrtime(); while ((nio = vdev_queue_io_to_issue(vq)) != NULL) { mutex_exit(&vq->vq_lock); if (nio->io_done == vdev_queue_agg_io_done) { zio_nowait(nio); } else { zio_vdev_io_reissue(nio); zio_execute(nio); } mutex_enter(&vq->vq_lock); } mutex_exit(&vq->vq_lock); } void vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority) { vdev_queue_t *vq = &zio->io_vd->vdev_queue; avl_tree_t *tree; /* * ZIO_PRIORITY_NOW is used by the vdev cache code and the aggregate zio * code to issue IOs without adding them to the vdev queue. In this * case, the zio is already going to be issued as quickly as possible * and so it doesn't need any reprioitization to help. */ if (zio->io_priority == ZIO_PRIORITY_NOW) return; ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); if (zio->io_type == ZIO_TYPE_READ) { if (priority != ZIO_PRIORITY_SYNC_READ && priority != ZIO_PRIORITY_ASYNC_READ && priority != ZIO_PRIORITY_SCRUB) priority = ZIO_PRIORITY_ASYNC_READ; } else { ASSERT(zio->io_type == ZIO_TYPE_WRITE); if (priority != ZIO_PRIORITY_SYNC_WRITE && priority != ZIO_PRIORITY_ASYNC_WRITE) priority = ZIO_PRIORITY_ASYNC_WRITE; } mutex_enter(&vq->vq_lock); /* * If the zio is in none of the queues we can simply change * the priority. If the zio is waiting to be submitted we must * remove it from the queue and re-insert it with the new priority. * Otherwise, the zio is currently active and we cannot change its * priority. */ tree = vdev_queue_class_tree(vq, zio->io_priority); if (avl_find(tree, zio, NULL) == zio) { avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio); zio->io_priority = priority; avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio); } else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) { zio->io_priority = priority; } mutex_exit(&vq->vq_lock); } /* * As these three methods are only used for load calculations we're not concerned * if we get an incorrect value on 32bit platforms due to lack of vq_lock mutex * use here, instead we prefer to keep it lock free for performance. */ int vdev_queue_length(vdev_t *vd) { return (avl_numnodes(&vd->vdev_queue.vq_active_tree)); } uint64_t vdev_queue_lastoffset(vdev_t *vd) { return (vd->vdev_queue.vq_lastoffset); } void vdev_queue_register_lastoffset(vdev_t *vd, zio_t *zio) { vd->vdev_queue.vq_lastoffset = zio->io_offset + zio->io_size; } Index: projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c =================================================================== --- projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c (revision 345026) @@ -1,1362 +1,1358 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved. */ /* * ZFS control directory (a.k.a. ".zfs") * * This directory provides a common location for all ZFS meta-objects. * Currently, this is only the 'snapshot' directory, but this may expand in the * future. The elements are built using the GFS primitives, as the hierarchy * does not actually exist on disk. * * For 'snapshot', we don't want to have all snapshots always mounted, because * this would take up a huge amount of space in /etc/mnttab. We have three * types of objects: * * ctldir ------> snapshotdir -------> snapshot * | * | * V * mounted fs * * The 'snapshot' node contains just enough information to lookup '..' and act * as a mountpoint for the snapshot. Whenever we lookup a specific snapshot, we * perform an automount of the underlying filesystem and return the * corresponding vnode. * * All mounts are handled automatically by the kernel, but unmounts are * (currently) handled from user land. The main reason is that there is no * reliable way to auto-unmount the filesystem when it's "no longer in use". * When the user unmounts a filesystem, we call zfsctl_unmount(), which * unmounts any snapshots within the snapshot directory. * * The '.zfs', '.zfs/snapshot', and all directories created under * '.zfs/snapshot' (ie: '.zfs/snapshot/') are all GFS nodes and * share the same vfs_t as the head filesystem (what '.zfs' lives under). * * File systems mounted ontop of the GFS nodes '.zfs/snapshot/' * (ie: snapshots) are ZFS nodes and have their own unique vfs_t. * However, vnodes within these mounted on file systems have their v_vfsp * fields set to the head filesystem to make NFS happy (see * zfsctl_snapdir_lookup()). We VFS_HOLD the head filesystem's vfs_t * so that it cannot be freed until all snapshots have been unmounted. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "zfs_namecheck.h" /* Common access mode for all virtual directories under the ctldir */ const u_short zfsctl_ctldir_mode = S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; /* * "Synthetic" filesystem implementation. */ /* * Assert that A implies B. */ #define KASSERT_IMPLY(A, B, msg) KASSERT(!(A) || (B), (msg)); static MALLOC_DEFINE(M_SFSNODES, "sfs_nodes", "synthetic-fs nodes"); typedef struct sfs_node { char sn_name[ZFS_MAX_DATASET_NAME_LEN]; uint64_t sn_parent_id; uint64_t sn_id; } sfs_node_t; /* * Check the parent's ID as well as the node's to account for a chance * that IDs originating from different domains (snapshot IDs, artifical * IDs, znode IDs) may clash. */ static int sfs_compare_ids(struct vnode *vp, void *arg) { sfs_node_t *n1 = vp->v_data; sfs_node_t *n2 = arg; bool equal; equal = n1->sn_id == n2->sn_id && n1->sn_parent_id == n2->sn_parent_id; /* Zero means equality. */ return (!equal); } static int sfs_vnode_get(const struct mount *mp, int flags, uint64_t parent_id, uint64_t id, struct vnode **vpp) { sfs_node_t search; int err; search.sn_id = id; search.sn_parent_id = parent_id; err = vfs_hash_get(mp, (u_int)id, flags, curthread, vpp, sfs_compare_ids, &search); return (err); } static int sfs_vnode_insert(struct vnode *vp, int flags, uint64_t parent_id, uint64_t id, struct vnode **vpp) { int err; KASSERT(vp->v_data != NULL, ("sfs_vnode_insert with NULL v_data")); err = vfs_hash_insert(vp, (u_int)id, flags, curthread, vpp, sfs_compare_ids, vp->v_data); return (err); } static void sfs_vnode_remove(struct vnode *vp) { vfs_hash_remove(vp); } typedef void sfs_vnode_setup_fn(vnode_t *vp, void *arg); static int sfs_vgetx(struct mount *mp, int flags, uint64_t parent_id, uint64_t id, const char *tag, struct vop_vector *vops, sfs_vnode_setup_fn setup, void *arg, struct vnode **vpp) { struct vnode *vp; int error; error = sfs_vnode_get(mp, flags, parent_id, id, vpp); if (error != 0 || *vpp != NULL) { KASSERT_IMPLY(error == 0, (*vpp)->v_data != NULL, "sfs vnode with no data"); return (error); } /* Allocate a new vnode/inode. */ error = getnewvnode(tag, mp, vops, &vp); if (error != 0) { *vpp = NULL; return (error); } /* * Exclusively lock the vnode vnode while it's being constructed. */ lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); error = insmntque(vp, mp); if (error != 0) { *vpp = NULL; return (error); } setup(vp, arg); error = sfs_vnode_insert(vp, flags, parent_id, id, vpp); if (error != 0 || *vpp != NULL) { KASSERT_IMPLY(error == 0, (*vpp)->v_data != NULL, "sfs vnode with no data"); return (error); } *vpp = vp; return (0); } static void sfs_print_node(sfs_node_t *node) { printf("\tname = %s\n", node->sn_name); printf("\tparent_id = %ju\n", (uintmax_t)node->sn_parent_id); printf("\tid = %ju\n", (uintmax_t)node->sn_id); } static sfs_node_t * sfs_alloc_node(size_t size, const char *name, uint64_t parent_id, uint64_t id) { struct sfs_node *node; KASSERT(strlen(name) < sizeof(node->sn_name), ("sfs node name is too long")); KASSERT(size >= sizeof(*node), ("sfs node size is too small")); node = malloc(size, M_SFSNODES, M_WAITOK | M_ZERO); strlcpy(node->sn_name, name, sizeof(node->sn_name)); node->sn_parent_id = parent_id; node->sn_id = id; return (node); } static void sfs_destroy_node(sfs_node_t *node) { free(node, M_SFSNODES); } static void * sfs_reclaim_vnode(vnode_t *vp) { sfs_node_t *node; void *data; sfs_vnode_remove(vp); data = vp->v_data; vp->v_data = NULL; return (data); } static int sfs_readdir_common(uint64_t parent_id, uint64_t id, struct vop_readdir_args *ap, uio_t *uio, off_t *offp) { struct dirent entry; int error; /* Reset ncookies for subsequent use of vfs_read_dirent. */ if (ap->a_ncookies != NULL) *ap->a_ncookies = 0; if (uio->uio_resid < sizeof(entry)) return (SET_ERROR(EINVAL)); if (uio->uio_offset < 0) return (SET_ERROR(EINVAL)); if (uio->uio_offset == 0) { entry.d_fileno = id; entry.d_type = DT_DIR; entry.d_name[0] = '.'; entry.d_namlen = 1; entry.d_reclen = sizeof(entry); dirent_terminate(&entry); error = vfs_read_dirent(ap, &entry, uio->uio_offset); if (error != 0) return (SET_ERROR(error)); } if (uio->uio_offset < sizeof(entry)) return (SET_ERROR(EINVAL)); if (uio->uio_offset == sizeof(entry)) { entry.d_fileno = parent_id; entry.d_type = DT_DIR; entry.d_name[0] = '.'; entry.d_name[1] = '.'; entry.d_namlen = 2; entry.d_reclen = sizeof(entry); dirent_terminate(&entry); error = vfs_read_dirent(ap, &entry, uio->uio_offset); if (error != 0) return (SET_ERROR(error)); } if (offp != NULL) *offp = 2 * sizeof(entry); return (0); } /* * .zfs inode namespace * * We need to generate unique inode numbers for all files and directories * within the .zfs pseudo-filesystem. We use the following scheme: * * ENTRY ZFSCTL_INODE * .zfs 1 * .zfs/snapshot 2 * .zfs/snapshot/ objectid(snap) */ #define ZFSCTL_INO_SNAP(id) (id) static struct vop_vector zfsctl_ops_root; static struct vop_vector zfsctl_ops_snapdir; static struct vop_vector zfsctl_ops_snapshot; static struct vop_vector zfsctl_ops_shares_dir; void zfsctl_init(void) { } void zfsctl_fini(void) { } boolean_t zfsctl_is_node(vnode_t *vp) { return (vn_matchops(vp, zfsctl_ops_root) || vn_matchops(vp, zfsctl_ops_snapdir) || vn_matchops(vp, zfsctl_ops_snapshot) || vn_matchops(vp, zfsctl_ops_shares_dir)); } typedef struct zfsctl_root { sfs_node_t node; sfs_node_t *snapdir; timestruc_t cmtime; } zfsctl_root_t; /* * Create the '.zfs' directory. */ void zfsctl_create(zfsvfs_t *zfsvfs) { zfsctl_root_t *dot_zfs; sfs_node_t *snapdir; vnode_t *rvp; uint64_t crtime[2]; ASSERT(zfsvfs->z_ctldir == NULL); snapdir = sfs_alloc_node(sizeof(*snapdir), "snapshot", ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR); dot_zfs = (zfsctl_root_t *)sfs_alloc_node(sizeof(*dot_zfs), ".zfs", 0, ZFSCTL_INO_ROOT); dot_zfs->snapdir = snapdir; VERIFY(VFS_ROOT(zfsvfs->z_vfs, LK_EXCLUSIVE, &rvp) == 0); VERIFY(0 == sa_lookup(VTOZ(rvp)->z_sa_hdl, SA_ZPL_CRTIME(zfsvfs), &crtime, sizeof(crtime))); ZFS_TIME_DECODE(&dot_zfs->cmtime, crtime); vput(rvp); zfsvfs->z_ctldir = dot_zfs; } /* * Destroy the '.zfs' directory. Only called when the filesystem is unmounted. * The nodes must not have any associated vnodes by now as they should be * vflush-ed. */ void zfsctl_destroy(zfsvfs_t *zfsvfs) { sfs_destroy_node(zfsvfs->z_ctldir->snapdir); sfs_destroy_node((sfs_node_t *)zfsvfs->z_ctldir); zfsvfs->z_ctldir = NULL; } static int zfsctl_fs_root_vnode(struct mount *mp, void *arg __unused, int flags, struct vnode **vpp) { return (VFS_ROOT(mp, flags, vpp)); } static void zfsctl_common_vnode_setup(vnode_t *vp, void *arg) { ASSERT_VOP_ELOCKED(vp, __func__); /* We support shared locking. */ VN_LOCK_ASHARE(vp); vp->v_type = VDIR; vp->v_data = arg; } static int zfsctl_root_vnode(struct mount *mp, void *arg __unused, int flags, struct vnode **vpp) { void *node; int err; node = ((zfsvfs_t*)mp->mnt_data)->z_ctldir; err = sfs_vgetx(mp, flags, 0, ZFSCTL_INO_ROOT, "zfs", &zfsctl_ops_root, zfsctl_common_vnode_setup, node, vpp); return (err); } static int zfsctl_snapdir_vnode(struct mount *mp, void *arg __unused, int flags, struct vnode **vpp) { void *node; int err; node = ((zfsvfs_t*)mp->mnt_data)->z_ctldir->snapdir; err = sfs_vgetx(mp, flags, ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR, "zfs", &zfsctl_ops_snapdir, zfsctl_common_vnode_setup, node, vpp); return (err); } /* * Given a root znode, retrieve the associated .zfs directory. * Add a hold to the vnode and return it. */ int zfsctl_root(zfsvfs_t *zfsvfs, int flags, vnode_t **vpp) { vnode_t *vp; int error; error = zfsctl_root_vnode(zfsvfs->z_vfs, NULL, flags, vpp); return (error); } /* * Common open routine. Disallow any write access. */ static int zfsctl_common_open(struct vop_open_args *ap) { int flags = ap->a_mode; if (flags & FWRITE) return (SET_ERROR(EACCES)); return (0); } /* * Common close routine. Nothing to do here. */ /* ARGSUSED */ static int zfsctl_common_close(struct vop_close_args *ap) { return (0); } /* * Common access routine. Disallow writes. */ static int zfsctl_common_access(ap) struct vop_access_args /* { struct vnode *a_vp; accmode_t a_accmode; struct ucred *a_cred; struct thread *a_td; } */ *ap; { accmode_t accmode = ap->a_accmode; if (accmode & VWRITE) return (SET_ERROR(EACCES)); return (0); } /* * Common getattr function. Fill in basic information. */ static void zfsctl_common_getattr(vnode_t *vp, vattr_t *vap) { timestruc_t now; sfs_node_t *node; node = vp->v_data; vap->va_uid = 0; vap->va_gid = 0; vap->va_rdev = 0; /* * We are a purely virtual object, so we have no * blocksize or allocated blocks. */ vap->va_blksize = 0; vap->va_nblocks = 0; vap->va_seq = 0; vn_fsid(vp, vap); vap->va_mode = zfsctl_ctldir_mode; vap->va_type = VDIR; /* * We live in the now (for atime). */ gethrestime(&now); vap->va_atime = now; /* FreeBSD: Reset chflags(2) flags. */ vap->va_flags = 0; vap->va_nodeid = node->sn_id; /* At least '.' and '..'. */ vap->va_nlink = 2; } static int zfsctl_common_fid(ap) struct vop_fid_args /* { struct vnode *a_vp; struct fid *a_fid; } */ *ap; { vnode_t *vp = ap->a_vp; fid_t *fidp = (void *)ap->a_fid; sfs_node_t *node = vp->v_data; uint64_t object = node->sn_id; zfid_short_t *zfid; int i; zfid = (zfid_short_t *)fidp; zfid->zf_len = SHORT_FID_LEN; for (i = 0; i < sizeof(zfid->zf_object); i++) zfid->zf_object[i] = (uint8_t)(object >> (8 * i)); /* .zfs nodes always have a generation number of 0 */ for (i = 0; i < sizeof(zfid->zf_gen); i++) zfid->zf_gen[i] = 0; return (0); } static int zfsctl_common_reclaim(ap) struct vop_reclaim_args /* { struct vnode *a_vp; struct thread *a_td; } */ *ap; { vnode_t *vp = ap->a_vp; (void) sfs_reclaim_vnode(vp); return (0); } static int zfsctl_common_print(ap) struct vop_print_args /* { struct vnode *a_vp; } */ *ap; { sfs_print_node(ap->a_vp->v_data); return (0); } /* * Get root directory attributes. */ static int zfsctl_root_getattr(ap) struct vop_getattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { struct vnode *vp = ap->a_vp; struct vattr *vap = ap->a_vap; zfsctl_root_t *node = vp->v_data; zfsctl_common_getattr(vp, vap); vap->va_ctime = node->cmtime; vap->va_mtime = vap->va_ctime; vap->va_birthtime = vap->va_ctime; vap->va_nlink += 1; /* snapdir */ vap->va_size = vap->va_nlink; return (0); } /* * When we lookup "." we still can be asked to lock it * differently, can't we? */ int zfsctl_relock_dot(vnode_t *dvp, int ltype) { vref(dvp); if (ltype != VOP_ISLOCKED(dvp)) { if (ltype == LK_EXCLUSIVE) vn_lock(dvp, LK_UPGRADE | LK_RETRY); else /* if (ltype == LK_SHARED) */ vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); /* Relock for the "." case may left us with reclaimed vnode. */ if ((dvp->v_iflag & VI_DOOMED) != 0) { vrele(dvp); return (SET_ERROR(ENOENT)); } } return (0); } /* * Special case the handling of "..". */ int zfsctl_root_lookup(ap) struct vop_lookup_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; } */ *ap; { struct componentname *cnp = ap->a_cnp; vnode_t *dvp = ap->a_dvp; vnode_t **vpp = ap->a_vpp; cred_t *cr = ap->a_cnp->cn_cred; int flags = ap->a_cnp->cn_flags; int lkflags = ap->a_cnp->cn_lkflags; int nameiop = ap->a_cnp->cn_nameiop; int err; int ltype; ASSERT(dvp->v_type == VDIR); if ((flags & ISLASTCN) != 0 && nameiop != LOOKUP) return (SET_ERROR(ENOTSUP)); if (cnp->cn_namelen == 1 && *cnp->cn_nameptr == '.') { err = zfsctl_relock_dot(dvp, lkflags & LK_TYPE_MASK); if (err == 0) *vpp = dvp; } else if ((flags & ISDOTDOT) != 0) { err = vn_vget_ino_gen(dvp, zfsctl_fs_root_vnode, NULL, lkflags, vpp); } else if (strncmp(cnp->cn_nameptr, "snapshot", cnp->cn_namelen) == 0) { err = zfsctl_snapdir_vnode(dvp->v_mount, NULL, lkflags, vpp); } else { err = SET_ERROR(ENOENT); } if (err != 0) *vpp = NULL; return (err); } static int zfsctl_root_readdir(ap) struct vop_readdir_args /* { struct vnode *a_vp; struct uio *a_uio; struct ucred *a_cred; int *a_eofflag; int *ncookies; u_long **a_cookies; } */ *ap; { struct dirent entry; vnode_t *vp = ap->a_vp; zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; zfsctl_root_t *node = vp->v_data; uio_t *uio = ap->a_uio; int *eofp = ap->a_eofflag; off_t dots_offset; int error; ASSERT(vp->v_type == VDIR); error = sfs_readdir_common(zfsvfs->z_root, ZFSCTL_INO_ROOT, ap, uio, &dots_offset); if (error != 0) { if (error == ENAMETOOLONG) /* ran out of destination space */ error = 0; return (error); } if (uio->uio_offset != dots_offset) return (SET_ERROR(EINVAL)); CTASSERT(sizeof(node->snapdir->sn_name) <= sizeof(entry.d_name)); entry.d_fileno = node->snapdir->sn_id; entry.d_type = DT_DIR; strcpy(entry.d_name, node->snapdir->sn_name); entry.d_namlen = strlen(entry.d_name); entry.d_reclen = sizeof(entry); dirent_terminate(&entry); error = vfs_read_dirent(ap, &entry, uio->uio_offset); if (error != 0) { if (error == ENAMETOOLONG) error = 0; return (SET_ERROR(error)); } if (eofp != NULL) *eofp = 1; return (0); } static int zfsctl_root_vptocnp(struct vop_vptocnp_args *ap) { static const char dotzfs_name[4] = ".zfs"; vnode_t *dvp; int error; if (*ap->a_buflen < sizeof (dotzfs_name)) return (SET_ERROR(ENOMEM)); error = vn_vget_ino_gen(ap->a_vp, zfsctl_fs_root_vnode, NULL, LK_SHARED, &dvp); if (error != 0) return (SET_ERROR(error)); VOP_UNLOCK(dvp, 0); *ap->a_vpp = dvp; *ap->a_buflen -= sizeof (dotzfs_name); bcopy(dotzfs_name, ap->a_buf + *ap->a_buflen, sizeof (dotzfs_name)); return (0); } static int zfsctl_common_pathconf(ap) struct vop_pathconf_args /* { struct vnode *a_vp; int a_name; int *a_retval; } */ *ap; { /* * We care about ACL variables so that user land utilities like ls * can display them correctly. Since the ctldir's st_dev is set to be * the same as the parent dataset, we must support all variables that * it supports. */ switch (ap->a_name) { case _PC_LINK_MAX: *ap->a_retval = MIN(LONG_MAX, ZFS_LINK_MAX); return (0); case _PC_FILESIZEBITS: *ap->a_retval = 64; return (0); case _PC_MIN_HOLE_SIZE: *ap->a_retval = (int)SPA_MINBLOCKSIZE; return (0); - case _PC_ACL_EXTENDED: - *ap->a_retval = 0; - return (0); - case _PC_ACL_NFS4: *ap->a_retval = 1; return (0); case _PC_ACL_PATH_MAX: *ap->a_retval = ACL_MAX_ENTRIES; return (0); case _PC_NAME_MAX: *ap->a_retval = NAME_MAX; return (0); default: return (vop_stdpathconf(ap)); } } /** * Returns a trivial ACL */ int zfsctl_common_getacl(ap) struct vop_getacl_args /* { struct vnode *vp; acl_type_t a_type; struct acl *a_aclp; struct ucred *cred; struct thread *td; } */ *ap; { int i; if (ap->a_type != ACL_TYPE_NFS4) return (EINVAL); acl_nfs4_sync_acl_from_mode(ap->a_aclp, zfsctl_ctldir_mode, 0); /* * acl_nfs4_sync_acl_from_mode assumes that the owner can always modify * attributes. That is not the case for the ctldir, so we must clear * those bits. We also must clear ACL_READ_NAMED_ATTRS, because xattrs * aren't supported by the ctldir. */ for (i = 0; i < ap->a_aclp->acl_cnt; i++) { struct acl_entry *entry; entry = &(ap->a_aclp->acl_entry[i]); uint32_t old_perm = entry->ae_perm; entry->ae_perm &= ~(ACL_WRITE_ACL | ACL_WRITE_OWNER | ACL_WRITE_ATTRIBUTES | ACL_WRITE_NAMED_ATTRS | ACL_READ_NAMED_ATTRS ); } return (0); } static struct vop_vector zfsctl_ops_root = { .vop_default = &default_vnodeops, .vop_open = zfsctl_common_open, .vop_close = zfsctl_common_close, .vop_ioctl = VOP_EINVAL, .vop_getattr = zfsctl_root_getattr, .vop_access = zfsctl_common_access, .vop_readdir = zfsctl_root_readdir, .vop_lookup = zfsctl_root_lookup, .vop_inactive = VOP_NULL, .vop_reclaim = zfsctl_common_reclaim, .vop_fid = zfsctl_common_fid, .vop_print = zfsctl_common_print, .vop_vptocnp = zfsctl_root_vptocnp, .vop_pathconf = zfsctl_common_pathconf, .vop_getacl = zfsctl_common_getacl, }; static int zfsctl_snapshot_zname(vnode_t *vp, const char *name, int len, char *zname) { objset_t *os = ((zfsvfs_t *)((vp)->v_vfsp->vfs_data))->z_os; dmu_objset_name(os, zname); if (strlen(zname) + 1 + strlen(name) >= len) return (SET_ERROR(ENAMETOOLONG)); (void) strcat(zname, "@"); (void) strcat(zname, name); return (0); } static int zfsctl_snapshot_lookup(vnode_t *vp, const char *name, uint64_t *id) { objset_t *os = ((zfsvfs_t *)((vp)->v_vfsp->vfs_data))->z_os; int err; err = dsl_dataset_snap_lookup(dmu_objset_ds(os), name, id); return (err); } /* * Given a vnode get a root vnode of a filesystem mounted on top of * the vnode, if any. The root vnode is referenced and locked. * If no filesystem is mounted then the orinal vnode remains referenced * and locked. If any error happens the orinal vnode is unlocked and * released. */ static int zfsctl_mounted_here(vnode_t **vpp, int flags) { struct mount *mp; int err; ASSERT_VOP_LOCKED(*vpp, __func__); ASSERT3S((*vpp)->v_type, ==, VDIR); if ((mp = (*vpp)->v_mountedhere) != NULL) { err = vfs_busy(mp, 0); KASSERT(err == 0, ("vfs_busy(mp, 0) failed with %d", err)); KASSERT(vrefcnt(*vpp) > 1, ("unreferenced mountpoint")); vput(*vpp); err = VFS_ROOT(mp, flags, vpp); vfs_unbusy(mp); return (err); } return (EJUSTRETURN); } typedef struct { const char *snap_name; uint64_t snap_id; } snapshot_setup_arg_t; static void zfsctl_snapshot_vnode_setup(vnode_t *vp, void *arg) { snapshot_setup_arg_t *ssa = arg; sfs_node_t *node; ASSERT_VOP_ELOCKED(vp, __func__); node = sfs_alloc_node(sizeof(sfs_node_t), ssa->snap_name, ZFSCTL_INO_SNAPDIR, ssa->snap_id); zfsctl_common_vnode_setup(vp, node); /* We have to support recursive locking. */ VN_LOCK_AREC(vp); } /* * Lookup entry point for the 'snapshot' directory. Try to open the * snapshot if it exist, creating the pseudo filesystem vnode as necessary. * Perform a mount of the associated dataset on top of the vnode. * There are four possibilities: * - the snapshot node and vnode do not exist * - the snapshot vnode is covered by the mounted snapshot * - the snapshot vnode is not covered yet, the mount operation is in progress * - the snapshot vnode is not covered, because the snapshot has been unmounted * The last two states are transient and should be relatively short-lived. */ int zfsctl_snapdir_lookup(ap) struct vop_lookup_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; } */ *ap; { vnode_t *dvp = ap->a_dvp; vnode_t **vpp = ap->a_vpp; struct componentname *cnp = ap->a_cnp; char name[NAME_MAX + 1]; char fullname[ZFS_MAX_DATASET_NAME_LEN]; char *mountpoint; size_t mountpoint_len; zfsvfs_t *zfsvfs = dvp->v_vfsp->vfs_data; uint64_t snap_id; int nameiop = cnp->cn_nameiop; int lkflags = cnp->cn_lkflags; int flags = cnp->cn_flags; int err; ASSERT(dvp->v_type == VDIR); if ((flags & ISLASTCN) != 0 && nameiop != LOOKUP) return (SET_ERROR(ENOTSUP)); if (cnp->cn_namelen == 1 && *cnp->cn_nameptr == '.') { err = zfsctl_relock_dot(dvp, lkflags & LK_TYPE_MASK); if (err == 0) *vpp = dvp; return (err); } if (flags & ISDOTDOT) { err = vn_vget_ino_gen(dvp, zfsctl_root_vnode, NULL, lkflags, vpp); return (err); } if (cnp->cn_namelen >= sizeof(name)) return (SET_ERROR(ENAMETOOLONG)); strlcpy(name, ap->a_cnp->cn_nameptr, ap->a_cnp->cn_namelen + 1); err = zfsctl_snapshot_lookup(dvp, name, &snap_id); if (err != 0) return (SET_ERROR(ENOENT)); for (;;) { snapshot_setup_arg_t ssa; ssa.snap_name = name; ssa.snap_id = snap_id; err = sfs_vgetx(dvp->v_mount, LK_SHARED, ZFSCTL_INO_SNAPDIR, snap_id, "zfs", &zfsctl_ops_snapshot, zfsctl_snapshot_vnode_setup, &ssa, vpp); if (err != 0) return (err); /* Check if a new vnode has just been created. */ if (VOP_ISLOCKED(*vpp) == LK_EXCLUSIVE) break; /* * Check if a snapshot is already mounted on top of the vnode. */ err = zfsctl_mounted_here(vpp, lkflags); if (err != EJUSTRETURN) return (err); /* * If the vnode is not covered, then either the mount operation * is in progress or the snapshot has already been unmounted * but the vnode hasn't been inactivated and reclaimed yet. * We can try to re-use the vnode in the latter case. */ VI_LOCK(*vpp); if (((*vpp)->v_iflag & VI_MOUNT) == 0) { /* Upgrade to exclusive lock in order to: * - avoid race conditions * - satisfy the contract of mount_snapshot() */ err = VOP_LOCK(*vpp, LK_TRYUPGRADE | LK_INTERLOCK); if (err == 0) break; } else { VI_UNLOCK(*vpp); } /* * In this state we can loop on uncontested locks and starve * the thread doing the lengthy, non-trivial mount operation. * So, yield to prevent that from happening. */ vput(*vpp); kern_yield(PRI_USER); } VERIFY0(zfsctl_snapshot_zname(dvp, name, sizeof(fullname), fullname)); mountpoint_len = strlen(dvp->v_vfsp->mnt_stat.f_mntonname) + strlen("/" ZFS_CTLDIR_NAME "/snapshot/") + strlen(name) + 1; mountpoint = kmem_alloc(mountpoint_len, KM_SLEEP); (void) snprintf(mountpoint, mountpoint_len, "%s/" ZFS_CTLDIR_NAME "/snapshot/%s", dvp->v_vfsp->mnt_stat.f_mntonname, name); err = mount_snapshot(curthread, vpp, "zfs", mountpoint, fullname, 0); kmem_free(mountpoint, mountpoint_len); if (err == 0) { /* * Fix up the root vnode mounted on .zfs/snapshot/. * * This is where we lie about our v_vfsp in order to * make .zfs/snapshot/ accessible over NFS * without requiring manual mounts of . */ ASSERT(VTOZ(*vpp)->z_zfsvfs != zfsvfs); VTOZ(*vpp)->z_zfsvfs->z_parent = zfsvfs; /* Clear the root flag (set via VFS_ROOT) as well. */ (*vpp)->v_vflag &= ~VV_ROOT; } if (err != 0) *vpp = NULL; return (err); } static int zfsctl_snapdir_readdir(ap) struct vop_readdir_args /* { struct vnode *a_vp; struct uio *a_uio; struct ucred *a_cred; int *a_eofflag; int *ncookies; u_long **a_cookies; } */ *ap; { char snapname[ZFS_MAX_DATASET_NAME_LEN]; struct dirent entry; vnode_t *vp = ap->a_vp; zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; uio_t *uio = ap->a_uio; int *eofp = ap->a_eofflag; off_t dots_offset; int error; ASSERT(vp->v_type == VDIR); error = sfs_readdir_common(ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR, ap, uio, &dots_offset); if (error != 0) { if (error == ENAMETOOLONG) /* ran out of destination space */ error = 0; return (error); } ZFS_ENTER(zfsvfs); for (;;) { uint64_t cookie; uint64_t id; cookie = uio->uio_offset - dots_offset; dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG); error = dmu_snapshot_list_next(zfsvfs->z_os, sizeof(snapname), snapname, &id, &cookie, NULL); dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG); if (error != 0) { if (error == ENOENT) { if (eofp != NULL) *eofp = 1; error = 0; } ZFS_EXIT(zfsvfs); return (error); } entry.d_fileno = id; entry.d_type = DT_DIR; strcpy(entry.d_name, snapname); entry.d_namlen = strlen(entry.d_name); entry.d_reclen = sizeof(entry); /* NOTE: d_off is the offset for the *next* entry. */ entry.d_off = cookie + dots_offset; dirent_terminate(&entry); error = vfs_read_dirent(ap, &entry, uio->uio_offset); if (error != 0) { if (error == ENAMETOOLONG) error = 0; ZFS_EXIT(zfsvfs); return (SET_ERROR(error)); } uio->uio_offset = cookie + dots_offset; } /* NOTREACHED */ } static int zfsctl_snapdir_getattr(ap) struct vop_getattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { vnode_t *vp = ap->a_vp; vattr_t *vap = ap->a_vap; zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; dsl_dataset_t *ds = dmu_objset_ds(zfsvfs->z_os); sfs_node_t *node = vp->v_data; uint64_t snap_count; int err; ZFS_ENTER(zfsvfs); zfsctl_common_getattr(vp, vap); vap->va_ctime = dmu_objset_snap_cmtime(zfsvfs->z_os); vap->va_mtime = vap->va_ctime; vap->va_birthtime = vap->va_ctime; if (dsl_dataset_phys(ds)->ds_snapnames_zapobj != 0) { err = zap_count(dmu_objset_pool(ds->ds_objset)->dp_meta_objset, dsl_dataset_phys(ds)->ds_snapnames_zapobj, &snap_count); if (err != 0) { ZFS_EXIT(zfsvfs); return (err); } vap->va_nlink += snap_count; } vap->va_size = vap->va_nlink; ZFS_EXIT(zfsvfs); return (0); } static struct vop_vector zfsctl_ops_snapdir = { .vop_default = &default_vnodeops, .vop_open = zfsctl_common_open, .vop_close = zfsctl_common_close, .vop_getattr = zfsctl_snapdir_getattr, .vop_access = zfsctl_common_access, .vop_readdir = zfsctl_snapdir_readdir, .vop_lookup = zfsctl_snapdir_lookup, .vop_reclaim = zfsctl_common_reclaim, .vop_fid = zfsctl_common_fid, .vop_print = zfsctl_common_print, .vop_pathconf = zfsctl_common_pathconf, .vop_getacl = zfsctl_common_getacl, }; static int zfsctl_snapshot_inactive(ap) struct vop_inactive_args /* { struct vnode *a_vp; struct thread *a_td; } */ *ap; { vnode_t *vp = ap->a_vp; VERIFY(vrecycle(vp) == 1); return (0); } static int zfsctl_snapshot_reclaim(ap) struct vop_reclaim_args /* { struct vnode *a_vp; struct thread *a_td; } */ *ap; { vnode_t *vp = ap->a_vp; void *data = vp->v_data; sfs_reclaim_vnode(vp); sfs_destroy_node(data); return (0); } static int zfsctl_snapshot_vptocnp(struct vop_vptocnp_args *ap) { struct mount *mp; vnode_t *dvp; vnode_t *vp; sfs_node_t *node; size_t len; int locked; int error; vp = ap->a_vp; node = vp->v_data; len = strlen(node->sn_name); if (*ap->a_buflen < len) return (SET_ERROR(ENOMEM)); /* * Prevent unmounting of the snapshot while the vnode lock * is not held. That is not strictly required, but allows * us to assert that an uncovered snapshot vnode is never * "leaked". */ mp = vp->v_mountedhere; if (mp == NULL) return (SET_ERROR(ENOENT)); error = vfs_busy(mp, 0); KASSERT(error == 0, ("vfs_busy(mp, 0) failed with %d", error)); /* * We can vput the vnode as we can now depend on the reference owned * by the busied mp. But we also need to hold the vnode, because * the reference may go after vfs_unbusy() which has to be called * before we can lock the vnode again. */ locked = VOP_ISLOCKED(vp); vhold(vp); vput(vp); /* Look up .zfs/snapshot, our parent. */ error = zfsctl_snapdir_vnode(vp->v_mount, NULL, LK_SHARED, &dvp); if (error == 0) { VOP_UNLOCK(dvp, 0); *ap->a_vpp = dvp; *ap->a_buflen -= len; bcopy(node->sn_name, ap->a_buf + *ap->a_buflen, len); } vfs_unbusy(mp); vget(vp, locked | LK_VNHELD | LK_RETRY, curthread); return (error); } /* * These VP's should never see the light of day. They should always * be covered. */ static struct vop_vector zfsctl_ops_snapshot = { .vop_default = NULL, /* ensure very restricted access */ .vop_inactive = zfsctl_snapshot_inactive, .vop_reclaim = zfsctl_snapshot_reclaim, .vop_vptocnp = zfsctl_snapshot_vptocnp, .vop_lock1 = vop_stdlock, .vop_unlock = vop_stdunlock, .vop_islocked = vop_stdislocked, .vop_advlockpurge = vop_stdadvlockpurge, /* called by vgone */ .vop_print = zfsctl_common_print, }; int zfsctl_lookup_objset(vfs_t *vfsp, uint64_t objsetid, zfsvfs_t **zfsvfsp) { struct mount *mp; zfsvfs_t *zfsvfs = vfsp->vfs_data; vnode_t *vp; int error; ASSERT(zfsvfs->z_ctldir != NULL); *zfsvfsp = NULL; error = sfs_vnode_get(vfsp, LK_EXCLUSIVE, ZFSCTL_INO_SNAPDIR, objsetid, &vp); if (error == 0 && vp != NULL) { /* * XXX Probably need to at least reference, if not busy, the mp. */ if (vp->v_mountedhere != NULL) *zfsvfsp = vp->v_mountedhere->mnt_data; vput(vp); } if (*zfsvfsp == NULL) return (SET_ERROR(EINVAL)); return (0); } /* * Unmount any snapshots for the given filesystem. This is called from * zfs_umount() - if we have a ctldir, then go through and unmount all the * snapshots. */ int zfsctl_umount_snapshots(vfs_t *vfsp, int fflags, cred_t *cr) { char snapname[ZFS_MAX_DATASET_NAME_LEN]; zfsvfs_t *zfsvfs = vfsp->vfs_data; struct mount *mp; vnode_t *dvp; vnode_t *vp; sfs_node_t *node; sfs_node_t *snap; uint64_t cookie; int error; ASSERT(zfsvfs->z_ctldir != NULL); cookie = 0; for (;;) { uint64_t id; dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG); error = dmu_snapshot_list_next(zfsvfs->z_os, sizeof(snapname), snapname, &id, &cookie, NULL); dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG); if (error != 0) { if (error == ENOENT) error = 0; break; } for (;;) { error = sfs_vnode_get(vfsp, LK_EXCLUSIVE, ZFSCTL_INO_SNAPDIR, id, &vp); if (error != 0 || vp == NULL) break; mp = vp->v_mountedhere; /* * v_mountedhere being NULL means that the * (uncovered) vnode is in a transient state * (mounting or unmounting), so loop until it * settles down. */ if (mp != NULL) break; vput(vp); } if (error != 0) break; if (vp == NULL) continue; /* no mountpoint, nothing to do */ /* * The mount-point vnode is kept locked to avoid spurious EBUSY * from a concurrent umount. * The vnode lock must have recursive locking enabled. */ vfs_ref(mp); error = dounmount(mp, fflags, curthread); KASSERT_IMPLY(error == 0, vrefcnt(vp) == 1, ("extra references after unmount")); vput(vp); if (error != 0) break; } KASSERT_IMPLY((fflags & MS_FORCE) != 0, error == 0, ("force unmounting failed")); return (error); } Index: projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris =================================================================== --- projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris (revision 345025) +++ projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris (revision 345026) Property changes on: projects/import-googletest-1.8.1/sys/cddl/contrib/opensolaris ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/cddl/contrib/opensolaris:r344997-345025 Index: projects/import-googletest-1.8.1/sys/dev/isp/isp.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/isp/isp.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/isp/isp.c (revision 345026) @@ -1,8192 +1,8197 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2018 Alexander Motin * Copyright (c) 1997-2009 by Matthew Jacob * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Machine and OS Independent (well, as best as possible) * code for the Qlogic ISP SCSI and FC-SCSI adapters. */ /* * Inspiration and ideas about this driver are from Erik Moe's Linux driver * (qlogicisp.c) and Dave Miller's SBus version of same (qlogicisp.c). Some * ideas dredged from the Solaris driver. */ /* * Include header file appropriate for platform we're building on. */ #ifdef __NetBSD__ #include __KERNEL_RCSID(0, "$NetBSD$"); #include #endif #ifdef __FreeBSD__ #include __FBSDID("$FreeBSD$"); #include #endif #ifdef __OpenBSD__ #include #endif #ifdef __linux__ #include "isp_linux.h" #endif #ifdef __svr4__ #include "isp_solaris.h" #endif /* * General defines */ #define MBOX_DELAY_COUNT 1000000 / 100 /* * Local static data */ static const char notresp[] = "Unknown IOCB in RESPONSE Queue (type 0x%x) @ idx %d (next %d)"; static const char bun[] = "bad underrun (count %d, resid %d, status %s)"; static const char lipd[] = "Chan %d LIP destroyed %d active commands"; static const char sacq[] = "unable to acquire scratch area"; static const uint8_t alpa_map[] = { 0xef, 0xe8, 0xe4, 0xe2, 0xe1, 0xe0, 0xdc, 0xda, 0xd9, 0xd6, 0xd5, 0xd4, 0xd3, 0xd2, 0xd1, 0xce, 0xcd, 0xcc, 0xcb, 0xca, 0xc9, 0xc7, 0xc6, 0xc5, 0xc3, 0xbc, 0xba, 0xb9, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xae, 0xad, 0xac, 0xab, 0xaa, 0xa9, 0xa7, 0xa6, 0xa5, 0xa3, 0x9f, 0x9e, 0x9d, 0x9b, 0x98, 0x97, 0x90, 0x8f, 0x88, 0x84, 0x82, 0x81, 0x80, 0x7c, 0x7a, 0x79, 0x76, 0x75, 0x74, 0x73, 0x72, 0x71, 0x6e, 0x6d, 0x6c, 0x6b, 0x6a, 0x69, 0x67, 0x66, 0x65, 0x63, 0x5c, 0x5a, 0x59, 0x56, 0x55, 0x54, 0x53, 0x52, 0x51, 0x4e, 0x4d, 0x4c, 0x4b, 0x4a, 0x49, 0x47, 0x46, 0x45, 0x43, 0x3c, 0x3a, 0x39, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x2e, 0x2d, 0x2c, 0x2b, 0x2a, 0x29, 0x27, 0x26, 0x25, 0x23, 0x1f, 0x1e, 0x1d, 0x1b, 0x18, 0x17, 0x10, 0x0f, 0x08, 0x04, 0x02, 0x01, 0x00 }; /* * Local function prototypes. */ static void isp_parse_async(ispsoftc_t *, uint16_t); static void isp_parse_async_fc(ispsoftc_t *, uint16_t); static int isp_handle_other_response(ispsoftc_t *, int, isphdr_t *, uint32_t *); static void isp_parse_status(ispsoftc_t *, ispstatusreq_t *, XS_T *, uint32_t *); static void isp_parse_status_24xx(ispsoftc_t *, isp24xx_statusreq_t *, XS_T *, uint32_t *); static void isp_fastpost_complete(ispsoftc_t *, uint32_t); static void isp_scsi_init(ispsoftc_t *); static void isp_scsi_channel_init(ispsoftc_t *, int); static void isp_fibre_init(ispsoftc_t *); static void isp_fibre_init_2400(ispsoftc_t *); static void isp_clear_portdb(ispsoftc_t *, int); static void isp_mark_portdb(ispsoftc_t *, int); static int isp_plogx(ispsoftc_t *, int, uint16_t, uint32_t, int); static int isp_port_login(ispsoftc_t *, uint16_t, uint32_t); static int isp_port_logout(ispsoftc_t *, uint16_t, uint32_t); static int isp_getpdb(ispsoftc_t *, int, uint16_t, isp_pdb_t *); static int isp_gethandles(ispsoftc_t *, int, uint16_t *, int *, int); static void isp_dump_chip_portdb(ispsoftc_t *, int); static uint64_t isp_get_wwn(ispsoftc_t *, int, int, int); static int isp_fclink_test(ispsoftc_t *, int, int); static int isp_pdb_sync(ispsoftc_t *, int); static int isp_scan_loop(ispsoftc_t *, int); static int isp_gid_pt(ispsoftc_t *, int); static int isp_scan_fabric(ispsoftc_t *, int); static int isp_login_device(ispsoftc_t *, int, uint32_t, isp_pdb_t *, uint16_t *); static int isp_send_change_request(ispsoftc_t *, int); static int isp_register_fc4_type(ispsoftc_t *, int); static int isp_register_fc4_features_24xx(ispsoftc_t *, int); static int isp_register_port_name_24xx(ispsoftc_t *, int); static int isp_register_node_name_24xx(ispsoftc_t *, int); static uint16_t isp_next_handle(ispsoftc_t *, uint16_t *); static int isp_fw_state(ispsoftc_t *, int); static void isp_mboxcmd(ispsoftc_t *, mbreg_t *); static void isp_spi_update(ispsoftc_t *, int); static void isp_setdfltsdparm(ispsoftc_t *); static void isp_setdfltfcparm(ispsoftc_t *, int); static int isp_read_nvram(ispsoftc_t *, int); static int isp_read_nvram_2400(ispsoftc_t *, uint8_t *); static void isp_rdnvram_word(ispsoftc_t *, int, uint16_t *); static void isp_rd_2400_nvram(ispsoftc_t *, uint32_t, uint32_t *); static void isp_parse_nvram_1020(ispsoftc_t *, uint8_t *); static void isp_parse_nvram_1080(ispsoftc_t *, int, uint8_t *); static void isp_parse_nvram_12160(ispsoftc_t *, int, uint8_t *); static void isp_parse_nvram_2100(ispsoftc_t *, uint8_t *); static void isp_parse_nvram_2400(ispsoftc_t *, uint8_t *); static void isp_change_fw_state(ispsoftc_t *isp, int chan, int state) { fcparam *fcp = FCPARAM(isp, chan); if (fcp->isp_fwstate == state) return; isp_prt(isp, ISP_LOGCONFIG|ISP_LOG_SANCFG, "Chan %d Firmware state <%s->%s>", chan, isp_fc_fw_statename(fcp->isp_fwstate), isp_fc_fw_statename(state)); fcp->isp_fwstate = state; } /* * Reset Hardware. * * Hit the chip over the head, download new f/w if available and set it running. * * Locking done elsewhere. */ void isp_reset(ispsoftc_t *isp, int do_load_defaults) { mbreg_t mbs; char *buf; uint64_t fwt; uint32_t code_org, val; int loops, i, dodnld = 1; const char *btype = "????"; static const char dcrc[] = "Downloaded RISC Code Checksum Failure"; /* * Basic types (SCSI, FibreChannel and PCI or SBus) * have been set in the MD code. We figure out more * here. Possibly more refined types based upon PCI * identification. Chip revision has been gathered. * * After we've fired this chip up, zero out the conf1 register * for SCSI adapters and do other settings for the 2100. */ isp->isp_state = ISP_NILSTATE; ISP_DISABLE_INTS(isp); /* * Put the board into PAUSE mode (so we can read the SXP registers * or write FPM/FBM registers). */ if (IS_24XX(isp)) { ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_HOST_INT); ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT); ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_PAUSE); } else { ISP_WRITE(isp, HCCR, HCCR_CMD_PAUSE); } if (IS_FC(isp)) { switch (isp->isp_type) { case ISP_HA_FC_2100: btype = "2100"; break; case ISP_HA_FC_2200: btype = "2200"; break; case ISP_HA_FC_2300: btype = "2300"; break; case ISP_HA_FC_2312: btype = "2312"; break; case ISP_HA_FC_2322: btype = "2322"; break; case ISP_HA_FC_2400: btype = "2422"; break; case ISP_HA_FC_2500: btype = "2532"; break; case ISP_HA_FC_2600: btype = "2600"; break; case ISP_HA_FC_2700: btype = "2700"; break; default: break; } if (!IS_24XX(isp)) { /* * While we're paused, reset the FPM module and FBM * fifos. */ ISP_WRITE(isp, BIU2100_CSR, BIU2100_FPM0_REGS); ISP_WRITE(isp, FPM_DIAG_CONFIG, FPM_SOFT_RESET); ISP_WRITE(isp, BIU2100_CSR, BIU2100_FB_REGS); ISP_WRITE(isp, FBM_CMD, FBMCMD_FIFO_RESET_ALL); ISP_WRITE(isp, BIU2100_CSR, BIU2100_RISC_REGS); } } else if (IS_1240(isp)) { sdparam *sdp; btype = "1240"; isp->isp_clock = 60; sdp = SDPARAM(isp, 0); sdp->isp_ultramode = 1; sdp = SDPARAM(isp, 1); sdp->isp_ultramode = 1; /* * XXX: Should probably do some bus sensing. */ } else if (IS_ULTRA3(isp)) { sdparam *sdp = isp->isp_param; isp->isp_clock = 100; if (IS_10160(isp)) btype = "10160"; else if (IS_12160(isp)) btype = "12160"; else btype = ""; sdp->isp_lvdmode = 1; if (IS_DUALBUS(isp)) { sdp++; sdp->isp_lvdmode = 1; } } else if (IS_ULTRA2(isp)) { static const char m[] = "bus %d is in %s Mode"; uint16_t l; sdparam *sdp = SDPARAM(isp, 0); isp->isp_clock = 100; if (IS_1280(isp)) btype = "1280"; else if (IS_1080(isp)) btype = "1080"; else btype = ""; l = ISP_READ(isp, SXP_PINS_DIFF) & ISP1080_MODE_MASK; switch (l) { case ISP1080_LVD_MODE: sdp->isp_lvdmode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 0, "LVD"); break; case ISP1080_HVD_MODE: sdp->isp_diffmode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 0, "Differential"); break; case ISP1080_SE_MODE: sdp->isp_ultramode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 0, "Single-Ended"); break; default: isp_prt(isp, ISP_LOGERR, "unknown mode on bus %d (0x%x)", 0, l); break; } if (IS_DUALBUS(isp)) { sdp = SDPARAM(isp, 1); l = ISP_READ(isp, SXP_PINS_DIFF|SXP_BANK1_SELECT); l &= ISP1080_MODE_MASK; switch (l) { case ISP1080_LVD_MODE: sdp->isp_lvdmode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 1, "LVD"); break; case ISP1080_HVD_MODE: sdp->isp_diffmode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 1, "Differential"); break; case ISP1080_SE_MODE: sdp->isp_ultramode = 1; isp_prt(isp, ISP_LOGCONFIG, m, 1, "Single-Ended"); break; default: isp_prt(isp, ISP_LOGERR, "unknown mode on bus %d (0x%x)", 1, l); break; } } } else { sdparam *sdp = SDPARAM(isp, 0); i = ISP_READ(isp, BIU_CONF0) & BIU_CONF0_HW_MASK; switch (i) { default: isp_prt(isp, ISP_LOGALL, "Unknown Chip Type 0x%x", i); /* FALLTHROUGH */ case 1: btype = "1020"; isp->isp_type = ISP_HA_SCSI_1020; isp->isp_clock = 40; break; case 2: /* * Some 1020A chips are Ultra Capable, but don't * run the clock rate up for that unless told to * do so by the Ultra Capable bits being set. */ btype = "1020A"; isp->isp_type = ISP_HA_SCSI_1020A; isp->isp_clock = 40; break; case 3: btype = "1040"; isp->isp_type = ISP_HA_SCSI_1040; isp->isp_clock = 60; break; case 4: btype = "1040A"; isp->isp_type = ISP_HA_SCSI_1040A; isp->isp_clock = 60; break; case 5: btype = "1040B"; isp->isp_type = ISP_HA_SCSI_1040B; isp->isp_clock = 60; break; case 6: btype = "1040C"; isp->isp_type = ISP_HA_SCSI_1040C; isp->isp_clock = 60; break; } /* * Now, while we're at it, gather info about ultra * and/or differential mode. */ if (ISP_READ(isp, SXP_PINS_DIFF) & SXP_PINS_DIFF_MODE) { isp_prt(isp, ISP_LOGCONFIG, "Differential Mode"); sdp->isp_diffmode = 1; } else { sdp->isp_diffmode = 0; } i = ISP_READ(isp, RISC_PSR); if (isp->isp_bustype == ISP_BT_SBUS) { i &= RISC_PSR_SBUS_ULTRA; } else { i &= RISC_PSR_PCI_ULTRA; } if (i != 0) { isp_prt(isp, ISP_LOGCONFIG, "Ultra Mode Capable"); sdp->isp_ultramode = 1; /* * If we're in Ultra Mode, we have to be 60MHz clock- * even for the SBus version. */ isp->isp_clock = 60; } else { sdp->isp_ultramode = 0; /* * Clock is known. Gronk. */ } /* * Machine dependent clock (if set) overrides * our generic determinations. */ if (isp->isp_mdvec->dv_clock) { if (isp->isp_mdvec->dv_clock < isp->isp_clock) { isp->isp_clock = isp->isp_mdvec->dv_clock; } } } /* * Hit the chip over the head with hammer, * and give it a chance to recover. */ if (IS_SCSI(isp)) { ISP_WRITE(isp, BIU_ICR, BIU_ICR_SOFT_RESET); /* * A slight delay... */ ISP_DELAY(100); /* * Clear data && control DMA engines. */ ISP_WRITE(isp, CDMA_CONTROL, DMA_CNTRL_CLEAR_CHAN | DMA_CNTRL_RESET_INT); ISP_WRITE(isp, DDMA_CONTROL, DMA_CNTRL_CLEAR_CHAN | DMA_CNTRL_RESET_INT); } else if (IS_24XX(isp)) { /* * Stop DMA and wait for it to stop. */ ISP_WRITE(isp, BIU2400_CSR, BIU2400_DMA_STOP|(3 << 4)); for (val = loops = 0; loops < 30000; loops++) { ISP_DELAY(10); val = ISP_READ(isp, BIU2400_CSR); if ((val & BIU2400_DMA_ACTIVE) == 0) { break; } } if (val & BIU2400_DMA_ACTIVE) { isp_prt(isp, ISP_LOGERR, "DMA Failed to Stop on Reset"); return; } /* * Hold it in SOFT_RESET and STOP state for 100us. */ ISP_WRITE(isp, BIU2400_CSR, BIU2400_SOFT_RESET|BIU2400_DMA_STOP|(3 << 4)); ISP_DELAY(100); for (loops = 0; loops < 10000; loops++) { ISP_DELAY(5); val = ISP_READ(isp, OUTMAILBOX0); } for (val = loops = 0; loops < 500000; loops ++) { val = ISP_READ(isp, BIU2400_CSR); if ((val & BIU2400_SOFT_RESET) == 0) { break; } } if (val & BIU2400_SOFT_RESET) { isp_prt(isp, ISP_LOGERR, "Failed to come out of reset"); return; } } else { ISP_WRITE(isp, BIU2100_CSR, BIU2100_SOFT_RESET); /* * A slight delay... */ ISP_DELAY(100); /* * Clear data && control DMA engines. */ ISP_WRITE(isp, CDMA2100_CONTROL, DMA_CNTRL2100_CLEAR_CHAN | DMA_CNTRL2100_RESET_INT); ISP_WRITE(isp, TDMA2100_CONTROL, DMA_CNTRL2100_CLEAR_CHAN | DMA_CNTRL2100_RESET_INT); ISP_WRITE(isp, RDMA2100_CONTROL, DMA_CNTRL2100_CLEAR_CHAN | DMA_CNTRL2100_RESET_INT); } /* * Wait for ISP to be ready to go... */ loops = MBOX_DELAY_COUNT; for (;;) { if (IS_SCSI(isp)) { if (!(ISP_READ(isp, BIU_ICR) & BIU_ICR_SOFT_RESET)) { break; } } else if (IS_24XX(isp)) { if (ISP_READ(isp, OUTMAILBOX0) == 0) { break; } } else { if (!(ISP_READ(isp, BIU2100_CSR) & BIU2100_SOFT_RESET)) break; } ISP_DELAY(100); if (--loops < 0) { ISP_DUMPREGS(isp, "chip reset timed out"); return; } } /* * After we've fired this chip up, zero out the conf1 register * for SCSI adapters and other settings for the 2100. */ if (IS_SCSI(isp)) { ISP_WRITE(isp, BIU_CONF1, 0); } else if (!IS_24XX(isp)) { ISP_WRITE(isp, BIU2100_CSR, 0); } /* * Reset RISC Processor */ if (IS_24XX(isp)) { ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_RESET); ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_RELEASE); ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RESET); } else { ISP_WRITE(isp, HCCR, HCCR_CMD_RESET); ISP_DELAY(100); ISP_WRITE(isp, BIU_SEMA, 0); } /* * Post-RISC Reset stuff. */ if (IS_24XX(isp)) { for (val = loops = 0; loops < 5000000; loops++) { ISP_DELAY(5); val = ISP_READ(isp, OUTMAILBOX0); if (val == 0) { break; } } if (val != 0) { isp_prt(isp, ISP_LOGERR, "reset didn't clear"); return; } } else if (IS_SCSI(isp)) { uint16_t tmp = isp->isp_mdvec->dv_conf1; /* * Busted FIFO. Turn off all but burst enables. */ if (isp->isp_type == ISP_HA_SCSI_1040A) { tmp &= BIU_BURST_ENABLE; } ISP_SETBITS(isp, BIU_CONF1, tmp); if (tmp & BIU_BURST_ENABLE) { ISP_SETBITS(isp, CDMA_CONF, DMA_ENABLE_BURST); ISP_SETBITS(isp, DDMA_CONF, DMA_ENABLE_BURST); } if (SDPARAM(isp, 0)->isp_ptisp) { if (SDPARAM(isp, 0)->isp_ultramode) { while (ISP_READ(isp, RISC_MTR) != 0x1313) { ISP_WRITE(isp, RISC_MTR, 0x1313); ISP_WRITE(isp, HCCR, HCCR_CMD_STEP); } } else { ISP_WRITE(isp, RISC_MTR, 0x1212); } /* * PTI specific register */ ISP_WRITE(isp, RISC_EMB, DUAL_BANK); } else { ISP_WRITE(isp, RISC_MTR, 0x1212); } ISP_WRITE(isp, HCCR, HCCR_CMD_RELEASE); } else { ISP_WRITE(isp, RISC_MTR2100, 0x1212); if (IS_2200(isp) || IS_23XX(isp)) { ISP_WRITE(isp, HCCR, HCCR_2X00_DISABLE_PARITY_PAUSE); } ISP_WRITE(isp, HCCR, HCCR_CMD_RELEASE); } /* * Set up default request/response queue in-pointer/out-pointer * register indices. */ if (IS_24XX(isp)) { isp->isp_rqstinrp = BIU2400_REQINP; isp->isp_rqstoutrp = BIU2400_REQOUTP; isp->isp_respinrp = BIU2400_RSPINP; isp->isp_respoutrp = BIU2400_RSPOUTP; } else if (IS_23XX(isp)) { isp->isp_rqstinrp = BIU_REQINP; isp->isp_rqstoutrp = BIU_REQOUTP; isp->isp_respinrp = BIU_RSPINP; isp->isp_respoutrp = BIU_RSPOUTP; } else { isp->isp_rqstinrp = INMAILBOX4; isp->isp_rqstoutrp = OUTMAILBOX4; isp->isp_respinrp = OUTMAILBOX5; isp->isp_respoutrp = INMAILBOX5; } ISP_WRITE(isp, isp->isp_rqstinrp, 0); ISP_WRITE(isp, isp->isp_rqstoutrp, 0); ISP_WRITE(isp, isp->isp_respinrp, 0); ISP_WRITE(isp, isp->isp_respoutrp, 0); if (IS_24XX(isp)) { if (!IS_26XX(isp)) { ISP_WRITE(isp, BIU2400_PRI_REQINP, 0); ISP_WRITE(isp, BIU2400_PRI_REQOUTP, 0); } ISP_WRITE(isp, BIU2400_ATIO_RSPINP, 0); ISP_WRITE(isp, BIU2400_ATIO_RSPOUTP, 0); } if (!IS_24XX(isp) && isp->isp_bustype == ISP_BT_PCI) { /* Make sure the BIOS is disabled */ ISP_WRITE(isp, HCCR, PCI_HCCR_CMD_BIOS); } /* * Wait for everything to finish firing up. * * Avoid doing this on early 2312s because you can generate a PCI * parity error (chip breakage). */ if (IS_2312(isp) && isp->isp_revision < 2) { ISP_DELAY(100); } else { loops = MBOX_DELAY_COUNT; while (ISP_READ(isp, OUTMAILBOX0) == MBOX_BUSY) { ISP_DELAY(100); if (--loops < 0) { isp_prt(isp, ISP_LOGERR, "MBOX_BUSY never cleared on reset"); return; } } } /* * Up until this point we've done everything by just reading or * setting registers. From this point on we rely on at least *some* * kind of firmware running in the card. */ /* * Do some sanity checking by running a NOP command. * If it succeeds, the ROM firmware is now running. */ MBSINIT(&mbs, MBOX_NO_OP, MBLOGALL, 0); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "NOP command failed (%x)", mbs.param[0]); return; } /* * Do some operational tests */ if (IS_SCSI(isp) || IS_24XX(isp)) { static const uint16_t patterns[MAX_MAILBOX] = { 0x0000, 0xdead, 0xbeef, 0xffff, 0xa5a5, 0x5a5a, 0x7f7f, 0x7ff7, 0x3421, 0xabcd, 0xdcba, 0xfeef, 0xbead, 0xdebe, 0x2222, 0x3333, 0x5555, 0x6666, 0x7777, 0xaaaa, 0xffff, 0xdddd, 0x9999, 0x1fbc, 0x6666, 0x6677, 0x1122, 0x33ff, 0x0000, 0x0001, 0x1000, 0x1010, }; int nmbox = ISP_NMBOX(isp); if (IS_SCSI(isp)) nmbox = 6; MBSINIT(&mbs, MBOX_MAILBOX_REG_TEST, MBLOGALL, 0); for (i = 1; i < nmbox; i++) { mbs.param[i] = patterns[i]; } isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } for (i = 1; i < nmbox; i++) { if (mbs.param[i] != patterns[i]) { isp_prt(isp, ISP_LOGERR, "Register Test Failed at Register %d: should have 0x%04x but got 0x%04x", i, patterns[i], mbs.param[i]); return; } } } /* * Download new Firmware, unless requested not to do so. * This is made slightly trickier in some cases where the * firmware of the ROM revision is newer than the revision * compiled into the driver. So, where we used to compare * versions of our f/w and the ROM f/w, now we just see * whether we have f/w at all and whether a config flag * has disabled our download. */ if ((isp->isp_mdvec->dv_ispfw == NULL) || (isp->isp_confopts & ISP_CFG_NORELOAD)) { dodnld = 0; } else { /* * Set up DMA for the request and response queues. * We do this now so we can use the request queue * for dma to load firmware from. */ if (ISP_MBOXDMASETUP(isp) != 0) { isp_prt(isp, ISP_LOGERR, "Cannot setup DMA"); return; } } if (IS_24XX(isp)) { code_org = ISP_CODE_ORG_2400; } else if (IS_23XX(isp)) { code_org = ISP_CODE_ORG_2300; } else { code_org = ISP_CODE_ORG; } isp->isp_loaded_fw = 0; if (dodnld && IS_24XX(isp)) { const uint32_t *ptr = isp->isp_mdvec->dv_ispfw; uint32_t la, wi, wl; /* * Keep loading until we run out of f/w. */ code_org = ptr[2]; /* 1st load address is our start addr */ for (;;) { isp_prt(isp, ISP_LOGDEBUG0, "load 0x%x words of code at load address 0x%x", ptr[3], ptr[2]); wi = 0; la = ptr[2]; wl = ptr[3]; while (wi < ptr[3]) { uint32_t *cp; uint32_t nw; nw = min(wl, ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(isp)) / 4); cp = isp->isp_rquest; for (i = 0; i < nw; i++) ISP_IOXPUT_32(isp, ptr[wi + i], &cp[i]); MEMORYBARRIER(isp, SYNC_REQUEST, 0, ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(isp)), -1); MBSINIT(&mbs, MBOX_LOAD_RISC_RAM, MBLOGALL, 0); mbs.param[1] = la; mbs.param[2] = DMA_WD1(isp->isp_rquest_dma); mbs.param[3] = DMA_WD0(isp->isp_rquest_dma); mbs.param[4] = nw >> 16; mbs.param[5] = nw; mbs.param[6] = DMA_WD3(isp->isp_rquest_dma); mbs.param[7] = DMA_WD2(isp->isp_rquest_dma); mbs.param[8] = la >> 16; isp_prt(isp, ISP_LOGDEBUG0, "LOAD RISC RAM %u words at load address 0x%x", nw, la); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "F/W download failed"); return; } la += nw; wi += nw; wl -= nw; } if (ptr[1] == 0) { break; } ptr += ptr[3]; } isp->isp_loaded_fw = 1; } else if (dodnld && IS_23XX(isp)) { const uint16_t *ptr = isp->isp_mdvec->dv_ispfw; uint16_t wi, wl, segno; uint32_t la; la = code_org; segno = 0; for (;;) { uint32_t nxtaddr; isp_prt(isp, ISP_LOGDEBUG0, "load 0x%x words of code at load address 0x%x", ptr[3], la); wi = 0; wl = ptr[3]; while (wi < ptr[3]) { uint16_t *cp; uint16_t nw; nw = min(wl, min((1 << 15), ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(isp)) / 2)); cp = isp->isp_rquest; for (i = 0; i < nw; i++) ISP_IOXPUT_16(isp, ptr[wi + i], &cp[i]); MEMORYBARRIER(isp, SYNC_REQUEST, 0, ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(isp)), -1); MBSINIT(&mbs, 0, MBLOGALL, 0); if (la < 0x10000) { mbs.param[0] = MBOX_LOAD_RISC_RAM_2100; mbs.param[1] = la; mbs.param[2] = DMA_WD1(isp->isp_rquest_dma); mbs.param[3] = DMA_WD0(isp->isp_rquest_dma); mbs.param[4] = nw; mbs.param[6] = DMA_WD3(isp->isp_rquest_dma); mbs.param[7] = DMA_WD2(isp->isp_rquest_dma); isp_prt(isp, ISP_LOGDEBUG1, "LOAD RISC RAM 2100 %u words at load address 0x%x\n", nw, la); } else { mbs.param[0] = MBOX_LOAD_RISC_RAM; mbs.param[1] = la; mbs.param[2] = DMA_WD1(isp->isp_rquest_dma); mbs.param[3] = DMA_WD0(isp->isp_rquest_dma); mbs.param[4] = nw; mbs.param[6] = DMA_WD3(isp->isp_rquest_dma); mbs.param[7] = DMA_WD2(isp->isp_rquest_dma); mbs.param[8] = la >> 16; isp_prt(isp, ISP_LOGDEBUG1, "LOAD RISC RAM %u words at load address 0x%x\n", nw, la); } isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "F/W download failed"); return; } la += nw; wi += nw; wl -= nw; } if (!IS_2322(isp)) { break; } if (++segno == 3) { break; } /* * If we're a 2322, the firmware actually comes in * three chunks. We loaded the first at the code_org * address. The other two chunks, which follow right * after each other in memory here, get loaded at * addresses specfied at offset 0x9..0xB. */ nxtaddr = ptr[3]; ptr = &ptr[nxtaddr]; la = ptr[5] | ((ptr[4] & 0x3f) << 16); } isp->isp_loaded_fw = 1; } else if (dodnld) { const uint16_t *ptr = isp->isp_mdvec->dv_ispfw; u_int i, wl; wl = ptr[3]; isp_prt(isp, ISP_LOGDEBUG1, "WRITE RAM %u words at load address 0x%x", wl, code_org); for (i = 0; i < wl; i++) { MBSINIT(&mbs, MBOX_WRITE_RAM_WORD, MBLOGNONE, 0); mbs.param[1] = code_org + i; mbs.param[2] = ptr[i]; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "F/W download failed at word %d", i); return; } } } else if (IS_26XX(isp)) { isp_prt(isp, ISP_LOGDEBUG1, "loading firmware from flash"); MBSINIT(&mbs, MBOX_LOAD_FLASH_FIRMWARE, MBLOGALL, 5000000); mbs.ibitm = 0x01; mbs.obitm = 0x07; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "Flash F/W load failed"); return; } } else { isp_prt(isp, ISP_LOGDEBUG2, "skipping f/w download"); } /* * If we loaded firmware, verify its checksum */ if (isp->isp_loaded_fw) { MBSINIT(&mbs, MBOX_VERIFY_CHECKSUM, MBLOGNONE, 0); if (IS_24XX(isp)) { mbs.param[1] = code_org >> 16; mbs.param[2] = code_org; } else { mbs.param[1] = code_org; } isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, dcrc); return; } } /* * Now start it rolling. * * If we didn't actually download f/w, * we still need to (re)start it. */ MBSINIT(&mbs, MBOX_EXEC_FIRMWARE, MBLOGALL, 5000000); if (IS_26XX(isp)) { mbs.param[1] = code_org >> 16; mbs.param[2] = code_org; } else if (IS_24XX(isp)) { mbs.param[1] = code_org >> 16; mbs.param[2] = code_org; if (isp->isp_loaded_fw) { mbs.param[3] = 0; } else { mbs.param[3] = 1; } } else if (IS_2322(isp)) { mbs.param[1] = code_org; if (isp->isp_loaded_fw) { mbs.param[2] = 0; } else { mbs.param[2] = 1; } } else { mbs.param[1] = code_org; } isp_mboxcmd(isp, &mbs); if (IS_2322(isp) || IS_24XX(isp)) { if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } } if (IS_SCSI(isp)) { /* * Set CLOCK RATE, but only if asked to. */ if (isp->isp_clock) { MBSINIT(&mbs, MBOX_SET_CLOCK_RATE, MBLOGALL, 0); mbs.param[1] = isp->isp_clock; isp_mboxcmd(isp, &mbs); /* we will try not to care if this fails */ } } /* * Ask the chip for the current firmware version. * This should prove that the new firmware is working. */ MBSINIT(&mbs, MBOX_ABOUT_FIRMWARE, MBLOGALL, 5000000); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } /* * The SBus firmware that we are using apparently does not return * major, minor, micro revisions in the mailbox registers, which * is really, really, annoying. */ if (ISP_SBUS_SUPPORTED && isp->isp_bustype == ISP_BT_SBUS) { if (dodnld) { #ifdef ISP_TARGET_MODE isp->isp_fwrev[0] = 7; isp->isp_fwrev[1] = 55; #else isp->isp_fwrev[0] = 1; isp->isp_fwrev[1] = 37; #endif isp->isp_fwrev[2] = 0; } } else { isp->isp_fwrev[0] = mbs.param[1]; isp->isp_fwrev[1] = mbs.param[2]; isp->isp_fwrev[2] = mbs.param[3]; } if (IS_FC(isp)) { /* * We do not believe firmware attributes for 2100 code less * than 1.17.0, unless it's the firmware we specifically * are loading. * * Note that all 22XX and later f/w is greater than 1.X.0. */ if ((ISP_FW_OLDER_THAN(isp, 1, 17, 1))) { #ifdef USE_SMALLER_2100_FIRMWARE isp->isp_fwattr = ISP_FW_ATTR_SCCLUN; #else isp->isp_fwattr = 0; #endif } else { isp->isp_fwattr = mbs.param[6]; } if (IS_24XX(isp)) { isp->isp_fwattr |= ((uint64_t) mbs.param[15]) << 16; if (isp->isp_fwattr & ISP2400_FW_ATTR_EXTNDED) { isp->isp_fwattr |= (((uint64_t) mbs.param[16]) << 32) | (((uint64_t) mbs.param[17]) << 48); } } } else { isp->isp_fwattr = 0; } isp_prt(isp, ISP_LOGCONFIG, "Board Type %s, Chip Revision 0x%x, %s F/W Revision %d.%d.%d", btype, isp->isp_revision, dodnld? "loaded" : "resident", isp->isp_fwrev[0], isp->isp_fwrev[1], isp->isp_fwrev[2]); fwt = isp->isp_fwattr; if (IS_24XX(isp)) { buf = FCPARAM(isp, 0)->isp_scanscratch; ISP_SNPRINTF(buf, ISP_FC_SCRLEN, "Attributes:"); if (fwt & ISP2400_FW_ATTR_CLASS2) { fwt ^=ISP2400_FW_ATTR_CLASS2; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s Class2", buf); } if (fwt & ISP2400_FW_ATTR_IP) { fwt ^=ISP2400_FW_ATTR_IP; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s IP", buf); } if (fwt & ISP2400_FW_ATTR_MULTIID) { fwt ^=ISP2400_FW_ATTR_MULTIID; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s MultiID", buf); } if (fwt & ISP2400_FW_ATTR_SB2) { fwt ^=ISP2400_FW_ATTR_SB2; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s SB2", buf); } if (fwt & ISP2400_FW_ATTR_T10CRC) { fwt ^=ISP2400_FW_ATTR_T10CRC; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s T10CRC", buf); } if (fwt & ISP2400_FW_ATTR_VI) { fwt ^=ISP2400_FW_ATTR_VI; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s VI", buf); } if (fwt & ISP2400_FW_ATTR_MQ) { fwt ^=ISP2400_FW_ATTR_MQ; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s MQ", buf); } if (fwt & ISP2400_FW_ATTR_MSIX) { fwt ^=ISP2400_FW_ATTR_MSIX; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s MSIX", buf); } if (fwt & ISP2400_FW_ATTR_FCOE) { fwt ^=ISP2400_FW_ATTR_FCOE; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s FCOE", buf); } if (fwt & ISP2400_FW_ATTR_VP0) { fwt ^= ISP2400_FW_ATTR_VP0; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s VP0_Decoupling", buf); } if (fwt & ISP2400_FW_ATTR_EXPFW) { fwt ^= ISP2400_FW_ATTR_EXPFW; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s (Experimental)", buf); } if (fwt & ISP2400_FW_ATTR_HOTFW) { fwt ^= ISP2400_FW_ATTR_HOTFW; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s HotFW", buf); } fwt &= ~ISP2400_FW_ATTR_EXTNDED; if (fwt & ISP2400_FW_ATTR_EXTVP) { fwt ^= ISP2400_FW_ATTR_EXTVP; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s ExtVP", buf); } if (fwt & ISP2400_FW_ATTR_VN2VN) { fwt ^= ISP2400_FW_ATTR_VN2VN; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s VN2VN", buf); } if (fwt & ISP2400_FW_ATTR_EXMOFF) { fwt ^= ISP2400_FW_ATTR_EXMOFF; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s EXMOFF", buf); } if (fwt & ISP2400_FW_ATTR_NPMOFF) { fwt ^= ISP2400_FW_ATTR_NPMOFF; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s NPMOFF", buf); } if (fwt & ISP2400_FW_ATTR_DIFCHOP) { fwt ^= ISP2400_FW_ATTR_DIFCHOP; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s DIFCHOP", buf); } if (fwt & ISP2400_FW_ATTR_SRIOV) { fwt ^= ISP2400_FW_ATTR_SRIOV; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s SRIOV", buf); } if (fwt & ISP2400_FW_ATTR_ASICTMP) { fwt ^= ISP2400_FW_ATTR_ASICTMP; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s ASICTMP", buf); } if (fwt & ISP2400_FW_ATTR_ATIOMQ) { fwt ^= ISP2400_FW_ATTR_ATIOMQ; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s ATIOMQ", buf); } if (fwt) { ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s (unknown 0x%08x%08x)", buf, (uint32_t) (fwt >> 32), (uint32_t) fwt); } isp_prt(isp, ISP_LOGCONFIG, "%s", buf); } else if (IS_FC(isp)) { buf = FCPARAM(isp, 0)->isp_scanscratch; ISP_SNPRINTF(buf, ISP_FC_SCRLEN, "Attributes:"); if (fwt & ISP_FW_ATTR_TMODE) { fwt ^=ISP_FW_ATTR_TMODE; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s TargetMode", buf); } if (fwt & ISP_FW_ATTR_SCCLUN) { fwt ^=ISP_FW_ATTR_SCCLUN; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s SCC-Lun", buf); } if (fwt & ISP_FW_ATTR_FABRIC) { fwt ^=ISP_FW_ATTR_FABRIC; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s Fabric", buf); } if (fwt & ISP_FW_ATTR_CLASS2) { fwt ^=ISP_FW_ATTR_CLASS2; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s Class2", buf); } if (fwt & ISP_FW_ATTR_FCTAPE) { fwt ^=ISP_FW_ATTR_FCTAPE; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s FC-Tape", buf); } if (fwt & ISP_FW_ATTR_IP) { fwt ^=ISP_FW_ATTR_IP; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s IP", buf); } if (fwt & ISP_FW_ATTR_VI) { fwt ^=ISP_FW_ATTR_VI; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s VI", buf); } if (fwt & ISP_FW_ATTR_VI_SOLARIS) { fwt ^=ISP_FW_ATTR_VI_SOLARIS; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s VI_SOLARIS", buf); } if (fwt & ISP_FW_ATTR_2KLOGINS) { fwt ^=ISP_FW_ATTR_2KLOGINS; ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s 2K-Login", buf); } if (fwt != 0) { ISP_SNPRINTF(buf, ISP_FC_SCRLEN - strlen(buf), "%s (unknown 0x%08x%08x)", buf, (uint32_t) (fwt >> 32), (uint32_t) fwt); } isp_prt(isp, ISP_LOGCONFIG, "%s", buf); } if (IS_24XX(isp)) { MBSINIT(&mbs, MBOX_GET_RESOURCE_COUNT, MBLOGALL, 0); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_maxcmds = mbs.param[3]; } else { MBSINIT(&mbs, MBOX_GET_FIRMWARE_STATUS, MBLOGALL, 0); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_maxcmds = mbs.param[2]; } isp_prt(isp, ISP_LOGCONFIG, "%d max I/O command limit set", isp->isp_maxcmds); /* * If we don't have Multi-ID f/w loaded, we need to restrict channels to one. * Only make this check for non-SCSI cards (I'm not sure firmware attributes * work for them). */ if (IS_FC(isp) && isp->isp_nchan > 1) { if (!ISP_CAP_MULTI_ID(isp)) { isp_prt(isp, ISP_LOGWARN, "non-MULTIID f/w loaded, " "only can enable 1 of %d channels", isp->isp_nchan); isp->isp_nchan = 1; } else if (!ISP_CAP_VP0(isp)) { isp_prt(isp, ISP_LOGWARN, "We can not use MULTIID " "feature properly without VP0_Decoupling"); isp->isp_nchan = 1; } } /* * Final DMA setup after we got isp_maxcmds. */ if (ISP_MBOXDMASETUP(isp) != 0) { isp_prt(isp, ISP_LOGERR, "Cannot setup DMA"); return; } /* * Setup interrupts. */ if (ISP_IRQSETUP(isp) != 0) { isp_prt(isp, ISP_LOGERR, "Cannot setup IRQ"); return; } ISP_ENABLE_INTS(isp); if (IS_FC(isp)) { for (i = 0; i < isp->isp_nchan; i++) isp_change_fw_state(isp, i, FW_CONFIG_WAIT); } isp->isp_state = ISP_RESETSTATE; /* * Okay- now that we have new firmware running, we now (re)set our * notion of how many luns we support. This is somewhat tricky because * if we haven't loaded firmware, we sometimes do not have an easy way * of knowing how many luns we support. * * Expanded lun firmware gives you 32 luns for SCSI cards and * unlimited luns for Fibre Channel cards. * * It turns out that even for QLogic 2100s with ROM 1.10 and above * we do get a firmware attributes word returned in mailbox register 6. * * Because the lun is in a different position in the Request Queue * Entry structure for Fibre Channel with expanded lun firmware, we * can only support one lun (lun zero) when we don't know what kind * of firmware we're running. */ if (IS_SCSI(isp)) { if (dodnld) { if (IS_ULTRA2(isp) || IS_ULTRA3(isp)) { isp->isp_maxluns = 32; } else { isp->isp_maxluns = 8; } } else { isp->isp_maxluns = 8; } } else { if (ISP_CAP_SCCFW(isp)) { isp->isp_maxluns = 0; /* No limit -- 2/8 bytes */ } else { isp->isp_maxluns = 16; } } /* * We get some default values established. As a side * effect, NVRAM is read here (unless overriden by * a configuration flag). */ if (do_load_defaults) { if (IS_SCSI(isp)) { isp_setdfltsdparm(isp); } else { for (i = 0; i < isp->isp_nchan; i++) { isp_setdfltfcparm(isp, i); } } } } /* * Clean firmware shutdown. */ static int isp_stop(ispsoftc_t *isp) { mbreg_t mbs; isp->isp_state = ISP_NILSTATE; MBSINIT(&mbs, MBOX_STOP_FIRMWARE, MBLOGALL, 500000); mbs.param[1] = 0; mbs.param[2] = 0; mbs.param[3] = 0; mbs.param[4] = 0; mbs.param[5] = 0; mbs.param[6] = 0; mbs.param[7] = 0; mbs.param[8] = 0; isp_mboxcmd(isp, &mbs); return (mbs.param[0] == MBOX_COMMAND_COMPLETE ? 0 : mbs.param[0]); } /* * Hardware shutdown. */ void isp_shutdown(ispsoftc_t *isp) { if (isp->isp_state >= ISP_RESETSTATE) isp_stop(isp); ISP_DISABLE_INTS(isp); if (IS_FC(isp)) { if (IS_24XX(isp)) { ISP_WRITE(isp, BIU2400_ICR, 0); ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_PAUSE); } else { ISP_WRITE(isp, BIU_ICR, 0); ISP_WRITE(isp, HCCR, HCCR_CMD_PAUSE); ISP_WRITE(isp, BIU2100_CSR, BIU2100_FPM0_REGS); ISP_WRITE(isp, FPM_DIAG_CONFIG, FPM_SOFT_RESET); ISP_WRITE(isp, BIU2100_CSR, BIU2100_FB_REGS); ISP_WRITE(isp, FBM_CMD, FBMCMD_FIFO_RESET_ALL); ISP_WRITE(isp, BIU2100_CSR, BIU2100_RISC_REGS); } } else { ISP_WRITE(isp, BIU_ICR, 0); ISP_WRITE(isp, HCCR, HCCR_CMD_PAUSE); } } /* * Initialize Parameters of Hardware to a known state. * * Locks are held before coming here. */ void isp_init(ispsoftc_t *isp) { if (IS_FC(isp)) { if (IS_24XX(isp)) { isp_fibre_init_2400(isp); } else { isp_fibre_init(isp); } } else { isp_scsi_init(isp); } } static void isp_scsi_init(ispsoftc_t *isp) { sdparam *sdp_chan0, *sdp_chan1; mbreg_t mbs; isp->isp_state = ISP_INITSTATE; sdp_chan0 = SDPARAM(isp, 0); sdp_chan1 = sdp_chan0; if (IS_DUALBUS(isp)) { sdp_chan1 = SDPARAM(isp, 1); } /* First do overall per-card settings. */ /* * If we have fast memory timing enabled, turn it on. */ if (sdp_chan0->isp_fast_mttr) { ISP_WRITE(isp, RISC_MTR, 0x1313); } /* * Set Retry Delay and Count. * You set both channels at the same time. */ MBSINIT(&mbs, MBOX_SET_RETRY_COUNT, MBLOGALL, 0); mbs.param[1] = sdp_chan0->isp_retry_count; mbs.param[2] = sdp_chan0->isp_retry_delay; mbs.param[6] = sdp_chan1->isp_retry_count; mbs.param[7] = sdp_chan1->isp_retry_delay; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } /* * Set ASYNC DATA SETUP time. This is very important. */ MBSINIT(&mbs, MBOX_SET_ASYNC_DATA_SETUP_TIME, MBLOGALL, 0); mbs.param[1] = sdp_chan0->isp_async_data_setup; mbs.param[2] = sdp_chan1->isp_async_data_setup; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } /* * Set ACTIVE Negation State. */ MBSINIT(&mbs, MBOX_SET_ACT_NEG_STATE, MBLOGNONE, 0); mbs.param[1] = (sdp_chan0->isp_req_ack_active_neg << 4) | (sdp_chan0->isp_data_line_active_neg << 5); mbs.param[2] = (sdp_chan1->isp_req_ack_active_neg << 4) | (sdp_chan1->isp_data_line_active_neg << 5); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "failed to set active negation state (%d,%d), (%d,%d)", sdp_chan0->isp_req_ack_active_neg, sdp_chan0->isp_data_line_active_neg, sdp_chan1->isp_req_ack_active_neg, sdp_chan1->isp_data_line_active_neg); /* * But don't return. */ } /* * Set the Tag Aging limit */ MBSINIT(&mbs, MBOX_SET_TAG_AGE_LIMIT, MBLOGALL, 0); mbs.param[1] = sdp_chan0->isp_tag_aging; mbs.param[2] = sdp_chan1->isp_tag_aging; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGERR, "failed to set tag age limit (%d,%d)", sdp_chan0->isp_tag_aging, sdp_chan1->isp_tag_aging); return; } /* * Set selection timeout. */ MBSINIT(&mbs, MBOX_SET_SELECT_TIMEOUT, MBLOGALL, 0); mbs.param[1] = sdp_chan0->isp_selection_timeout; mbs.param[2] = sdp_chan1->isp_selection_timeout; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } /* now do per-channel settings */ isp_scsi_channel_init(isp, 0); if (IS_DUALBUS(isp)) isp_scsi_channel_init(isp, 1); /* * Now enable request/response queues */ if (IS_ULTRA2(isp) || IS_1240(isp)) { MBSINIT(&mbs, MBOX_INIT_RES_QUEUE_A64, MBLOGALL, 0); mbs.param[1] = RESULT_QUEUE_LEN(isp); mbs.param[2] = DMA_WD1(isp->isp_result_dma); mbs.param[3] = DMA_WD0(isp->isp_result_dma); mbs.param[4] = 0; mbs.param[6] = DMA_WD3(isp->isp_result_dma); mbs.param[7] = DMA_WD2(isp->isp_result_dma); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_residx = isp->isp_resodx = mbs.param[5]; MBSINIT(&mbs, MBOX_INIT_REQ_QUEUE_A64, MBLOGALL, 0); mbs.param[1] = RQUEST_QUEUE_LEN(isp); mbs.param[2] = DMA_WD1(isp->isp_rquest_dma); mbs.param[3] = DMA_WD0(isp->isp_rquest_dma); mbs.param[5] = 0; mbs.param[6] = DMA_WD3(isp->isp_result_dma); mbs.param[7] = DMA_WD2(isp->isp_result_dma); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_reqidx = isp->isp_reqodx = mbs.param[4]; } else { MBSINIT(&mbs, MBOX_INIT_RES_QUEUE, MBLOGALL, 0); mbs.param[1] = RESULT_QUEUE_LEN(isp); mbs.param[2] = DMA_WD1(isp->isp_result_dma); mbs.param[3] = DMA_WD0(isp->isp_result_dma); mbs.param[4] = 0; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_residx = isp->isp_resodx = mbs.param[5]; MBSINIT(&mbs, MBOX_INIT_REQ_QUEUE, MBLOGALL, 0); mbs.param[1] = RQUEST_QUEUE_LEN(isp); mbs.param[2] = DMA_WD1(isp->isp_rquest_dma); mbs.param[3] = DMA_WD0(isp->isp_rquest_dma); mbs.param[5] = 0; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_reqidx = isp->isp_reqodx = mbs.param[4]; } /* * Turn on LVD transitions for ULTRA2 or better and other features * * Now that we have 32 bit handles, don't do any fast posting * any more. For Ultra2/Ultra3 cards, we can turn on 32 bit RIO * operation or use fast posting. To be conservative, we'll only * do this for Ultra3 cards now because the other cards are so * rare for this author to find and test with. */ MBSINIT(&mbs, MBOX_SET_FW_FEATURES, MBLOGALL, 0); if (IS_ULTRA2(isp)) mbs.param[1] |= FW_FEATURE_LVD_NOTIFY; #ifdef ISP_NO_RIO if (IS_ULTRA3(isp)) mbs.param[1] |= FW_FEATURE_FAST_POST; #else if (IS_ULTRA3(isp)) mbs.param[1] |= FW_FEATURE_RIO_32BIT; #endif if (mbs.param[1] != 0) { uint16_t sfeat = mbs.param[1]; isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { isp_prt(isp, ISP_LOGINFO, "Enabled FW features (0x%x)", sfeat); } } isp->isp_state = ISP_RUNSTATE; } static void isp_scsi_channel_init(ispsoftc_t *isp, int chan) { sdparam *sdp; mbreg_t mbs; int tgt; sdp = SDPARAM(isp, chan); /* * Set (possibly new) Initiator ID. */ MBSINIT(&mbs, MBOX_SET_INIT_SCSI_ID, MBLOGALL, 0); mbs.param[1] = (chan << 7) | sdp->isp_initiator_id; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp_prt(isp, ISP_LOGINFO, "Chan %d Initiator ID is %d", chan, sdp->isp_initiator_id); /* * Set current per-target parameters to an initial safe minimum. */ for (tgt = 0; tgt < MAX_TARGETS; tgt++) { int lun; uint16_t sdf; if (sdp->isp_devparam[tgt].dev_enable == 0) { continue; } #ifndef ISP_TARGET_MODE sdf = sdp->isp_devparam[tgt].goal_flags; sdf &= DPARM_SAFE_DFLT; /* * It is not quite clear when this changed over so that * we could force narrow and async for 1000/1020 cards, * but assume that this is only the case for loaded * firmware. */ if (isp->isp_loaded_fw) { sdf |= DPARM_NARROW | DPARM_ASYNC; } #else /* * The !$*!)$!$)* f/w uses the same index into some * internal table to decide how to respond to negotiations, * so if we've said "let's be safe" for ID X, and ID X * selects *us*, the negotiations will back to 'safe' * (as in narrow/async). What the f/w *should* do is * use the initiator id settings to decide how to respond. */ sdp->isp_devparam[tgt].goal_flags = sdf = DPARM_DEFAULT; #endif MBSINIT(&mbs, MBOX_SET_TARGET_PARAMS, MBLOGNONE, 0); mbs.param[1] = (chan << 15) | (tgt << 8); mbs.param[2] = sdf; if ((sdf & DPARM_SYNC) == 0) { mbs.param[3] = 0; } else { mbs.param[3] = (sdp->isp_devparam[tgt].goal_offset << 8) | (sdp->isp_devparam[tgt].goal_period); } isp_prt(isp, ISP_LOGDEBUG0, "Initial Settings bus%d tgt%d flags 0x%x off 0x%x per 0x%x", chan, tgt, mbs.param[2], mbs.param[3] >> 8, mbs.param[3] & 0xff); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { sdf = DPARM_SAFE_DFLT; MBSINIT(&mbs, MBOX_SET_TARGET_PARAMS, MBLOGALL, 0); mbs.param[1] = (tgt << 8) | (chan << 15); mbs.param[2] = sdf; mbs.param[3] = 0; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { continue; } } /* * We don't update any information directly from the f/w * because we need to run at least one command to cause a * new state to be latched up. So, we just assume that we * converge to the values we just had set. * * Ensure that we don't believe tagged queuing is enabled yet. * It turns out that sometimes the ISP just ignores our * attempts to set parameters for devices that it hasn't * seen yet. */ sdp->isp_devparam[tgt].actv_flags = sdf & ~DPARM_TQING; for (lun = 0; lun < (int) isp->isp_maxluns; lun++) { MBSINIT(&mbs, MBOX_SET_DEV_QUEUE_PARAMS, MBLOGALL, 0); mbs.param[1] = (chan << 15) | (tgt << 8) | lun; mbs.param[2] = sdp->isp_max_queue_depth; mbs.param[3] = sdp->isp_devparam[tgt].exc_throttle; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { break; } } } for (tgt = 0; tgt < MAX_TARGETS; tgt++) { if (sdp->isp_devparam[tgt].dev_refresh) { sdp->sendmarker = 1; sdp->update = 1; break; } } } /* * Fibre Channel specific initialization. */ static void isp_fibre_init(ispsoftc_t *isp) { fcparam *fcp; isp_icb_t local, *icbp = &local; mbreg_t mbs; int ownloopid; /* * We only support one channel on non-24XX cards */ fcp = FCPARAM(isp, 0); if (fcp->role == ISP_ROLE_NONE) return; isp->isp_state = ISP_INITSTATE; ISP_MEMZERO(icbp, sizeof (*icbp)); icbp->icb_version = ICB_VERSION1; icbp->icb_fwoptions = fcp->isp_fwoptions; /* * Firmware Options are either retrieved from NVRAM or * are patched elsewhere. We check them for sanity here * and make changes based on board revision, but otherwise * let others decide policy. */ /* * If this is a 2100 < revision 5, we have to turn off FAIRNESS. */ if (IS_2100(isp) && isp->isp_revision < 5) { icbp->icb_fwoptions &= ~ICBOPT_FAIRNESS; } /* * We have to use FULL LOGIN even though it resets the loop too much * because otherwise port database entries don't get updated after * a LIP- this is a known f/w bug for 2100 f/w less than 1.17.0. */ if (!ISP_FW_NEWER_THAN(isp, 1, 17, 0)) { icbp->icb_fwoptions |= ICBOPT_FULL_LOGIN; } /* * Insist on Port Database Update Async notifications */ icbp->icb_fwoptions |= ICBOPT_PDBCHANGE_AE; /* * Make sure that target role reflects into fwoptions. */ if (fcp->role & ISP_ROLE_TARGET) { icbp->icb_fwoptions |= ICBOPT_TGT_ENABLE; } else { icbp->icb_fwoptions &= ~ICBOPT_TGT_ENABLE; } /* * For some reason my 2200 does not generate ATIOs in target mode * if initiator is disabled. Extra logins are better then target * not working at all. */ if ((fcp->role & ISP_ROLE_INITIATOR) || IS_2100(isp) || IS_2200(isp)) { icbp->icb_fwoptions &= ~ICBOPT_INI_DISABLE; } else { icbp->icb_fwoptions |= ICBOPT_INI_DISABLE; } icbp->icb_maxfrmlen = DEFAULT_FRAMESIZE(isp); if (icbp->icb_maxfrmlen < ICB_MIN_FRMLEN || icbp->icb_maxfrmlen > ICB_MAX_FRMLEN) { isp_prt(isp, ISP_LOGERR, "bad frame length (%d) from NVRAM- using %d", DEFAULT_FRAMESIZE(isp), ICB_DFLT_FRMLEN); icbp->icb_maxfrmlen = ICB_DFLT_FRMLEN; } icbp->icb_maxalloc = fcp->isp_maxalloc; if (icbp->icb_maxalloc < 1) { isp_prt(isp, ISP_LOGERR, "bad maximum allocation (%d)- using 16", fcp->isp_maxalloc); icbp->icb_maxalloc = 16; } icbp->icb_execthrottle = DEFAULT_EXEC_THROTTLE(isp); if (icbp->icb_execthrottle < 1) { isp_prt(isp, ISP_LOGERR, "bad execution throttle of %d- using %d", DEFAULT_EXEC_THROTTLE(isp), ICB_DFLT_THROTTLE); icbp->icb_execthrottle = ICB_DFLT_THROTTLE; } icbp->icb_retry_delay = fcp->isp_retry_delay; icbp->icb_retry_count = fcp->isp_retry_count; icbp->icb_hardaddr = fcp->isp_loopid; ownloopid = (isp->isp_confopts & ISP_CFG_OWNLOOPID) != 0; if (icbp->icb_hardaddr >= LOCAL_LOOP_LIM) { icbp->icb_hardaddr = 0; ownloopid = 0; } /* * Our life seems so much better with 2200s and later with * the latest f/w if we set Hard Address. */ if (ownloopid || ISP_FW_NEWER_THAN(isp, 2, 2, 5)) { icbp->icb_fwoptions |= ICBOPT_HARD_ADDRESS; } /* * Right now we just set extended options to prefer point-to-point * over loop based upon some soft config options. * * NB: for the 2300, ICBOPT_EXTENDED is required. */ if (IS_2100(isp)) { /* * We can't have Fast Posting any more- we now * have 32 bit handles. */ icbp->icb_fwoptions &= ~ICBOPT_FAST_POST; } else if (IS_2200(isp) || IS_23XX(isp)) { icbp->icb_fwoptions |= ICBOPT_EXTENDED; icbp->icb_xfwoptions = fcp->isp_xfwoptions; if (ISP_CAP_FCTAPE(isp)) { if (isp->isp_confopts & ISP_CFG_NOFCTAPE) icbp->icb_xfwoptions &= ~ICBXOPT_FCTAPE; if (isp->isp_confopts & ISP_CFG_FCTAPE) icbp->icb_xfwoptions |= ICBXOPT_FCTAPE; if (icbp->icb_xfwoptions & ICBXOPT_FCTAPE) { icbp->icb_fwoptions &= ~ICBOPT_FULL_LOGIN; /* per documents */ icbp->icb_xfwoptions |= ICBXOPT_FCTAPE_CCQ|ICBXOPT_FCTAPE_CONFIRM; FCPARAM(isp, 0)->fctape_enabled = 1; } else { FCPARAM(isp, 0)->fctape_enabled = 0; } } else { icbp->icb_xfwoptions &= ~ICBXOPT_FCTAPE; FCPARAM(isp, 0)->fctape_enabled = 0; } /* * Prefer or force Point-To-Point instead Loop? */ switch (isp->isp_confopts & ISP_CFG_PORT_PREF) { case ISP_CFG_LPORT_ONLY: icbp->icb_xfwoptions &= ~ICBXOPT_TOPO_MASK; icbp->icb_xfwoptions |= ICBXOPT_LOOP_ONLY; break; case ISP_CFG_NPORT_ONLY: icbp->icb_xfwoptions &= ~ICBXOPT_TOPO_MASK; icbp->icb_xfwoptions |= ICBXOPT_PTP_ONLY; break; case ISP_CFG_LPORT: icbp->icb_xfwoptions &= ~ICBXOPT_TOPO_MASK; icbp->icb_xfwoptions |= ICBXOPT_LOOP_2_PTP; break; case ISP_CFG_NPORT: icbp->icb_xfwoptions &= ~ICBXOPT_TOPO_MASK; icbp->icb_xfwoptions |= ICBXOPT_PTP_2_LOOP; break; default: /* Let NVRAM settings define it if they are sane */ switch (icbp->icb_xfwoptions & ICBXOPT_TOPO_MASK) { case ICBXOPT_PTP_2_LOOP: case ICBXOPT_PTP_ONLY: case ICBXOPT_LOOP_ONLY: case ICBXOPT_LOOP_2_PTP: break; default: icbp->icb_xfwoptions &= ~ICBXOPT_TOPO_MASK; icbp->icb_xfwoptions |= ICBXOPT_LOOP_2_PTP; } break; } if (IS_2200(isp)) { /* * We can't have Fast Posting any more- we now * have 32 bit handles. * * RIO seemed to have to much breakage. * * Just opt for safety. */ icbp->icb_xfwoptions &= ~ICBXOPT_RIO_16BIT; icbp->icb_fwoptions &= ~ICBOPT_FAST_POST; } else { /* * QLogic recommends that FAST Posting be turned * off for 23XX cards and instead allow the HBA * to write response queue entries and interrupt * after a delay (ZIO). */ icbp->icb_fwoptions &= ~ICBOPT_FAST_POST; if ((fcp->isp_xfwoptions & ICBXOPT_TIMER_MASK) == ICBXOPT_ZIO) { icbp->icb_xfwoptions |= ICBXOPT_ZIO; icbp->icb_idelaytimer = 10; } icbp->icb_zfwoptions = fcp->isp_zfwoptions; if (isp->isp_confopts & ISP_CFG_1GB) { icbp->icb_zfwoptions &= ~ICBZOPT_RATE_MASK; icbp->icb_zfwoptions |= ICBZOPT_RATE_1GB; } else if (isp->isp_confopts & ISP_CFG_2GB) { icbp->icb_zfwoptions &= ~ICBZOPT_RATE_MASK; icbp->icb_zfwoptions |= ICBZOPT_RATE_2GB; } else { switch (icbp->icb_zfwoptions & ICBZOPT_RATE_MASK) { case ICBZOPT_RATE_1GB: case ICBZOPT_RATE_2GB: case ICBZOPT_RATE_AUTO: break; default: icbp->icb_zfwoptions &= ~ICBZOPT_RATE_MASK; icbp->icb_zfwoptions |= ICBZOPT_RATE_AUTO; break; } } } } /* * For 22XX > 2.1.26 && 23XX, set some options. */ if (ISP_FW_NEWER_THAN(isp, 2, 26, 0)) { MBSINIT(&mbs, MBOX_SET_FIRMWARE_OPTIONS, MBLOGALL, 0); mbs.param[1] = IFCOPT1_DISF7SWTCH|IFCOPT1_LIPASYNC|IFCOPT1_LIPF8; mbs.param[2] = 0; mbs.param[3] = 0; if (ISP_FW_NEWER_THAN(isp, 3, 16, 0)) { mbs.param[1] |= IFCOPT1_EQFQASYNC|IFCOPT1_CTIO_RETRY; if (fcp->role & ISP_ROLE_TARGET) { if (ISP_FW_NEWER_THAN(isp, 3, 25, 0)) { mbs.param[1] |= IFCOPT1_ENAPURE; } mbs.param[3] = IFCOPT3_NOPRLI; } } isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } } icbp->icb_logintime = ICB_LOGIN_TOV; #ifdef ISP_TARGET_MODE if (icbp->icb_fwoptions & ICBOPT_TGT_ENABLE) { icbp->icb_lunenables = 0xffff; icbp->icb_ccnt = 0xff; icbp->icb_icnt = 0xff; icbp->icb_lunetimeout = ICB_LUN_ENABLE_TOV; } #endif if (fcp->isp_wwnn && fcp->isp_wwpn) { icbp->icb_fwoptions |= ICBOPT_BOTH_WWNS; MAKE_NODE_NAME_FROM_WWN(icbp->icb_nodename, fcp->isp_wwnn); MAKE_NODE_NAME_FROM_WWN(icbp->icb_portname, fcp->isp_wwpn); isp_prt(isp, ISP_LOGDEBUG1, "Setting ICB Node 0x%08x%08x Port 0x%08x%08x", ((uint32_t) (fcp->isp_wwnn >> 32)), ((uint32_t) (fcp->isp_wwnn)), ((uint32_t) (fcp->isp_wwpn >> 32)), ((uint32_t) (fcp->isp_wwpn))); } else if (fcp->isp_wwpn) { icbp->icb_fwoptions &= ~ICBOPT_BOTH_WWNS; MAKE_NODE_NAME_FROM_WWN(icbp->icb_portname, fcp->isp_wwpn); isp_prt(isp, ISP_LOGDEBUG1, "Setting ICB Port 0x%08x%08x", ((uint32_t) (fcp->isp_wwpn >> 32)), ((uint32_t) (fcp->isp_wwpn))); } else { isp_prt(isp, ISP_LOGERR, "No valid WWNs to use"); return; } icbp->icb_rqstqlen = RQUEST_QUEUE_LEN(isp); if (icbp->icb_rqstqlen < 1) { isp_prt(isp, ISP_LOGERR, "bad request queue length"); } icbp->icb_rsltqlen = RESULT_QUEUE_LEN(isp); if (icbp->icb_rsltqlen < 1) { isp_prt(isp, ISP_LOGERR, "bad result queue length"); } icbp->icb_rqstaddr[RQRSP_ADDR0015] = DMA_WD0(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR1631] = DMA_WD1(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR3247] = DMA_WD2(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR4863] = DMA_WD3(isp->isp_rquest_dma); icbp->icb_respaddr[RQRSP_ADDR0015] = DMA_WD0(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR1631] = DMA_WD1(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR3247] = DMA_WD2(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR4863] = DMA_WD3(isp->isp_result_dma); if (FC_SCRATCH_ACQUIRE(isp, 0)) { isp_prt(isp, ISP_LOGERR, sacq); return; } isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init: fwopt 0x%x xfwopt 0x%x zfwopt 0x%x", icbp->icb_fwoptions, icbp->icb_xfwoptions, icbp->icb_zfwoptions); isp_put_icb(isp, icbp, (isp_icb_t *)fcp->isp_scratch); if (isp->isp_dblev & ISP_LOGDEBUG1) { isp_print_bytes(isp, "isp_fibre_init", sizeof(*icbp), fcp->isp_scratch); } /* * Init the firmware */ MBSINIT(&mbs, MBOX_INIT_FIRMWARE, MBLOGALL, 30000000); mbs.param[1] = 0; mbs.param[2] = DMA_WD1(fcp->isp_scdma); mbs.param[3] = DMA_WD0(fcp->isp_scdma); mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); isp_prt(isp, ISP_LOGDEBUG0, "INIT F/W from %p (%08x%08x)", fcp->isp_scratch, (uint32_t) ((uint64_t)fcp->isp_scdma >> 32), (uint32_t) fcp->isp_scdma); MEMORYBARRIER(isp, SYNC_SFORDEV, 0, sizeof (*icbp), 0); isp_mboxcmd(isp, &mbs); FC_SCRATCH_RELEASE(isp, 0); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) return; isp->isp_reqidx = 0; isp->isp_reqodx = 0; isp->isp_residx = 0; isp->isp_resodx = 0; /* * Whatever happens, we're now committed to being here. */ isp->isp_state = ISP_RUNSTATE; } static void isp_fibre_init_2400(ispsoftc_t *isp) { fcparam *fcp; isp_icb_2400_t local, *icbp = &local; mbreg_t mbs; int chan; int ownloopid = 0; /* * Check to see whether all channels have *some* kind of role */ for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role != ISP_ROLE_NONE) { break; } } if (chan == isp->isp_nchan) { isp_prt(isp, ISP_LOG_WARN1, "all %d channels with role 'none'", chan); return; } isp->isp_state = ISP_INITSTATE; /* * Start with channel 0. */ fcp = FCPARAM(isp, 0); /* * Turn on LIP F8 async event (1) */ MBSINIT(&mbs, MBOX_SET_FIRMWARE_OPTIONS, MBLOGALL, 0); mbs.param[1] = 1; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } ISP_MEMZERO(icbp, sizeof (*icbp)); icbp->icb_fwoptions1 = fcp->isp_fwoptions; icbp->icb_fwoptions2 = fcp->isp_xfwoptions; icbp->icb_fwoptions3 = fcp->isp_zfwoptions; if (isp->isp_nchan > 1 && ISP_CAP_VP0(isp)) { icbp->icb_fwoptions1 &= ~ICB2400_OPT1_INI_DISABLE; icbp->icb_fwoptions1 |= ICB2400_OPT1_TGT_ENABLE; } else { if (fcp->role & ISP_ROLE_TARGET) icbp->icb_fwoptions1 |= ICB2400_OPT1_TGT_ENABLE; else icbp->icb_fwoptions1 &= ~ICB2400_OPT1_TGT_ENABLE; if (fcp->role & ISP_ROLE_INITIATOR) icbp->icb_fwoptions1 &= ~ICB2400_OPT1_INI_DISABLE; else icbp->icb_fwoptions1 |= ICB2400_OPT1_INI_DISABLE; } icbp->icb_version = ICB_VERSION1; icbp->icb_maxfrmlen = DEFAULT_FRAMESIZE(isp); if (icbp->icb_maxfrmlen < ICB_MIN_FRMLEN || icbp->icb_maxfrmlen > ICB_MAX_FRMLEN) { isp_prt(isp, ISP_LOGERR, "bad frame length (%d) from NVRAM- using %d", DEFAULT_FRAMESIZE(isp), ICB_DFLT_FRMLEN); icbp->icb_maxfrmlen = ICB_DFLT_FRMLEN; } icbp->icb_execthrottle = DEFAULT_EXEC_THROTTLE(isp); if (icbp->icb_execthrottle < 1 && !IS_26XX(isp)) { isp_prt(isp, ISP_LOGERR, "bad execution throttle of %d- using %d", DEFAULT_EXEC_THROTTLE(isp), ICB_DFLT_THROTTLE); icbp->icb_execthrottle = ICB_DFLT_THROTTLE; } /* * Set target exchange count. Take half if we are supporting both roles. */ if (icbp->icb_fwoptions1 & ICB2400_OPT1_TGT_ENABLE) { icbp->icb_xchgcnt = isp->isp_maxcmds; if ((icbp->icb_fwoptions1 & ICB2400_OPT1_INI_DISABLE) == 0) icbp->icb_xchgcnt >>= 1; } ownloopid = (isp->isp_confopts & ISP_CFG_OWNLOOPID) != 0; icbp->icb_hardaddr = fcp->isp_loopid; if (icbp->icb_hardaddr >= LOCAL_LOOP_LIM) { icbp->icb_hardaddr = 0; ownloopid = 0; } if (ownloopid) icbp->icb_fwoptions1 |= ICB2400_OPT1_HARD_ADDRESS; if (isp->isp_confopts & ISP_CFG_NOFCTAPE) { icbp->icb_fwoptions2 &= ~ICB2400_OPT2_FCTAPE; } if (isp->isp_confopts & ISP_CFG_FCTAPE) { icbp->icb_fwoptions2 |= ICB2400_OPT2_FCTAPE; } for (chan = 0; chan < isp->isp_nchan; chan++) { if (icbp->icb_fwoptions2 & ICB2400_OPT2_FCTAPE) FCPARAM(isp, chan)->fctape_enabled = 1; else FCPARAM(isp, chan)->fctape_enabled = 0; } switch (isp->isp_confopts & ISP_CFG_PORT_PREF) { case ISP_CFG_LPORT_ONLY: icbp->icb_fwoptions2 &= ~ICB2400_OPT2_TOPO_MASK; icbp->icb_fwoptions2 |= ICB2400_OPT2_LOOP_ONLY; break; case ISP_CFG_NPORT_ONLY: icbp->icb_fwoptions2 &= ~ICB2400_OPT2_TOPO_MASK; icbp->icb_fwoptions2 |= ICB2400_OPT2_PTP_ONLY; break; case ISP_CFG_NPORT: /* ISP_CFG_PTP_2_LOOP not available in 24XX/25XX */ case ISP_CFG_LPORT: icbp->icb_fwoptions2 &= ~ICB2400_OPT2_TOPO_MASK; icbp->icb_fwoptions2 |= ICB2400_OPT2_LOOP_2_PTP; break; default: /* Let NVRAM settings define it if they are sane */ switch (icbp->icb_fwoptions2 & ICB2400_OPT2_TOPO_MASK) { case ICB2400_OPT2_LOOP_ONLY: case ICB2400_OPT2_PTP_ONLY: case ICB2400_OPT2_LOOP_2_PTP: break; default: icbp->icb_fwoptions2 &= ~ICB2400_OPT2_TOPO_MASK; icbp->icb_fwoptions2 |= ICB2400_OPT2_LOOP_2_PTP; } break; } switch (icbp->icb_fwoptions2 & ICB2400_OPT2_TIMER_MASK) { case ICB2400_OPT2_ZIO: case ICB2400_OPT2_ZIO1: icbp->icb_idelaytimer = 0; break; case 0: break; default: isp_prt(isp, ISP_LOGWARN, "bad value %x in fwopt2 timer field", icbp->icb_fwoptions2 & ICB2400_OPT2_TIMER_MASK); icbp->icb_fwoptions2 &= ~ICB2400_OPT2_TIMER_MASK; break; } if (IS_26XX(isp)) { /* Use handshake to reduce global lock congestion. */ icbp->icb_fwoptions2 |= ICB2400_OPT2_ENA_IHR; icbp->icb_fwoptions2 |= ICB2400_OPT2_ENA_IHA; } if ((icbp->icb_fwoptions3 & ICB2400_OPT3_RSPSZ_MASK) == 0) { icbp->icb_fwoptions3 |= ICB2400_OPT3_RSPSZ_24; } if (isp->isp_confopts & ISP_CFG_1GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_1GB; } else if (isp->isp_confopts & ISP_CFG_2GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_2GB; } else if (isp->isp_confopts & ISP_CFG_4GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_4GB; } else if (isp->isp_confopts & ISP_CFG_8GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_8GB; } else if (isp->isp_confopts & ISP_CFG_16GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_16GB; } else if (isp->isp_confopts & ISP_CFG_32GB) { icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_32GB; } else { switch (icbp->icb_fwoptions3 & ICB2400_OPT3_RATE_MASK) { case ICB2400_OPT3_RATE_4GB: case ICB2400_OPT3_RATE_8GB: case ICB2400_OPT3_RATE_16GB: case ICB2400_OPT3_RATE_32GB: case ICB2400_OPT3_RATE_AUTO: break; case ICB2400_OPT3_RATE_2GB: if (isp->isp_type <= ISP_HA_FC_2500) break; /*FALLTHROUGH*/ case ICB2400_OPT3_RATE_1GB: if (isp->isp_type <= ISP_HA_FC_2400) break; /*FALLTHROUGH*/ default: icbp->icb_fwoptions3 &= ~ICB2400_OPT3_RATE_MASK; icbp->icb_fwoptions3 |= ICB2400_OPT3_RATE_AUTO; break; } } if (ownloopid == 0) { icbp->icb_fwoptions3 |= ICB2400_OPT3_SOFTID; } icbp->icb_logintime = ICB_LOGIN_TOV; if (fcp->isp_wwnn && fcp->isp_wwpn) { icbp->icb_fwoptions1 |= ICB2400_OPT1_BOTH_WWNS; MAKE_NODE_NAME_FROM_WWN(icbp->icb_portname, fcp->isp_wwpn); MAKE_NODE_NAME_FROM_WWN(icbp->icb_nodename, fcp->isp_wwnn); isp_prt(isp, ISP_LOGDEBUG1, "Setting ICB Node 0x%08x%08x Port 0x%08x%08x", ((uint32_t) (fcp->isp_wwnn >> 32)), ((uint32_t) (fcp->isp_wwnn)), ((uint32_t) (fcp->isp_wwpn >> 32)), ((uint32_t) (fcp->isp_wwpn))); } else if (fcp->isp_wwpn) { icbp->icb_fwoptions1 &= ~ICB2400_OPT1_BOTH_WWNS; MAKE_NODE_NAME_FROM_WWN(icbp->icb_portname, fcp->isp_wwpn); isp_prt(isp, ISP_LOGDEBUG1, "Setting ICB Node to be same as Port 0x%08x%08x", ((uint32_t) (fcp->isp_wwpn >> 32)), ((uint32_t) (fcp->isp_wwpn))); } else { isp_prt(isp, ISP_LOGERR, "No valid WWNs to use"); return; } icbp->icb_retry_count = fcp->isp_retry_count; icbp->icb_rqstqlen = RQUEST_QUEUE_LEN(isp); if (icbp->icb_rqstqlen < 8) { isp_prt(isp, ISP_LOGERR, "bad request queue length %d", icbp->icb_rqstqlen); return; } icbp->icb_rsltqlen = RESULT_QUEUE_LEN(isp); if (icbp->icb_rsltqlen < 8) { isp_prt(isp, ISP_LOGERR, "bad result queue length %d", icbp->icb_rsltqlen); return; } icbp->icb_rqstaddr[RQRSP_ADDR0015] = DMA_WD0(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR1631] = DMA_WD1(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR3247] = DMA_WD2(isp->isp_rquest_dma); icbp->icb_rqstaddr[RQRSP_ADDR4863] = DMA_WD3(isp->isp_rquest_dma); icbp->icb_respaddr[RQRSP_ADDR0015] = DMA_WD0(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR1631] = DMA_WD1(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR3247] = DMA_WD2(isp->isp_result_dma); icbp->icb_respaddr[RQRSP_ADDR4863] = DMA_WD3(isp->isp_result_dma); #ifdef ISP_TARGET_MODE /* unconditionally set up the ATIO queue if we support target mode */ icbp->icb_atioqlen = RESULT_QUEUE_LEN(isp); if (icbp->icb_atioqlen < 8) { isp_prt(isp, ISP_LOGERR, "bad ATIO queue length %d", icbp->icb_atioqlen); return; } icbp->icb_atioqaddr[RQRSP_ADDR0015] = DMA_WD0(isp->isp_atioq_dma); icbp->icb_atioqaddr[RQRSP_ADDR1631] = DMA_WD1(isp->isp_atioq_dma); icbp->icb_atioqaddr[RQRSP_ADDR3247] = DMA_WD2(isp->isp_atioq_dma); icbp->icb_atioqaddr[RQRSP_ADDR4863] = DMA_WD3(isp->isp_atioq_dma); isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init_2400: atioq %04x%04x%04x%04x", DMA_WD3(isp->isp_atioq_dma), DMA_WD2(isp->isp_atioq_dma), DMA_WD1(isp->isp_atioq_dma), DMA_WD0(isp->isp_atioq_dma)); #endif if (ISP_CAP_MSIX(isp) && isp->isp_nirq >= 2) { icbp->icb_msixresp = 1; if (IS_26XX(isp) && isp->isp_nirq >= 3) icbp->icb_msixatio = 2; } isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init_2400: fwopt1 0x%x fwopt2 0x%x fwopt3 0x%x", icbp->icb_fwoptions1, icbp->icb_fwoptions2, icbp->icb_fwoptions3); isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init_2400: rqst %04x%04x%04x%04x rsp %04x%04x%04x%04x", DMA_WD3(isp->isp_rquest_dma), DMA_WD2(isp->isp_rquest_dma), DMA_WD1(isp->isp_rquest_dma), DMA_WD0(isp->isp_rquest_dma), DMA_WD3(isp->isp_result_dma), DMA_WD2(isp->isp_result_dma), DMA_WD1(isp->isp_result_dma), DMA_WD0(isp->isp_result_dma)); if (FC_SCRATCH_ACQUIRE(isp, 0)) { isp_prt(isp, ISP_LOGERR, sacq); return; } ISP_MEMZERO(fcp->isp_scratch, ISP_FC_SCRLEN); isp_put_icb_2400(isp, icbp, fcp->isp_scratch); if (isp->isp_dblev & ISP_LOGDEBUG1) { isp_print_bytes(isp, "isp_fibre_init_2400", sizeof (*icbp), fcp->isp_scratch); } /* * Now fill in information about any additional channels */ if (isp->isp_nchan > 1) { isp_icb_2400_vpinfo_t vpinfo, *vdst; vp_port_info_t pi, *pdst; size_t amt = 0; uint8_t *off; vpinfo.vp_global_options = ICB2400_VPGOPT_GEN_RIDA; if (ISP_CAP_VP0(isp)) { vpinfo.vp_global_options |= ICB2400_VPGOPT_VP0_DECOUPLE; vpinfo.vp_count = isp->isp_nchan; chan = 0; } else { vpinfo.vp_count = isp->isp_nchan - 1; chan = 1; } off = fcp->isp_scratch; off += ICB2400_VPINFO_OFF; vdst = (isp_icb_2400_vpinfo_t *) off; isp_put_icb_2400_vpinfo(isp, &vpinfo, vdst); amt = ICB2400_VPINFO_OFF + sizeof (isp_icb_2400_vpinfo_t); for (; chan < isp->isp_nchan; chan++) { fcparam *fcp2; ISP_MEMZERO(&pi, sizeof (pi)); fcp2 = FCPARAM(isp, chan); if (fcp2->role != ISP_ROLE_NONE) { pi.vp_port_options = ICB2400_VPOPT_ENABLED | ICB2400_VPOPT_ENA_SNSLOGIN; if (fcp2->role & ISP_ROLE_INITIATOR) pi.vp_port_options |= ICB2400_VPOPT_INI_ENABLE; if ((fcp2->role & ISP_ROLE_TARGET) == 0) pi.vp_port_options |= ICB2400_VPOPT_TGT_DISABLE; if (fcp2->isp_loopid < LOCAL_LOOP_LIM) { pi.vp_port_loopid = fcp2->isp_loopid; if (isp->isp_confopts & ISP_CFG_OWNLOOPID) pi.vp_port_options |= ICB2400_VPOPT_HARD_ADDRESS; } } MAKE_NODE_NAME_FROM_WWN(pi.vp_port_portname, fcp2->isp_wwpn); MAKE_NODE_NAME_FROM_WWN(pi.vp_port_nodename, fcp2->isp_wwnn); off = fcp->isp_scratch; if (ISP_CAP_VP0(isp)) off += ICB2400_VPINFO_PORT_OFF(chan); else off += ICB2400_VPINFO_PORT_OFF(chan - 1); pdst = (vp_port_info_t *) off; isp_put_vp_port_info(isp, &pi, pdst); amt += ICB2400_VPOPT_WRITE_SIZE; } if (isp->isp_dblev & ISP_LOGDEBUG1) { isp_print_bytes(isp, "isp_fibre_init_2400", amt - ICB2400_VPINFO_OFF, (char *)fcp->isp_scratch + ICB2400_VPINFO_OFF); } } /* * Init the firmware */ MBSINIT(&mbs, 0, MBLOGALL, 30000000); if (isp->isp_nchan > 1) { mbs.param[0] = MBOX_INIT_FIRMWARE_MULTI_ID; } else { mbs.param[0] = MBOX_INIT_FIRMWARE; } mbs.param[1] = 0; mbs.param[2] = DMA_WD1(fcp->isp_scdma); mbs.param[3] = DMA_WD0(fcp->isp_scdma); mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); isp_prt(isp, ISP_LOGDEBUG0, "INIT F/W from %04x%04x%04x%04x", DMA_WD3(fcp->isp_scdma), DMA_WD2(fcp->isp_scdma), DMA_WD1(fcp->isp_scdma), DMA_WD0(fcp->isp_scdma)); MEMORYBARRIER(isp, SYNC_SFORDEV, 0, sizeof (*icbp), 0); isp_mboxcmd(isp, &mbs); FC_SCRATCH_RELEASE(isp, 0); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return; } isp->isp_reqidx = 0; isp->isp_reqodx = 0; isp->isp_residx = 0; isp->isp_resodx = 0; isp->isp_atioodx = 0; /* * Whatever happens, we're now committed to being here. */ isp->isp_state = ISP_RUNSTATE; } static int isp_fc_enable_vp(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); vp_modify_t vp; void *reqp; uint8_t resp[QENTRY_LEN]; /* Build a VP MODIFY command in memory */ ISP_MEMZERO(&vp, sizeof(vp)); vp.vp_mod_hdr.rqs_entry_type = RQSTYPE_VP_MODIFY; vp.vp_mod_hdr.rqs_entry_count = 1; vp.vp_mod_cnt = 1; vp.vp_mod_idx0 = chan; vp.vp_mod_cmd = VP_MODIFY_ENA; vp.vp_mod_ports[0].options = ICB2400_VPOPT_ENABLED | ICB2400_VPOPT_ENA_SNSLOGIN; if (fcp->role & ISP_ROLE_INITIATOR) vp.vp_mod_ports[0].options |= ICB2400_VPOPT_INI_ENABLE; if ((fcp->role & ISP_ROLE_TARGET) == 0) vp.vp_mod_ports[0].options |= ICB2400_VPOPT_TGT_DISABLE; if (fcp->isp_loopid < LOCAL_LOOP_LIM) { vp.vp_mod_ports[0].loopid = fcp->isp_loopid; if (isp->isp_confopts & ISP_CFG_OWNLOOPID) vp.vp_mod_ports[0].options |= ICB2400_VPOPT_HARD_ADDRESS; } MAKE_NODE_NAME_FROM_WWN(vp.vp_mod_ports[0].wwpn, fcp->isp_wwpn); MAKE_NODE_NAME_FROM_WWN(vp.vp_mod_ports[0].wwnn, fcp->isp_wwnn); /* Prepare space for response in memory */ memset(resp, 0xff, sizeof(resp)); vp.vp_mod_hdl = isp_allocate_handle(isp, resp, ISP_HANDLE_CTRL); if (vp.vp_mod_hdl == 0) { isp_prt(isp, ISP_LOGERR, "%s: VP_MODIFY of Chan %d out of handles", __func__, chan); return (EIO); } /* Send request and wait for response. */ reqp = isp_getrqentry(isp); if (reqp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: VP_MODIFY of Chan %d out of rqent", __func__, chan); isp_destroy_handle(isp, vp.vp_mod_hdl); return (EIO); } isp_put_vp_modify(isp, &vp, (vp_modify_t *)reqp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB VP_MODIFY", QENTRY_LEN, reqp); ISP_SYNC_REQUEST(isp); if (msleep(resp, &isp->isp_lock, 0, "VP_MODIFY", 5*hz) == EWOULDBLOCK) { isp_prt(isp, ISP_LOGERR, "%s: VP_MODIFY of Chan %d timed out", __func__, chan); isp_destroy_handle(isp, vp.vp_mod_hdl); return (EIO); } if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB VP_MODIFY response", QENTRY_LEN, resp); isp_get_vp_modify(isp, (vp_modify_t *)resp, &vp); if (vp.vp_mod_hdr.rqs_flags != 0 || vp.vp_mod_status != VP_STS_OK) { isp_prt(isp, ISP_LOGERR, "%s: VP_MODIFY of Chan %d failed with flags %x status %d", __func__, chan, vp.vp_mod_hdr.rqs_flags, vp.vp_mod_status); return (EIO); } return (0); } static int isp_fc_disable_vp(ispsoftc_t *isp, int chan) { vp_ctrl_info_t vp; void *reqp; uint8_t resp[QENTRY_LEN]; /* Build a VP CTRL command in memory */ ISP_MEMZERO(&vp, sizeof(vp)); vp.vp_ctrl_hdr.rqs_entry_type = RQSTYPE_VP_CTRL; vp.vp_ctrl_hdr.rqs_entry_count = 1; if (ISP_CAP_VP0(isp)) { vp.vp_ctrl_status = 1; } else { vp.vp_ctrl_status = 0; chan--; /* VP0 can not be controlled in this case. */ } vp.vp_ctrl_command = VP_CTRL_CMD_DISABLE_VP_LOGO_ALL; vp.vp_ctrl_vp_count = 1; vp.vp_ctrl_idmap[chan / 16] |= (1 << chan % 16); /* Prepare space for response in memory */ memset(resp, 0xff, sizeof(resp)); vp.vp_ctrl_handle = isp_allocate_handle(isp, resp, ISP_HANDLE_CTRL); if (vp.vp_ctrl_handle == 0) { isp_prt(isp, ISP_LOGERR, "%s: VP_CTRL of Chan %d out of handles", __func__, chan); return (EIO); } /* Send request and wait for response. */ reqp = isp_getrqentry(isp); if (reqp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: VP_CTRL of Chan %d out of rqent", __func__, chan); isp_destroy_handle(isp, vp.vp_ctrl_handle); return (EIO); } isp_put_vp_ctrl_info(isp, &vp, (vp_ctrl_info_t *)reqp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB VP_CTRL", QENTRY_LEN, reqp); ISP_SYNC_REQUEST(isp); if (msleep(resp, &isp->isp_lock, 0, "VP_CTRL", 5*hz) == EWOULDBLOCK) { isp_prt(isp, ISP_LOGERR, "%s: VP_CTRL of Chan %d timed out", __func__, chan); isp_destroy_handle(isp, vp.vp_ctrl_handle); return (EIO); } if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB VP_CTRL response", QENTRY_LEN, resp); isp_get_vp_ctrl_info(isp, (vp_ctrl_info_t *)resp, &vp); if (vp.vp_ctrl_hdr.rqs_flags != 0 || vp.vp_ctrl_status != 0) { isp_prt(isp, ISP_LOGERR, "%s: VP_CTRL of Chan %d failed with flags %x status %d %d", __func__, chan, vp.vp_ctrl_hdr.rqs_flags, vp.vp_ctrl_status, vp.vp_ctrl_index_fail); return (EIO); } return (0); } static int isp_fc_change_role(ispsoftc_t *isp, int chan, int new_role) { fcparam *fcp = FCPARAM(isp, chan); int i, was, res = 0; if (chan >= isp->isp_nchan) { isp_prt(isp, ISP_LOGWARN, "%s: bad channel %d", __func__, chan); return (ENXIO); } if (fcp->role == new_role) return (0); for (was = 0, i = 0; i < isp->isp_nchan; i++) { if (FCPARAM(isp, i)->role != ISP_ROLE_NONE) was++; } if (was == 0 || (was == 1 && fcp->role != ISP_ROLE_NONE)) { fcp->role = new_role; return (isp_reinit(isp, 0)); } if (fcp->role != ISP_ROLE_NONE) { res = isp_fc_disable_vp(isp, chan); isp_clear_portdb(isp, chan); } fcp->role = new_role; if (fcp->role != ISP_ROLE_NONE) res = isp_fc_enable_vp(isp, chan); return (res); } static void isp_clear_portdb(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; int i; for (i = 0; i < MAX_FC_TARG; i++) { lp = &fcp->portdb[i]; switch (lp->state) { case FC_PORTDB_STATE_DEAD: case FC_PORTDB_STATE_CHANGED: case FC_PORTDB_STATE_VALID: lp->state = FC_PORTDB_STATE_NIL; isp_async(isp, ISPASYNC_DEV_GONE, chan, lp); break; case FC_PORTDB_STATE_NIL: case FC_PORTDB_STATE_NEW: lp->state = FC_PORTDB_STATE_NIL; break; case FC_PORTDB_STATE_ZOMBIE: break; default: panic("Don't know how to clear state %d\n", lp->state); } } } static void isp_mark_portdb(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; int i; for (i = 0; i < MAX_FC_TARG; i++) { lp = &fcp->portdb[i]; if (lp->state == FC_PORTDB_STATE_NIL) continue; if (lp->portid >= DOMAIN_CONTROLLER_BASE && lp->portid <= DOMAIN_CONTROLLER_END) continue; fcp->portdb[i].probational = 1; } } /* * Perform an IOCB PLOGI or LOGO via EXECUTE IOCB A64 for 24XX cards * or via FABRIC LOGIN/FABRIC LOGOUT for other cards. */ static int isp_plogx(ispsoftc_t *isp, int chan, uint16_t handle, uint32_t portid, int flags) { isp_plogx_t pl; void *reqp; uint8_t resp[QENTRY_LEN]; uint32_t sst, parm1; int rval, lev; const char *msg; char buf[64]; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d PLOGX %s PortID 0x%06x nphdl 0x%x", chan, (flags & PLOGX_FLG_CMD_MASK) == PLOGX_FLG_CMD_PLOGI ? "Login":"Logout", portid, handle); if (!IS_24XX(isp)) { int action = flags & PLOGX_FLG_CMD_MASK; if (action == PLOGX_FLG_CMD_PLOGI) { return (isp_port_login(isp, handle, portid)); } else if (action == PLOGX_FLG_CMD_LOGO) { return (isp_port_logout(isp, handle, portid)); } else { return (MBOX_INVALID_COMMAND); } } ISP_MEMZERO(&pl, sizeof(pl)); pl.plogx_header.rqs_entry_count = 1; pl.plogx_header.rqs_entry_type = RQSTYPE_LOGIN; pl.plogx_nphdl = handle; pl.plogx_vphdl = chan; pl.plogx_portlo = portid; pl.plogx_rspsz_porthi = (portid >> 16) & 0xff; pl.plogx_flags = flags; /* Prepare space for response in memory */ memset(resp, 0xff, sizeof(resp)); pl.plogx_handle = isp_allocate_handle(isp, resp, ISP_HANDLE_CTRL); if (pl.plogx_handle == 0) { isp_prt(isp, ISP_LOGERR, "%s: PLOGX of Chan %d out of handles", __func__, chan); return (-1); } /* Send request and wait for response. */ reqp = isp_getrqentry(isp); if (reqp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: PLOGX of Chan %d out of rqent", __func__, chan); isp_destroy_handle(isp, pl.plogx_handle); return (-1); } isp_put_plogx(isp, &pl, (isp_plogx_t *)reqp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB LOGX", QENTRY_LEN, reqp); FCPARAM(isp, chan)->isp_login_hdl = handle; ISP_SYNC_REQUEST(isp); if (msleep(resp, &isp->isp_lock, 0, "PLOGX", 3 * ICB_LOGIN_TOV * hz) == EWOULDBLOCK) { isp_prt(isp, ISP_LOGERR, "%s: PLOGX of Chan %d timed out", __func__, chan); isp_destroy_handle(isp, pl.plogx_handle); return (-1); } FCPARAM(isp, chan)->isp_login_hdl = NIL_HANDLE; if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB LOGX response", QENTRY_LEN, resp); isp_get_plogx(isp, (isp_plogx_t *)resp, &pl); if (pl.plogx_status == PLOGX_STATUS_OK) { return (0); } else if (pl.plogx_status != PLOGX_STATUS_IOCBERR) { isp_prt(isp, ISP_LOGWARN, "status 0x%x on port login IOCB channel %d", pl.plogx_status, chan); return (-1); } sst = pl.plogx_ioparm[0].lo16 | (pl.plogx_ioparm[0].hi16 << 16); parm1 = pl.plogx_ioparm[1].lo16 | (pl.plogx_ioparm[1].hi16 << 16); rval = -1; lev = ISP_LOGERR; msg = NULL; switch (sst) { case PLOGX_IOCBERR_NOLINK: msg = "no link"; break; case PLOGX_IOCBERR_NOIOCB: msg = "no IOCB buffer"; break; case PLOGX_IOCBERR_NOXGHG: msg = "no Exchange Control Block"; break; case PLOGX_IOCBERR_FAILED: ISP_SNPRINTF(buf, sizeof (buf), "reason 0x%x (last LOGIN state 0x%x)", parm1 & 0xff, (parm1 >> 8) & 0xff); msg = buf; break; case PLOGX_IOCBERR_NOFABRIC: msg = "no fabric"; break; case PLOGX_IOCBERR_NOTREADY: msg = "firmware not ready"; break; case PLOGX_IOCBERR_NOLOGIN: ISP_SNPRINTF(buf, sizeof (buf), "not logged in (last state 0x%x)", parm1); msg = buf; rval = MBOX_NOT_LOGGED_IN; break; case PLOGX_IOCBERR_REJECT: ISP_SNPRINTF(buf, sizeof (buf), "LS_RJT = 0x%x", parm1); msg = buf; break; case PLOGX_IOCBERR_NOPCB: msg = "no PCB allocated"; break; case PLOGX_IOCBERR_EINVAL: ISP_SNPRINTF(buf, sizeof (buf), "invalid parameter at offset 0x%x", parm1); msg = buf; break; case PLOGX_IOCBERR_PORTUSED: lev = ISP_LOG_SANCFG|ISP_LOG_WARN1; ISP_SNPRINTF(buf, sizeof (buf), "already logged in with N-Port handle 0x%x", parm1); msg = buf; rval = MBOX_PORT_ID_USED | (parm1 << 16); break; case PLOGX_IOCBERR_HNDLUSED: lev = ISP_LOG_SANCFG|ISP_LOG_WARN1; ISP_SNPRINTF(buf, sizeof (buf), "handle already used for PortID 0x%06x", parm1); msg = buf; rval = MBOX_LOOP_ID_USED; break; case PLOGX_IOCBERR_NOHANDLE: msg = "no handle allocated"; break; case PLOGX_IOCBERR_NOFLOGI: msg = "no FLOGI_ACC"; break; default: ISP_SNPRINTF(buf, sizeof (buf), "status %x from %x", pl.plogx_status, flags); msg = buf; break; } if (msg) { isp_prt(isp, lev, "Chan %d PLOGX PortID 0x%06x to N-Port handle 0x%x: %s", chan, portid, handle, msg); } return (rval); } static int isp_port_login(ispsoftc_t *isp, uint16_t handle, uint32_t portid) { mbreg_t mbs; MBSINIT(&mbs, MBOX_FABRIC_LOGIN, MBLOGNONE, 500000); if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = handle; mbs.ibits = (1 << 10); } else { mbs.param[1] = handle << 8; } mbs.param[2] = portid >> 16; mbs.param[3] = portid; mbs.logval = MBLOGNONE; mbs.timeout = 500000; isp_mboxcmd(isp, &mbs); switch (mbs.param[0]) { case MBOX_PORT_ID_USED: isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "isp_port_login: portid 0x%06x already logged in as 0x%x", portid, mbs.param[1]); return (MBOX_PORT_ID_USED | (mbs.param[1] << 16)); case MBOX_LOOP_ID_USED: isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "isp_port_login: handle 0x%x in use for port id 0x%02xXXXX", handle, mbs.param[1] & 0xff); return (MBOX_LOOP_ID_USED); case MBOX_COMMAND_COMPLETE: return (0); case MBOX_COMMAND_ERROR: isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "isp_port_login: error 0x%x in PLOGI to port 0x%06x", mbs.param[1], portid); return (MBOX_COMMAND_ERROR); case MBOX_ALL_IDS_USED: isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "isp_port_login: all IDs used for fabric login"); return (MBOX_ALL_IDS_USED); default: isp_prt(isp, ISP_LOG_SANCFG, "isp_port_login: error 0x%x on port login of 0x%06x@0x%0x", mbs.param[0], portid, handle); return (mbs.param[0]); } } /* * Pre-24XX fabric port logout * * Note that portid is not used */ static int isp_port_logout(ispsoftc_t *isp, uint16_t handle, uint32_t portid) { mbreg_t mbs; MBSINIT(&mbs, MBOX_FABRIC_LOGOUT, MBLOGNONE, 500000); if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = handle; mbs.ibits = (1 << 10); } else { mbs.param[1] = handle << 8; } isp_mboxcmd(isp, &mbs); return (mbs.param[0] == MBOX_COMMAND_COMPLETE? 0 : mbs.param[0]); } static int isp_getpdb(ispsoftc_t *isp, int chan, uint16_t id, isp_pdb_t *pdb) { mbreg_t mbs; union { isp_pdb_21xx_t fred; isp_pdb_24xx_t bill; } un; MBSINIT(&mbs, MBOX_GET_PORT_DB, MBLOGALL & ~MBLOGMASK(MBOX_COMMAND_PARAM_ERROR), 250000); if (IS_24XX(isp)) { mbs.ibits = (1 << 9)|(1 << 10); mbs.param[1] = id; mbs.param[9] = chan; } else if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = id; } else { mbs.param[1] = id << 8; } mbs.param[2] = DMA_WD1(isp->isp_iocb_dma); mbs.param[3] = DMA_WD0(isp->isp_iocb_dma); mbs.param[6] = DMA_WD3(isp->isp_iocb_dma); mbs.param[7] = DMA_WD2(isp->isp_iocb_dma); MEMORYBARRIER(isp, SYNC_IFORDEV, 0, sizeof(un), chan); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) return (mbs.param[0] | (mbs.param[1] << 16)); MEMORYBARRIER(isp, SYNC_IFORCPU, 0, sizeof(un), chan); if (IS_24XX(isp)) { isp_get_pdb_24xx(isp, isp->isp_iocb, &un.bill); pdb->handle = un.bill.pdb_handle; + pdb->prli_word0 = un.bill.pdb_prli_svc0; pdb->prli_word3 = un.bill.pdb_prli_svc3; pdb->portid = BITS2WORD_24XX(un.bill.pdb_portid_bits); ISP_MEMCPY(pdb->portname, un.bill.pdb_portname, 8); ISP_MEMCPY(pdb->nodename, un.bill.pdb_nodename, 8); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d handle 0x%x Port 0x%06x flags 0x%x curstate %x laststate %x", chan, id, pdb->portid, un.bill.pdb_flags, un.bill.pdb_curstate, un.bill.pdb_laststate); if (un.bill.pdb_curstate < PDB2400_STATE_PLOGI_DONE || un.bill.pdb_curstate > PDB2400_STATE_LOGGED_IN) { mbs.param[0] = MBOX_NOT_LOGGED_IN; return (mbs.param[0]); } } else { isp_get_pdb_21xx(isp, isp->isp_iocb, &un.fred); pdb->handle = un.fred.pdb_loopid; + pdb->prli_word0 = un.fred.pdb_prli_svc0; pdb->prli_word3 = un.fred.pdb_prli_svc3; pdb->portid = BITS2WORD(un.fred.pdb_portid_bits); ISP_MEMCPY(pdb->portname, un.fred.pdb_portname, 8); ISP_MEMCPY(pdb->nodename, un.fred.pdb_nodename, 8); isp_prt(isp, ISP_LOGDEBUG1, "Chan %d handle 0x%x Port 0x%06x", chan, id, pdb->portid); } return (0); } static int isp_gethandles(ispsoftc_t *isp, int chan, uint16_t *handles, int *num, int loop) { fcparam *fcp = FCPARAM(isp, chan); mbreg_t mbs; isp_pnhle_21xx_t el1, *elp1; isp_pnhle_23xx_t el3, *elp3; isp_pnhle_24xx_t el4, *elp4; int i, j; uint32_t p; uint16_t h; MBSINIT(&mbs, MBOX_GET_ID_LIST, MBLOGALL, 250000); if (IS_24XX(isp)) { mbs.param[2] = DMA_WD1(fcp->isp_scdma); mbs.param[3] = DMA_WD0(fcp->isp_scdma); mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); mbs.param[8] = ISP_FC_SCRLEN; mbs.param[9] = chan; } else { mbs.ibits = (1 << 1)|(1 << 2)|(1 << 3)|(1 << 6); mbs.param[1] = DMA_WD1(fcp->isp_scdma); mbs.param[2] = DMA_WD0(fcp->isp_scdma); mbs.param[3] = DMA_WD3(fcp->isp_scdma); mbs.param[6] = DMA_WD2(fcp->isp_scdma); } if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } MEMORYBARRIER(isp, SYNC_SFORDEV, 0, ISP_FC_SCRLEN, chan); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { FC_SCRATCH_RELEASE(isp, chan); return (mbs.param[0] | (mbs.param[1] << 16)); } MEMORYBARRIER(isp, SYNC_SFORCPU, 0, ISP_FC_SCRLEN, chan); elp1 = fcp->isp_scratch; elp3 = fcp->isp_scratch; elp4 = fcp->isp_scratch; for (i = 0, j = 0; i < mbs.param[1] && j < *num; i++) { if (IS_24XX(isp)) { isp_get_pnhle_24xx(isp, &elp4[i], &el4); p = el4.pnhle_port_id_lo | (el4.pnhle_port_id_hi << 16); h = el4.pnhle_handle; } else if (IS_23XX(isp)) { isp_get_pnhle_23xx(isp, &elp3[i], &el3); p = el3.pnhle_port_id_lo | (el3.pnhle_port_id_hi << 16); h = el3.pnhle_handle; } else { /* 21xx */ isp_get_pnhle_21xx(isp, &elp1[i], &el1); p = el1.pnhle_port_id_lo | ((el1.pnhle_port_id_hi_handle & 0xff) << 16); h = el1.pnhle_port_id_hi_handle >> 8; } if (loop && (p >> 8) != (fcp->isp_portid >> 8)) continue; handles[j++] = h; } *num = j; FC_SCRATCH_RELEASE(isp, chan); return (0); } static void isp_dump_chip_portdb(ispsoftc_t *isp, int chan) { isp_pdb_t pdb; uint16_t lim, nphdl; isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGINFO, "Chan %d chip port dump", chan); if (ISP_CAP_2KLOGIN(isp)) { lim = NPH_MAX_2K; } else { lim = NPH_MAX; } for (nphdl = 0; nphdl != lim; nphdl++) { if (isp_getpdb(isp, chan, nphdl, &pdb)) { continue; } isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGINFO, "Chan %d Handle 0x%04x " "PortID 0x%06x WWPN 0x%02x%02x%02x%02x%02x%02x%02x%02x", chan, nphdl, pdb.portid, pdb.portname[0], pdb.portname[1], pdb.portname[2], pdb.portname[3], pdb.portname[4], pdb.portname[5], pdb.portname[6], pdb.portname[7]); } } static uint64_t isp_get_wwn(ispsoftc_t *isp, int chan, int nphdl, int nodename) { uint64_t wwn = INI_NONE; mbreg_t mbs; MBSINIT(&mbs, MBOX_GET_PORT_NAME, MBLOGALL & ~MBLOGMASK(MBOX_COMMAND_PARAM_ERROR), 500000); if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = nphdl; if (nodename) { mbs.param[10] = 1; } mbs.param[9] = chan; } else { mbs.ibitm = 3; mbs.param[1] = nphdl << 8; if (nodename) { mbs.param[1] |= 1; } } isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return (wwn); } if (IS_24XX(isp)) { wwn = (((uint64_t)(mbs.param[2] >> 8)) << 56) | (((uint64_t)(mbs.param[2] & 0xff)) << 48) | (((uint64_t)(mbs.param[3] >> 8)) << 40) | (((uint64_t)(mbs.param[3] & 0xff)) << 32) | (((uint64_t)(mbs.param[6] >> 8)) << 24) | (((uint64_t)(mbs.param[6] & 0xff)) << 16) | (((uint64_t)(mbs.param[7] >> 8)) << 8) | (((uint64_t)(mbs.param[7] & 0xff))); } else { wwn = (((uint64_t)(mbs.param[2] & 0xff)) << 56) | (((uint64_t)(mbs.param[2] >> 8)) << 48) | (((uint64_t)(mbs.param[3] & 0xff)) << 40) | (((uint64_t)(mbs.param[3] >> 8)) << 32) | (((uint64_t)(mbs.param[6] & 0xff)) << 24) | (((uint64_t)(mbs.param[6] >> 8)) << 16) | (((uint64_t)(mbs.param[7] & 0xff)) << 8) | (((uint64_t)(mbs.param[7] >> 8))); } return (wwn); } /* * Make sure we have good FC link. */ static int isp_fclink_test(ispsoftc_t *isp, int chan, int usdelay) { mbreg_t mbs; int i, r; uint16_t nphdl; fcparam *fcp; isp_pdb_t pdb; NANOTIME_T hra, hrb; fcp = FCPARAM(isp, chan); if (fcp->isp_loopstate < LOOP_HAVE_LINK) return (-1); if (fcp->isp_loopstate >= LOOP_LTEST_DONE) return (0); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC link test", chan); /* * Wait up to N microseconds for F/W to go to a ready state. */ GET_NANOTIME(&hra); while (1) { isp_change_fw_state(isp, chan, isp_fw_state(isp, chan)); if (fcp->isp_fwstate == FW_READY) { break; } if (fcp->isp_loopstate < LOOP_HAVE_LINK) goto abort; GET_NANOTIME(&hrb); if ((NANOTIME_SUB(&hrb, &hra) / 1000 + 1000 >= usdelay)) break; ISP_SLEEP(isp, 1000); } if (fcp->isp_fwstate != FW_READY) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Firmware is not ready (%s)", chan, isp_fc_fw_statename(fcp->isp_fwstate)); return (-1); } /* * Get our Loop ID and Port ID. */ MBSINIT(&mbs, MBOX_GET_LOOP_ID, MBLOGALL, 0); mbs.param[9] = chan; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { return (-1); } if (IS_2100(isp)) { /* * Don't bother with fabric if we are using really old * 2100 firmware. It's just not worth it. */ if (ISP_FW_NEWER_THAN(isp, 1, 15, 37)) fcp->isp_topo = TOPO_FL_PORT; else fcp->isp_topo = TOPO_NL_PORT; } else { int topo = (int) mbs.param[6]; if (topo < TOPO_NL_PORT || topo > TOPO_PTP_STUB) { topo = TOPO_PTP_STUB; } fcp->isp_topo = topo; } fcp->isp_portid = mbs.param[2] | (mbs.param[3] << 16); if (!TOPO_IS_FABRIC(fcp->isp_topo)) { fcp->isp_loopid = mbs.param[1] & 0xff; } else if (fcp->isp_topo != TOPO_F_PORT) { uint8_t alpa = fcp->isp_portid; for (i = 0; alpa_map[i]; i++) { if (alpa_map[i] == alpa) break; } if (alpa_map[i]) fcp->isp_loopid = i; } #if 0 fcp->isp_loopstate = LOOP_HAVE_ADDR; #endif fcp->isp_loopstate = LOOP_TESTING_LINK; if (fcp->isp_topo == TOPO_F_PORT || fcp->isp_topo == TOPO_FL_PORT) { nphdl = IS_24XX(isp) ? NPH_FL_ID : FL_ID; r = isp_getpdb(isp, chan, nphdl, &pdb); if (r != 0 || pdb.portid == 0) { if (IS_2100(isp)) { fcp->isp_topo = TOPO_NL_PORT; } else { isp_prt(isp, ISP_LOGWARN, "fabric topology, but cannot get info about fabric controller (0x%x)", r); fcp->isp_topo = TOPO_PTP_STUB; } goto not_on_fabric; } if (IS_24XX(isp)) { fcp->isp_fabric_params = mbs.param[7]; fcp->isp_sns_hdl = NPH_SNS_ID; r = isp_register_fc4_type(isp, chan); if (fcp->isp_loopstate < LOOP_TESTING_LINK) goto abort; if (r != 0) goto not_on_fabric; r = isp_register_fc4_features_24xx(isp, chan); if (fcp->isp_loopstate < LOOP_TESTING_LINK) goto abort; if (r != 0) goto not_on_fabric; r = isp_register_port_name_24xx(isp, chan); if (fcp->isp_loopstate < LOOP_TESTING_LINK) goto abort; if (r != 0) goto not_on_fabric; isp_register_node_name_24xx(isp, chan); if (fcp->isp_loopstate < LOOP_TESTING_LINK) goto abort; } else { fcp->isp_sns_hdl = SNS_ID; r = isp_register_fc4_type(isp, chan); if (r != 0) goto not_on_fabric; if (fcp->role == ISP_ROLE_TARGET) isp_send_change_request(isp, chan); } } not_on_fabric: /* Get link speed. */ fcp->isp_gbspeed = 1; if (IS_23XX(isp) || IS_24XX(isp)) { MBSINIT(&mbs, MBOX_GET_SET_DATA_RATE, MBLOGALL, 3000000); mbs.param[1] = MBGSD_GET_RATE; /* mbs.param[2] undefined if we're just getting rate */ isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { if (mbs.param[1] == MBGSD_10GB) fcp->isp_gbspeed = 10; else if (mbs.param[1] == MBGSD_32GB) fcp->isp_gbspeed = 32; else if (mbs.param[1] == MBGSD_16GB) fcp->isp_gbspeed = 16; else if (mbs.param[1] == MBGSD_8GB) fcp->isp_gbspeed = 8; else if (mbs.param[1] == MBGSD_4GB) fcp->isp_gbspeed = 4; else if (mbs.param[1] == MBGSD_2GB) fcp->isp_gbspeed = 2; else if (mbs.param[1] == MBGSD_1GB) fcp->isp_gbspeed = 1; } } if (fcp->isp_loopstate < LOOP_TESTING_LINK) { abort: isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC link test aborted", chan); return (1); } fcp->isp_loopstate = LOOP_LTEST_DONE; isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGCONFIG, "Chan %d WWPN %016jx WWNN %016jx", chan, (uintmax_t)fcp->isp_wwpn, (uintmax_t)fcp->isp_wwnn); isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGCONFIG, "Chan %d %dGb %s PortID 0x%06x LoopID 0x%02x", chan, fcp->isp_gbspeed, isp_fc_toponame(fcp), fcp->isp_portid, fcp->isp_loopid); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC link test done", chan); return (0); } /* * Complete the synchronization of our Port Database. * * At this point, we've scanned the local loop (if any) and the fabric * and performed fabric logins on all new devices. * * Our task here is to go through our port database removing any entities * that are still marked probational (issuing PLOGO for ones which we had * PLOGI'd into) or are dead, and notifying upper layers about new/changed * devices. */ static int isp_pdb_sync(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; uint16_t dbidx; if (fcp->isp_loopstate < LOOP_FSCAN_DONE) return (-1); if (fcp->isp_loopstate >= LOOP_READY) return (0); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC PDB sync", chan); fcp->isp_loopstate = LOOP_SYNCING_PDB; for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) { lp = &fcp->portdb[dbidx]; if (lp->state == FC_PORTDB_STATE_NIL) continue; if (lp->probational && lp->state != FC_PORTDB_STATE_ZOMBIE) lp->state = FC_PORTDB_STATE_DEAD; switch (lp->state) { case FC_PORTDB_STATE_DEAD: lp->state = FC_PORTDB_STATE_NIL; isp_async(isp, ISPASYNC_DEV_GONE, chan, lp); if ((lp->portid & 0xffff00) != 0) { (void) isp_plogx(isp, chan, lp->handle, lp->portid, PLOGX_FLG_CMD_LOGO | PLOGX_FLG_IMPLICIT | PLOGX_FLG_FREE_NPHDL); } /* * Note that we might come out of this with our state * set to FC_PORTDB_STATE_ZOMBIE. */ break; case FC_PORTDB_STATE_NEW: lp->state = FC_PORTDB_STATE_VALID; isp_async(isp, ISPASYNC_DEV_ARRIVED, chan, lp); break; case FC_PORTDB_STATE_CHANGED: lp->state = FC_PORTDB_STATE_VALID; isp_async(isp, ISPASYNC_DEV_CHANGED, chan, lp); lp->portid = lp->new_portid; + lp->prli_word0 = lp->new_prli_word0; lp->prli_word3 = lp->new_prli_word3; break; case FC_PORTDB_STATE_VALID: isp_async(isp, ISPASYNC_DEV_STAYED, chan, lp); break; case FC_PORTDB_STATE_ZOMBIE: break; default: isp_prt(isp, ISP_LOGWARN, "isp_pdb_sync: state %d for idx %d", lp->state, dbidx); isp_dump_portdb(isp, chan); } } if (fcp->isp_loopstate < LOOP_SYNCING_PDB) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC PDB sync aborted", chan); return (1); } fcp->isp_loopstate = LOOP_READY; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC PDB sync done", chan); return (0); } static void isp_pdb_add_update(ispsoftc_t *isp, int chan, isp_pdb_t *pdb) { fcportdb_t *lp; uint64_t wwnn, wwpn; MAKE_WWN_FROM_NODE_NAME(wwnn, pdb->nodename); MAKE_WWN_FROM_NODE_NAME(wwpn, pdb->portname); /* Search port database for the same WWPN. */ if (isp_find_pdb_by_wwpn(isp, chan, wwpn, &lp)) { if (!lp->probational) { isp_prt(isp, ISP_LOGERR, "Chan %d Port 0x%06x@0x%04x [%d] is not probational (0x%x)", chan, lp->portid, lp->handle, FC_PORTDB_TGT(isp, chan, lp), lp->state); isp_dump_portdb(isp, chan); return; } lp->probational = 0; lp->node_wwn = wwnn; /* Old device, nothing new. */ if (lp->portid == pdb->portid && lp->handle == pdb->handle && - lp->prli_word3 == pdb->prli_word3) { + lp->prli_word3 == pdb->prli_word3 && + ((pdb->prli_word0 & PRLI_WD0_EST_IMAGE_PAIR) == 0)) { if (lp->state != FC_PORTDB_STATE_NEW) lp->state = FC_PORTDB_STATE_VALID; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x@0x%04x is valid", chan, pdb->portid, pdb->handle); return; } /* Something has changed. */ lp->state = FC_PORTDB_STATE_CHANGED; lp->handle = pdb->handle; lp->new_portid = pdb->portid; + lp->new_prli_word0 = pdb->prli_word0; lp->new_prli_word3 = pdb->prli_word3; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x@0x%04x is changed", chan, pdb->portid, pdb->handle); return; } /* It seems like a new port. Find an empty slot for it. */ if (!isp_find_pdb_empty(isp, chan, &lp)) { isp_prt(isp, ISP_LOGERR, "Chan %d out of portdb entries", chan); return; } ISP_MEMZERO(lp, sizeof (fcportdb_t)); lp->probational = 0; lp->state = FC_PORTDB_STATE_NEW; lp->portid = lp->new_portid = pdb->portid; lp->prli_word3 = lp->new_prli_word3 = pdb->prli_word3; lp->handle = pdb->handle; lp->port_wwn = wwpn; lp->node_wwn = wwnn; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x@0x%04x is new", chan, pdb->portid, pdb->handle); } /* * Fix port IDs for logged-in initiators on pre-2400 chips. * For those chips we are not receiving login events, adding initiators * based on ATIO requests, but there is no port ID in that structure. */ static void isp_fix_portids(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); isp_pdb_t pdb; uint64_t wwpn; int i, r; for (i = 0; i < MAX_FC_TARG; i++) { fcportdb_t *lp = &fcp->portdb[i]; if (lp->state == FC_PORTDB_STATE_NIL || lp->state == FC_PORTDB_STATE_ZOMBIE) continue; if (VALID_PORT(lp->portid)) continue; r = isp_getpdb(isp, chan, lp->handle, &pdb); if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) return; if (r != 0) { isp_prt(isp, ISP_LOGDEBUG1, "Chan %d FC Scan Loop handle %d returned %x", chan, lp->handle, r); continue; } MAKE_WWN_FROM_NODE_NAME(wwpn, pdb.portname); if (lp->port_wwn != wwpn) continue; lp->portid = lp->new_portid = pdb.portid; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x@0x%04x is fixed", chan, pdb.portid, pdb.handle); } } /* * Scan local loop for devices. */ static int isp_scan_loop(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); int idx, lim, r; isp_pdb_t pdb; uint16_t *handles; uint16_t handle; if (fcp->isp_loopstate < LOOP_LTEST_DONE) return (-1); if (fcp->isp_loopstate >= LOOP_LSCAN_DONE) return (0); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC loop scan", chan); fcp->isp_loopstate = LOOP_SCANNING_LOOP; if (TOPO_IS_FABRIC(fcp->isp_topo)) { if (!IS_24XX(isp)) { isp_fix_portids(isp, chan); if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) goto abort; } isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC loop scan done (no loop)", chan); fcp->isp_loopstate = LOOP_LSCAN_DONE; return (0); } handles = (uint16_t *)fcp->isp_scanscratch; lim = ISP_FC_SCRLEN / 2; r = isp_gethandles(isp, chan, handles, &lim, 1); if (r != 0) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Getting list of handles failed with %x", chan, r); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC loop scan done (bad)", chan); return (-1); } isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Got %d handles", chan, lim); /* * Run through the list and get the port database info for each one. */ isp_mark_portdb(isp, chan); for (idx = 0; idx < lim; idx++) { handle = handles[idx]; /* * Don't scan "special" ids. */ if (ISP_CAP_2KLOGIN(isp)) { if (handle >= NPH_RESERVED) continue; } else { if (handle >= FL_ID && handle <= SNS_ID) continue; } /* * In older cards with older f/w GET_PORT_DATABASE has been * known to hang. This trick gets around that problem. */ if (IS_2100(isp) || IS_2200(isp)) { uint64_t node_wwn = isp_get_wwn(isp, chan, handle, 1); if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) { abort: isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC loop scan aborted", chan); return (1); } if (node_wwn == INI_NONE) { continue; } } /* * Get the port database entity for this index. */ r = isp_getpdb(isp, chan, handle, &pdb); if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) goto abort; if (r != 0) { isp_prt(isp, ISP_LOGDEBUG1, "Chan %d FC Scan Loop handle %d returned %x", chan, handle, r); continue; } isp_pdb_add_update(isp, chan, &pdb); } if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) goto abort; fcp->isp_loopstate = LOOP_LSCAN_DONE; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC loop scan done", chan); return (0); } static int isp_ct_sns(ispsoftc_t *isp, int chan, uint32_t cmd_bcnt, uint32_t rsp_bcnt) { fcparam *fcp = FCPARAM(isp, chan); mbreg_t mbs; if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT SNS request", cmd_bcnt, fcp->isp_scratch); MEMORYBARRIER(isp, SYNC_SFORDEV, 0, cmd_bcnt, chan); MBSINIT(&mbs, MBOX_SEND_SNS, MBLOGALL, 10000000); mbs.param[1] = cmd_bcnt >> 1; mbs.param[2] = DMA_WD1(fcp->isp_scdma); mbs.param[3] = DMA_WD0(fcp->isp_scdma); mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { if (mbs.param[0] == MBOX_INVALID_COMMAND) { return (1); } else { return (-1); } } MEMORYBARRIER(isp, SYNC_SFORCPU, 0, rsp_bcnt, chan); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT response", rsp_bcnt, fcp->isp_scratch); return (0); } static int isp_ct_passthru(ispsoftc_t *isp, int chan, uint32_t cmd_bcnt, uint32_t rsp_bcnt) { fcparam *fcp = FCPARAM(isp, chan); isp_ct_pt_t pt; void *reqp; uint8_t resp[QENTRY_LEN]; if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT request", cmd_bcnt, fcp->isp_scratch); /* * Build a Passthrough IOCB in memory. */ ISP_MEMZERO(&pt, sizeof(pt)); pt.ctp_header.rqs_entry_count = 1; pt.ctp_header.rqs_entry_type = RQSTYPE_CT_PASSTHRU; pt.ctp_nphdl = fcp->isp_sns_hdl; pt.ctp_cmd_cnt = 1; pt.ctp_vpidx = ISP_GET_VPIDX(isp, chan); pt.ctp_time = 10; pt.ctp_rsp_cnt = 1; pt.ctp_rsp_bcnt = rsp_bcnt; pt.ctp_cmd_bcnt = cmd_bcnt; pt.ctp_dataseg[0].ds_base = DMA_LO32(fcp->isp_scdma); pt.ctp_dataseg[0].ds_basehi = DMA_HI32(fcp->isp_scdma); pt.ctp_dataseg[0].ds_count = cmd_bcnt; pt.ctp_dataseg[1].ds_base = DMA_LO32(fcp->isp_scdma); pt.ctp_dataseg[1].ds_basehi = DMA_HI32(fcp->isp_scdma); pt.ctp_dataseg[1].ds_count = rsp_bcnt; /* Prepare space for response in memory */ memset(resp, 0xff, sizeof(resp)); pt.ctp_handle = isp_allocate_handle(isp, resp, ISP_HANDLE_CTRL); if (pt.ctp_handle == 0) { isp_prt(isp, ISP_LOGERR, "%s: CTP of Chan %d out of handles", __func__, chan); return (-1); } /* Send request and wait for response. */ reqp = isp_getrqentry(isp); if (reqp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: CTP of Chan %d out of rqent", __func__, chan); isp_destroy_handle(isp, pt.ctp_handle); return (-1); } isp_put_ct_pt(isp, &pt, (isp_ct_pt_t *)reqp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT IOCB request", QENTRY_LEN, reqp); ISP_SYNC_REQUEST(isp); if (msleep(resp, &isp->isp_lock, 0, "CTP", pt.ctp_time*hz) == EWOULDBLOCK) { isp_prt(isp, ISP_LOGERR, "%s: CTP of Chan %d timed out", __func__, chan); isp_destroy_handle(isp, pt.ctp_handle); return (-1); } if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT IOCB response", QENTRY_LEN, resp); isp_get_ct_pt(isp, (isp_ct_pt_t *)resp, &pt); if (pt.ctp_status && pt.ctp_status != RQCS_DATA_UNDERRUN) { isp_prt(isp, ISP_LOGWARN, "Chan %d CT pass-through returned 0x%x", chan, pt.ctp_status); return (-1); } if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT response", rsp_bcnt, fcp->isp_scratch); return (0); } /* * Scan the fabric for devices and add them to our port database. * * Use the GID_PT command to get list of all Nx_Port IDs SNS knows. * Use GFF_ID and GFT_ID to check port type (FCP) and features (target). * * For 2100-23XX cards, we use the SNS mailbox command to pass simple name * server commands to the switch management server via the QLogic f/w. * * For the 24XX and above card, we use CT Pass-through IOCB. */ #define GIDLEN ISP_FC_SCRLEN #define NGENT ((GIDLEN - 16) >> 2) static int isp_gid_pt(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t ct; sns_gid_pt_req_t rq; uint8_t *scp = fcp->isp_scratch; isp_prt(isp, ISP_LOGDEBUG0, "Chan %d requesting GID_PT", chan); if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } if (IS_24XX(isp)) { /* Build the CT command and execute via pass-through. */ ISP_MEMZERO(&ct, sizeof (ct)); ct.ct_revision = CT_REVISION; ct.ct_fcs_type = CT_FC_TYPE_FC; ct.ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct.ct_cmd_resp = SNS_GID_PT; ct.ct_bcnt_resid = (GIDLEN - 16) >> 2; isp_put_ct_hdr(isp, &ct, (ct_hdr_t *)scp); scp[sizeof(ct)] = 0x7f; /* Port Type = Nx_Port */ scp[sizeof(ct)+1] = 0; /* Domain_ID = any */ scp[sizeof(ct)+2] = 0; /* Area_ID = any */ scp[sizeof(ct)+3] = 0; /* Flags = no Area_ID */ if (isp_ct_passthru(isp, chan, sizeof(ct) + sizeof(uint32_t), GIDLEN)) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } } else { /* Build the SNS request and execute via firmware. */ ISP_MEMZERO(&rq, SNS_GID_PT_REQ_SIZE); rq.snscb_rblen = GIDLEN >> 1; rq.snscb_addr[RQRSP_ADDR0015] = DMA_WD0(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR1631] = DMA_WD1(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR3247] = DMA_WD2(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR4863] = DMA_WD3(fcp->isp_scdma); rq.snscb_sblen = 6; rq.snscb_cmd = SNS_GID_PT; rq.snscb_mword_div_2 = NGENT; rq.snscb_port_type = 0x7f; /* Port Type = Nx_Port */ rq.snscb_domain = 0; /* Domain_ID = any */ rq.snscb_area = 0; /* Area_ID = any */ rq.snscb_flags = 0; /* Flags = no Area_ID */ isp_put_gid_pt_request(isp, &rq, (sns_gid_pt_req_t *)scp); if (isp_ct_sns(isp, chan, sizeof(rq), NGENT)) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } } isp_get_gid_xx_response(isp, (sns_gid_xx_rsp_t *)scp, (sns_gid_xx_rsp_t *)fcp->isp_scanscratch, NGENT); FC_SCRATCH_RELEASE(isp, chan); return (0); } static int isp_gff_id(ispsoftc_t *isp, int chan, uint32_t portid) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t ct; uint32_t *rp; uint8_t *scp = fcp->isp_scratch; sns_gff_id_rsp_t rsp; int i, res = -1; if (!fcp->isp_use_gff_id) /* User may block GFF_ID use. */ return (res); if (!IS_24XX(isp)) /* Old chips can't request GFF_ID. */ return (res); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d requesting GFF_ID", chan); if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (res); } /* Build the CT command and execute via pass-through. */ ISP_MEMZERO(&ct, sizeof (ct)); ct.ct_revision = CT_REVISION; ct.ct_fcs_type = CT_FC_TYPE_FC; ct.ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct.ct_cmd_resp = SNS_GFF_ID; ct.ct_bcnt_resid = (SNS_GFF_ID_RESP_SIZE - sizeof(ct)) / 4; isp_put_ct_hdr(isp, &ct, (ct_hdr_t *)scp); rp = (uint32_t *) &scp[sizeof(ct)]; ISP_IOZPUT_32(isp, portid, rp); if (isp_ct_passthru(isp, chan, sizeof(ct) + sizeof(uint32_t), SNS_GFF_ID_RESP_SIZE)) { FC_SCRATCH_RELEASE(isp, chan); return (res); } isp_get_gff_id_response(isp, (sns_gff_id_rsp_t *)scp, &rsp); if (rsp.snscb_cthdr.ct_cmd_resp == LS_ACC) { for (i = 0; i < 32; i++) { if (rsp.snscb_fc4_features[i] != 0) { res = 0; break; } } if (((rsp.snscb_fc4_features[FC4_SCSI / 8] >> ((FC4_SCSI % 8) * 4)) & 0x01) != 0) res = 1; /* Workaround for broken Brocade firmware. */ if (((ISP_SWAP32(isp, rsp.snscb_fc4_features[FC4_SCSI / 8]) >> ((FC4_SCSI % 8) * 4)) & 0x01) != 0) res = 1; } FC_SCRATCH_RELEASE(isp, chan); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d GFF_ID result is %d", chan, res); return (res); } static int isp_gft_id(ispsoftc_t *isp, int chan, uint32_t portid) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t ct; sns_gxx_id_req_t rq; uint32_t *rp; uint8_t *scp = fcp->isp_scratch; sns_gft_id_rsp_t rsp; int i, res = -1; if (!fcp->isp_use_gft_id) /* User may block GFT_ID use. */ return (res); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d requesting GFT_ID", chan); if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (res); } if (IS_24XX(isp)) { /* Build the CT command and execute via pass-through. */ ISP_MEMZERO(&ct, sizeof (ct)); ct.ct_revision = CT_REVISION; ct.ct_fcs_type = CT_FC_TYPE_FC; ct.ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct.ct_cmd_resp = SNS_GFT_ID; ct.ct_bcnt_resid = (SNS_GFT_ID_RESP_SIZE - sizeof(ct)) / 4; isp_put_ct_hdr(isp, &ct, (ct_hdr_t *)scp); rp = (uint32_t *) &scp[sizeof(ct)]; ISP_IOZPUT_32(isp, portid, rp); if (isp_ct_passthru(isp, chan, sizeof(ct) + sizeof(uint32_t), SNS_GFT_ID_RESP_SIZE)) { FC_SCRATCH_RELEASE(isp, chan); return (res); } } else { /* Build the SNS request and execute via firmware. */ ISP_MEMZERO(&rq, SNS_GXX_ID_REQ_SIZE); rq.snscb_rblen = SNS_GFT_ID_RESP_SIZE >> 1; rq.snscb_addr[RQRSP_ADDR0015] = DMA_WD0(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR1631] = DMA_WD1(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR3247] = DMA_WD2(fcp->isp_scdma); rq.snscb_addr[RQRSP_ADDR4863] = DMA_WD3(fcp->isp_scdma); rq.snscb_sblen = 6; rq.snscb_cmd = SNS_GFT_ID; rq.snscb_mword_div_2 = (SNS_GFT_ID_RESP_SIZE - sizeof(ct)) / 4; rq.snscb_portid = portid; isp_put_gxx_id_request(isp, &rq, (sns_gxx_id_req_t *)scp); if (isp_ct_sns(isp, chan, sizeof(rq), SNS_GFT_ID_RESP_SIZE)) { FC_SCRATCH_RELEASE(isp, chan); return (res); } } isp_get_gft_id_response(isp, (sns_gft_id_rsp_t *)scp, &rsp); if (rsp.snscb_cthdr.ct_cmd_resp == LS_ACC) { for (i = 0; i < 8; i++) { if (rsp.snscb_fc4_types[i] != 0) { res = 0; break; } } if (((rsp.snscb_fc4_types[FC4_SCSI / 32] >> (FC4_SCSI % 32)) & 0x01) != 0) res = 1; } FC_SCRATCH_RELEASE(isp, chan); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d GFT_ID result is %d", chan, res); return (res); } static int isp_scan_fabric(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; uint32_t portid; uint16_t nphdl; isp_pdb_t pdb; int portidx, portlim, r; sns_gid_xx_rsp_t *rs; if (fcp->isp_loopstate < LOOP_LSCAN_DONE) return (-1); if (fcp->isp_loopstate >= LOOP_FSCAN_DONE) return (0); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC fabric scan", chan); fcp->isp_loopstate = LOOP_SCANNING_FABRIC; if (!TOPO_IS_FABRIC(fcp->isp_topo)) { fcp->isp_loopstate = LOOP_FSCAN_DONE; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC fabric scan done (no fabric)", chan); return (0); } if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) { abort: FC_SCRATCH_RELEASE(isp, chan); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC fabric scan aborted", chan); return (1); } /* * Make sure we still are logged into the fabric controller. */ nphdl = IS_24XX(isp) ? NPH_FL_ID : FL_ID; r = isp_getpdb(isp, chan, nphdl, &pdb); if ((r & 0xffff) == MBOX_NOT_LOGGED_IN) { isp_dump_chip_portdb(isp, chan); } if (r) { fcp->isp_loopstate = LOOP_LTEST_DONE; fail: isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC fabric scan done (bad)", chan); return (-1); } /* Get list of port IDs from SNS. */ r = isp_gid_pt(isp, chan); if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) goto abort; if (r > 0) { fcp->isp_loopstate = LOOP_FSCAN_DONE; return (-1); } else if (r < 0) { fcp->isp_loopstate = LOOP_LTEST_DONE; /* try again */ return (-1); } rs = (sns_gid_xx_rsp_t *) fcp->isp_scanscratch; if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) goto abort; if (rs->snscb_cthdr.ct_cmd_resp != LS_ACC) { int level; /* FC-4 Type and Port Type not registered are not errors. */ if (rs->snscb_cthdr.ct_reason == 9 && (rs->snscb_cthdr.ct_explanation == 0x07 || rs->snscb_cthdr.ct_explanation == 0x0a)) { level = ISP_LOG_SANCFG; } else { level = ISP_LOGWARN; } isp_prt(isp, level, "Chan %d Fabric Nameserver rejected GID_PT" " (Reason=0x%x Expl=0x%x)", chan, rs->snscb_cthdr.ct_reason, rs->snscb_cthdr.ct_explanation); fcp->isp_loopstate = LOOP_FSCAN_DONE; return (-1); } /* Check our buffer was big enough to get the full list. */ for (portidx = 0; portidx < NGENT-1; portidx++) { if (rs->snscb_ports[portidx].control & 0x80) break; } if ((rs->snscb_ports[portidx].control & 0x80) == 0) { isp_prt(isp, ISP_LOGWARN, "fabric too big for scratch area: increase ISP_FC_SCRLEN"); } portlim = portidx + 1; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Got %d ports back from name server", chan, portlim); /* Go through the list and remove duplicate port ids. */ for (portidx = 0; portidx < portlim; portidx++) { int npidx; portid = ((rs->snscb_ports[portidx].portid[0]) << 16) | ((rs->snscb_ports[portidx].portid[1]) << 8) | ((rs->snscb_ports[portidx].portid[2])); for (npidx = portidx + 1; npidx < portlim; npidx++) { uint32_t new_portid = ((rs->snscb_ports[npidx].portid[0]) << 16) | ((rs->snscb_ports[npidx].portid[1]) << 8) | ((rs->snscb_ports[npidx].portid[2])); if (new_portid == portid) { break; } } if (npidx < portlim) { rs->snscb_ports[npidx].portid[0] = 0; rs->snscb_ports[npidx].portid[1] = 0; rs->snscb_ports[npidx].portid[2] = 0; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d removing duplicate PortID 0x%06x entry from list", chan, portid); } } /* * We now have a list of Port IDs for all FC4 SCSI devices * that the Fabric Name server knows about. * * For each entry on this list go through our port database looking * for probational entries- if we find one, then an old entry is * maybe still this one. We get some information to find out. * * Otherwise, it's a new fabric device, and we log into it * (unconditionally). After searching the entire database * again to make sure that we never ever ever ever have more * than one entry that has the same PortID or the same * WWNN/WWPN duple, we enter the device into our database. */ isp_mark_portdb(isp, chan); for (portidx = 0; portidx < portlim; portidx++) { portid = ((rs->snscb_ports[portidx].portid[0]) << 16) | ((rs->snscb_ports[portidx].portid[1]) << 8) | ((rs->snscb_ports[portidx].portid[2])); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Checking fabric port 0x%06x", chan, portid); if (portid == 0) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port at idx %d is zero", chan, portidx); continue; } if (portid == fcp->isp_portid) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x is our", chan, portid); continue; } /* Now search the entire port database for the same portid. */ if (isp_find_pdb_by_portid(isp, chan, portid, &lp)) { if (!lp->probational) { isp_prt(isp, ISP_LOGERR, "Chan %d Port 0x%06x@0x%04x [%d] is not probational (0x%x)", chan, lp->portid, lp->handle, FC_PORTDB_TGT(isp, chan, lp), lp->state); isp_dump_portdb(isp, chan); goto fail; } if (lp->state == FC_PORTDB_STATE_ZOMBIE) goto relogin; /* * See if we're still logged into it. * * If we aren't, mark it as a dead device and * leave the new portid in the database entry * for somebody further along to decide what to * do (policy choice). * * If we are, check to see if it's the same * device still (it should be). If for some * reason it isn't, mark it as a changed device * and leave the new portid and role in the * database entry for somebody further along to * decide what to do (policy choice). */ r = isp_getpdb(isp, chan, lp->handle, &pdb); if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) goto abort; if (r != 0) { lp->state = FC_PORTDB_STATE_DEAD; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x handle 0x%x is dead (%d)", chan, portid, lp->handle, r); goto relogin; } isp_pdb_add_update(isp, chan, &pdb); continue; } relogin: if ((fcp->role & ISP_ROLE_INITIATOR) == 0) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x is not logged in", chan, portid); continue; } r = isp_gff_id(isp, chan, portid); if (r == 0) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x is not an FCP target", chan, portid); continue; } if (r < 0) r = isp_gft_id(isp, chan, portid); if (r == 0) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Port 0x%06x is not FCP", chan, portid); continue; } if (isp_login_device(isp, chan, portid, &pdb, &FCPARAM(isp, 0)->isp_lasthdl)) { if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) goto abort; continue; } isp_pdb_add_update(isp, chan, &pdb); } if (fcp->isp_loopstate < LOOP_SCANNING_FABRIC) goto abort; fcp->isp_loopstate = LOOP_FSCAN_DONE; isp_prt(isp, ISP_LOG_SANCFG, "Chan %d FC fabric scan done", chan); return (0); } /* * Find an unused handle and try and use to login to a port. */ static int isp_login_device(ispsoftc_t *isp, int chan, uint32_t portid, isp_pdb_t *p, uint16_t *ohp) { int lim, i, r; uint16_t handle; if (ISP_CAP_2KLOGIN(isp)) { lim = NPH_MAX_2K; } else { lim = NPH_MAX; } handle = isp_next_handle(isp, ohp); for (i = 0; i < lim; i++) { if (FCPARAM(isp, chan)->isp_loopstate != LOOP_SCANNING_FABRIC) return (-1); /* Check if this handle is free. */ r = isp_getpdb(isp, chan, handle, p); if (r == 0) { if (p->portid != portid) { /* This handle is busy, try next one. */ handle = isp_next_handle(isp, ohp); continue; } break; } if (FCPARAM(isp, chan)->isp_loopstate != LOOP_SCANNING_FABRIC) return (-1); /* * Now try and log into the device */ r = isp_plogx(isp, chan, handle, portid, PLOGX_FLG_CMD_PLOGI); if (r == 0) { break; } else if ((r & 0xffff) == MBOX_PORT_ID_USED) { /* * If we get here, then the firmwware still thinks we're logged into this device, but with a different * handle. We need to break that association. We used to try and just substitute the handle, but then * failed to get any data via isp_getpdb (below). */ if (isp_plogx(isp, chan, r >> 16, portid, PLOGX_FLG_CMD_LOGO | PLOGX_FLG_IMPLICIT | PLOGX_FLG_FREE_NPHDL)) { isp_prt(isp, ISP_LOGERR, "baw... logout of %x failed", r >> 16); } if (FCPARAM(isp, chan)->isp_loopstate != LOOP_SCANNING_FABRIC) return (-1); r = isp_plogx(isp, chan, handle, portid, PLOGX_FLG_CMD_PLOGI); if (r != 0) i = lim; break; } else if ((r & 0xffff) == MBOX_LOOP_ID_USED) { /* Try the next handle. */ handle = isp_next_handle(isp, ohp); } else { /* Give up. */ i = lim; break; } } if (i == lim) { isp_prt(isp, ISP_LOGWARN, "Chan %d PLOGI 0x%06x failed", chan, portid); return (-1); } /* * If we successfully logged into it, get the PDB for it * so we can crosscheck that it is still what we think it * is and that we also have the role it plays */ r = isp_getpdb(isp, chan, handle, p); if (r != 0) { isp_prt(isp, ISP_LOGERR, "Chan %d new device 0x%06x@0x%x disappeared", chan, portid, handle); return (-1); } if (p->handle != handle || p->portid != portid) { isp_prt(isp, ISP_LOGERR, "Chan %d new device 0x%06x@0x%x changed (0x%06x@0x%0x)", chan, portid, handle, p->portid, p->handle); return (-1); } return (0); } static int isp_send_change_request(ispsoftc_t *isp, int chan) { mbreg_t mbs; MBSINIT(&mbs, MBOX_SEND_CHANGE_REQUEST, MBLOGALL, 500000); mbs.param[1] = 0x03; mbs.param[9] = chan; isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { return (0); } else { isp_prt(isp, ISP_LOGWARN, "Chan %d Send Change Request: 0x%x", chan, mbs.param[0]); return (-1); } } static int isp_register_fc4_type(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); rft_id_t rp; ct_hdr_t *ct = &rp.rftid_hdr; uint8_t local[SNS_RFT_ID_REQ_SIZE]; sns_screq_t *reqp = (sns_screq_t *) local; uint8_t *scp = fcp->isp_scratch; if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } if (IS_24XX(isp)) { /* Build the CT command and execute via pass-through. */ ISP_MEMZERO(&rp, sizeof(rp)); ct->ct_revision = CT_REVISION; ct->ct_fcs_type = CT_FC_TYPE_FC; ct->ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct->ct_cmd_resp = SNS_RFT_ID; ct->ct_bcnt_resid = (sizeof (rft_id_t) - sizeof (ct_hdr_t)) >> 2; rp.rftid_portid[0] = fcp->isp_portid >> 16; rp.rftid_portid[1] = fcp->isp_portid >> 8; rp.rftid_portid[2] = fcp->isp_portid; rp.rftid_fc4types[FC4_SCSI >> 5] = 1 << (FC4_SCSI & 0x1f); isp_put_rft_id(isp, &rp, (rft_id_t *)scp); if (isp_ct_passthru(isp, chan, sizeof(rft_id_t), sizeof(ct_hdr_t))) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } } else { /* Build the SNS request and execute via firmware. */ ISP_MEMZERO((void *) reqp, SNS_RFT_ID_REQ_SIZE); reqp->snscb_rblen = sizeof (ct_hdr_t) >> 1; reqp->snscb_addr[RQRSP_ADDR0015] = DMA_WD0(fcp->isp_scdma); reqp->snscb_addr[RQRSP_ADDR1631] = DMA_WD1(fcp->isp_scdma); reqp->snscb_addr[RQRSP_ADDR3247] = DMA_WD2(fcp->isp_scdma); reqp->snscb_addr[RQRSP_ADDR4863] = DMA_WD3(fcp->isp_scdma); reqp->snscb_sblen = 22; reqp->snscb_data[0] = SNS_RFT_ID; reqp->snscb_data[4] = fcp->isp_portid & 0xffff; reqp->snscb_data[5] = (fcp->isp_portid >> 16) & 0xff; reqp->snscb_data[6] = (1 << FC4_SCSI); isp_put_sns_request(isp, reqp, (sns_screq_t *)scp); if (isp_ct_sns(isp, chan, SNS_RFT_ID_REQ_SIZE, sizeof(ct_hdr_t))) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } } isp_get_ct_hdr(isp, (ct_hdr_t *) scp, ct); FC_SCRATCH_RELEASE(isp, chan); if (ct->ct_cmd_resp == LS_RJT) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "Chan %d Register FC4 Type rejected", chan); return (-1); } else if (ct->ct_cmd_resp == LS_ACC) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Register FC4 Type accepted", chan); } else { isp_prt(isp, ISP_LOGWARN, "Chan %d Register FC4 Type: 0x%x", chan, ct->ct_cmd_resp); return (-1); } return (0); } static int isp_register_fc4_features_24xx(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t *ct; rff_id_t rp; uint8_t *scp = fcp->isp_scratch; if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } /* * Build the CT header and command in memory. */ ISP_MEMZERO(&rp, sizeof(rp)); ct = &rp.rffid_hdr; ct->ct_revision = CT_REVISION; ct->ct_fcs_type = CT_FC_TYPE_FC; ct->ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct->ct_cmd_resp = SNS_RFF_ID; ct->ct_bcnt_resid = (sizeof (rff_id_t) - sizeof (ct_hdr_t)) >> 2; rp.rffid_portid[0] = fcp->isp_portid >> 16; rp.rffid_portid[1] = fcp->isp_portid >> 8; rp.rffid_portid[2] = fcp->isp_portid; rp.rffid_fc4features = 0; if (fcp->role & ISP_ROLE_TARGET) rp.rffid_fc4features |= 1; if (fcp->role & ISP_ROLE_INITIATOR) rp.rffid_fc4features |= 2; rp.rffid_fc4type = FC4_SCSI; isp_put_rff_id(isp, &rp, (rff_id_t *)scp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "CT request", sizeof(rft_id_t), scp); if (isp_ct_passthru(isp, chan, sizeof(rft_id_t), sizeof(ct_hdr_t))) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } isp_get_ct_hdr(isp, (ct_hdr_t *) scp, ct); FC_SCRATCH_RELEASE(isp, chan); if (ct->ct_cmd_resp == LS_RJT) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "Chan %d Register FC4 Features rejected", chan); return (-1); } else if (ct->ct_cmd_resp == LS_ACC) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Register FC4 Features accepted", chan); } else { isp_prt(isp, ISP_LOGWARN, "Chan %d Register FC4 Features: 0x%x", chan, ct->ct_cmd_resp); return (-1); } return (0); } static int isp_register_port_name_24xx(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t *ct; rspn_id_t rp; uint8_t *scp = fcp->isp_scratch; int len; if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } /* * Build the CT header and command in memory. */ ISP_MEMZERO(&rp, sizeof(rp)); ct = &rp.rspnid_hdr; ct->ct_revision = CT_REVISION; ct->ct_fcs_type = CT_FC_TYPE_FC; ct->ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct->ct_cmd_resp = SNS_RSPN_ID; rp.rspnid_portid[0] = fcp->isp_portid >> 16; rp.rspnid_portid[1] = fcp->isp_portid >> 8; rp.rspnid_portid[2] = fcp->isp_portid; rp.rspnid_length = 0; len = offsetof(rspn_id_t, rspnid_name); mtx_lock(&prison0.pr_mtx); rp.rspnid_length += sprintf(&scp[len + rp.rspnid_length], "%s", prison0.pr_hostname[0] ? prison0.pr_hostname : "FreeBSD"); mtx_unlock(&prison0.pr_mtx); rp.rspnid_length += sprintf(&scp[len + rp.rspnid_length], ":%s", device_get_nameunit(isp->isp_dev)); if (chan != 0) { rp.rspnid_length += sprintf(&scp[len + rp.rspnid_length], "/%d", chan); } len += rp.rspnid_length; ct->ct_bcnt_resid = (len - sizeof(ct_hdr_t)) >> 2; isp_put_rspn_id(isp, &rp, (rspn_id_t *)scp); if (isp_ct_passthru(isp, chan, len, sizeof(ct_hdr_t))) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } isp_get_ct_hdr(isp, (ct_hdr_t *) scp, ct); FC_SCRATCH_RELEASE(isp, chan); if (ct->ct_cmd_resp == LS_RJT) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "Chan %d Register Symbolic Port Name rejected", chan); return (-1); } else if (ct->ct_cmd_resp == LS_ACC) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Register Symbolic Port Name accepted", chan); } else { isp_prt(isp, ISP_LOGWARN, "Chan %d Register Symbolic Port Name: 0x%x", chan, ct->ct_cmd_resp); return (-1); } return (0); } static int isp_register_node_name_24xx(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); ct_hdr_t *ct; rsnn_nn_t rp; uint8_t *scp = fcp->isp_scratch; int len; if (FC_SCRATCH_ACQUIRE(isp, chan)) { isp_prt(isp, ISP_LOGERR, sacq); return (-1); } /* * Build the CT header and command in memory. */ ISP_MEMZERO(&rp, sizeof(rp)); ct = &rp.rsnnnn_hdr; ct->ct_revision = CT_REVISION; ct->ct_fcs_type = CT_FC_TYPE_FC; ct->ct_fcs_subtype = CT_FC_SUBTYPE_NS; ct->ct_cmd_resp = SNS_RSNN_NN; MAKE_NODE_NAME_FROM_WWN(rp.rsnnnn_nodename, fcp->isp_wwnn); rp.rsnnnn_length = 0; len = offsetof(rsnn_nn_t, rsnnnn_name); mtx_lock(&prison0.pr_mtx); rp.rsnnnn_length += sprintf(&scp[len + rp.rsnnnn_length], "%s", prison0.pr_hostname[0] ? prison0.pr_hostname : "FreeBSD"); mtx_unlock(&prison0.pr_mtx); len += rp.rsnnnn_length; ct->ct_bcnt_resid = (len - sizeof(ct_hdr_t)) >> 2; isp_put_rsnn_nn(isp, &rp, (rsnn_nn_t *)scp); if (isp_ct_passthru(isp, chan, len, sizeof(ct_hdr_t))) { FC_SCRATCH_RELEASE(isp, chan); return (-1); } isp_get_ct_hdr(isp, (ct_hdr_t *) scp, ct); FC_SCRATCH_RELEASE(isp, chan); if (ct->ct_cmd_resp == LS_RJT) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOG_WARN1, "Chan %d Register Symbolic Node Name rejected", chan); return (-1); } else if (ct->ct_cmd_resp == LS_ACC) { isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Register Symbolic Node Name accepted", chan); } else { isp_prt(isp, ISP_LOGWARN, "Chan %d Register Symbolic Node Name: 0x%x", chan, ct->ct_cmd_resp); return (-1); } return (0); } static uint16_t isp_next_handle(ispsoftc_t *isp, uint16_t *ohp) { fcparam *fcp; int i, chan, wrap; uint16_t handle, minh, maxh; handle = *ohp; if (ISP_CAP_2KLOGIN(isp)) { minh = 0; maxh = NPH_RESERVED - 1; } else { minh = SNS_ID + 1; maxh = NPH_MAX - 1; } wrap = 0; next: if (handle == NIL_HANDLE) { handle = minh; } else { handle++; if (handle > maxh) { if (++wrap >= 2) { isp_prt(isp, ISP_LOGERR, "Out of port handles!"); return (NIL_HANDLE); } handle = minh; } } for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; for (i = 0; i < MAX_FC_TARG; i++) { if (fcp->portdb[i].state != FC_PORTDB_STATE_NIL && fcp->portdb[i].handle == handle) goto next; } } *ohp = handle; return (handle); } /* * Start a command. Locking is assumed done in the caller. */ int isp_start(XS_T *xs) { ispsoftc_t *isp; uint32_t cdblen; uint8_t local[QENTRY_LEN]; ispreq_t *reqp; void *cdbp, *qep; uint16_t *tptr; fcportdb_t *lp; int target, dmaresult; XS_INITERR(xs); isp = XS_ISP(xs); /* * Check command CDB length, etc.. We really are limited to 16 bytes * for Fibre Channel, but can do up to 44 bytes in parallel SCSI, * but probably only if we're running fairly new firmware (we'll * let the old f/w choke on an extended command queue entry). */ if (XS_CDBLEN(xs) > (IS_FC(isp)? 16 : 44) || XS_CDBLEN(xs) == 0) { isp_prt(isp, ISP_LOGERR, "unsupported cdb length (%d, CDB[0]=0x%x)", XS_CDBLEN(xs), XS_CDBP(xs)[0] & 0xff); XS_SETERR(xs, HBA_REQINVAL); return (CMD_COMPLETE); } /* * Translate the target to device handle as appropriate, checking * for correct device state as well. */ target = XS_TGT(xs); if (IS_FC(isp)) { fcparam *fcp = FCPARAM(isp, XS_CHANNEL(xs)); if ((fcp->role & ISP_ROLE_INITIATOR) == 0) { isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx I am not an initiator", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_SELTIMEOUT); return (CMD_COMPLETE); } if (isp->isp_state != ISP_RUNSTATE) { isp_prt(isp, ISP_LOGERR, "Adapter not at RUNSTATE"); XS_SETERR(xs, HBA_BOTCH); return (CMD_COMPLETE); } isp_prt(isp, ISP_LOGDEBUG2, "XS_TGT(xs)=%d", target); lp = &fcp->portdb[target]; if (target < 0 || target >= MAX_FC_TARG || lp->is_target == 0) { XS_SETERR(xs, HBA_SELTIMEOUT); return (CMD_COMPLETE); } if (fcp->isp_loopstate != LOOP_READY) { isp_prt(isp, ISP_LOGDEBUG1, "%d.%d.%jx loop is not ready", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs)); return (CMD_RQLATER); } if (lp->state == FC_PORTDB_STATE_ZOMBIE) { isp_prt(isp, ISP_LOGDEBUG1, "%d.%d.%jx target zombie", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs)); return (CMD_RQLATER); } if (lp->state != FC_PORTDB_STATE_VALID) { isp_prt(isp, ISP_LOGDEBUG1, "%d.%d.%jx bad db port state 0x%x", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs), lp->state); XS_SETERR(xs, HBA_SELTIMEOUT); return (CMD_COMPLETE); } } else { sdparam *sdp = SDPARAM(isp, XS_CHANNEL(xs)); if (isp->isp_state != ISP_RUNSTATE) { isp_prt(isp, ISP_LOGERR, "Adapter not at RUNSTATE"); XS_SETERR(xs, HBA_BOTCH); return (CMD_COMPLETE); } if (sdp->update) { isp_spi_update(isp, XS_CHANNEL(xs)); } lp = NULL; } start_again: qep = isp_getrqentry(isp); if (qep == NULL) { isp_prt(isp, ISP_LOG_WARN1, "Request Queue Overflow"); XS_SETERR(xs, HBA_BOTCH); return (CMD_EAGAIN); } XS_SETERR(xs, HBA_NOERROR); /* * Now see if we need to synchronize the ISP with respect to anything. * We do dual duty here (cough) for synchronizing for buses other * than which we got here to send a command to. */ reqp = (ispreq_t *) local; ISP_MEMZERO(local, QENTRY_LEN); if (ISP_TST_SENDMARKER(isp, XS_CHANNEL(xs))) { if (IS_24XX(isp)) { isp_marker_24xx_t *m = (isp_marker_24xx_t *) reqp; m->mrk_header.rqs_entry_count = 1; m->mrk_header.rqs_entry_type = RQSTYPE_MARKER; m->mrk_modifier = SYNC_ALL; m->mrk_vphdl = XS_CHANNEL(xs); isp_put_marker_24xx(isp, m, qep); } else { isp_marker_t *m = (isp_marker_t *) reqp; m->mrk_header.rqs_entry_count = 1; m->mrk_header.rqs_entry_type = RQSTYPE_MARKER; m->mrk_target = (XS_CHANNEL(xs) << 7); /* bus # */ m->mrk_modifier = SYNC_ALL; isp_put_marker(isp, m, qep); } ISP_SYNC_REQUEST(isp); ISP_SET_SENDMARKER(isp, XS_CHANNEL(xs), 0); goto start_again; } reqp->req_header.rqs_entry_count = 1; /* * Select and install Header Code. * Note that it might be overridden before going out * if we're on a 64 bit platform. The lower level * code (isp_send_cmd) will select the appropriate * 64 bit variant if it needs to. */ if (IS_24XX(isp)) { reqp->req_header.rqs_entry_type = RQSTYPE_T7RQS; } else if (IS_FC(isp)) { reqp->req_header.rqs_entry_type = RQSTYPE_T2RQS; } else { if (XS_CDBLEN(xs) > 12) { reqp->req_header.rqs_entry_type = RQSTYPE_CMDONLY; } else { reqp->req_header.rqs_entry_type = RQSTYPE_REQUEST; } } /* * Set task attributes */ if (IS_24XX(isp)) { int ttype; if (XS_TAG_P(xs)) { ttype = XS_TAG_TYPE(xs); } else { ttype = REQFLAG_STAG; } if (ttype == REQFLAG_OTAG) { ttype = FCP_CMND_TASK_ATTR_ORDERED; } else if (ttype == REQFLAG_HTAG) { ttype = FCP_CMND_TASK_ATTR_HEAD; } else { ttype = FCP_CMND_TASK_ATTR_SIMPLE; } ((ispreqt7_t *)reqp)->req_task_attribute = ttype; } else if (IS_FC(isp)) { /* * See comment in isp_intr_respq */ /* XS_SET_RESID(xs, 0); */ /* * Fibre Channel always requires some kind of tag. * The Qlogic drivers seem be happy not to use a tag, * but this breaks for some devices (IBM drives). */ if (XS_TAG_P(xs)) { ((ispreqt2_t *)reqp)->req_flags = XS_TAG_TYPE(xs); } else { ((ispreqt2_t *)reqp)->req_flags = REQFLAG_STAG; } } else { sdparam *sdp = SDPARAM(isp, XS_CHANNEL(xs)); if ((sdp->isp_devparam[target].actv_flags & DPARM_TQING) && XS_TAG_P(xs)) { reqp->req_flags = XS_TAG_TYPE(xs); } } /* * NB: we do not support long CDBs (yet) */ cdblen = XS_CDBLEN(xs); if (IS_SCSI(isp)) { if (cdblen > sizeof (reqp->req_cdb)) { isp_prt(isp, ISP_LOGERR, "Command Length %u too long for this chip", cdblen); XS_SETERR(xs, HBA_REQINVAL); return (CMD_COMPLETE); } reqp->req_target = target | (XS_CHANNEL(xs) << 7); reqp->req_lun_trn = XS_LUN(xs); reqp->req_cdblen = cdblen; tptr = &reqp->req_time; cdbp = reqp->req_cdb; } else if (IS_24XX(isp)) { ispreqt7_t *t7 = (ispreqt7_t *)local; if (cdblen > sizeof (t7->req_cdb)) { isp_prt(isp, ISP_LOGERR, "Command Length %u too long for this chip", cdblen); XS_SETERR(xs, HBA_REQINVAL); return (CMD_COMPLETE); } t7->req_nphdl = lp->handle; t7->req_tidlo = lp->portid; t7->req_tidhi = lp->portid >> 16; t7->req_vpidx = ISP_GET_VPIDX(isp, XS_CHANNEL(xs)); be64enc(t7->req_lun, CAM_EXTLUN_BYTE_SWIZZLE(XS_LUN(xs))); if (FCPARAM(isp, XS_CHANNEL(xs))->fctape_enabled && (lp->prli_word3 & PRLI_WD3_RETRY)) { if (FCP_NEXT_CRN(isp, &t7->req_crn, xs)) { isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx cannot generate next CRN", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_BOTCH); return (CMD_EAGAIN); } } tptr = &t7->req_time; cdbp = t7->req_cdb; } else { ispreqt2_t *t2 = (ispreqt2_t *)local; if (cdblen > sizeof t2->req_cdb) { isp_prt(isp, ISP_LOGERR, "Command Length %u too long for this chip", cdblen); XS_SETERR(xs, HBA_REQINVAL); return (CMD_COMPLETE); } if (FCPARAM(isp, XS_CHANNEL(xs))->fctape_enabled && (lp->prli_word3 & PRLI_WD3_RETRY)) { if (FCP_NEXT_CRN(isp, &t2->req_crn, xs)) { isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx cannot generate next CRN", XS_CHANNEL(xs), target, (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_BOTCH); return (CMD_EAGAIN); } } if (ISP_CAP_2KLOGIN(isp)) { ispreqt2e_t *t2e = (ispreqt2e_t *)local; t2e->req_target = lp->handle; t2e->req_scclun = XS_LUN(xs); tptr = &t2e->req_time; cdbp = t2e->req_cdb; } else if (ISP_CAP_SCCFW(isp)) { t2->req_target = lp->handle; t2->req_scclun = XS_LUN(xs); tptr = &t2->req_time; cdbp = t2->req_cdb; } else { t2->req_target = lp->handle; t2->req_lun_trn = XS_LUN(xs); tptr = &t2->req_time; cdbp = t2->req_cdb; } } *tptr = XS_TIME(xs); ISP_MEMCPY(cdbp, XS_CDBP(xs), cdblen); /* Whew. Thankfully the same for type 7 requests */ reqp->req_handle = isp_allocate_handle(isp, xs, ISP_HANDLE_INITIATOR); if (reqp->req_handle == 0) { isp_prt(isp, ISP_LOG_WARN1, "out of xflist pointers"); XS_SETERR(xs, HBA_BOTCH); return (CMD_EAGAIN); } /* * Set up DMA and/or do any platform dependent swizzling of the request entry * so that the Qlogic F/W understands what is being asked of it. * * The callee is responsible for adding all requests at this point. */ dmaresult = ISP_DMASETUP(isp, xs, reqp); if (dmaresult != CMD_QUEUED) { isp_destroy_handle(isp, reqp->req_handle); /* * dmasetup sets actual error in packet, and * return what we were given to return. */ return (dmaresult); } isp_xs_prt(isp, xs, ISP_LOGDEBUG0, "START cmd cdb[0]=0x%x datalen %ld", XS_CDBP(xs)[0], (long) XS_XFRLEN(xs)); return (CMD_QUEUED); } /* * isp control * Locks (ints blocked) assumed held. */ int isp_control(ispsoftc_t *isp, ispctl_t ctl, ...) { XS_T *xs; mbreg_t *mbr, mbs; int chan, tgt; uint32_t handle; va_list ap; switch (ctl) { case ISPCTL_RESET_BUS: /* * Issue a bus reset. */ if (IS_24XX(isp)) { isp_prt(isp, ISP_LOGERR, "BUS RESET NOT IMPLEMENTED"); break; } else if (IS_FC(isp)) { mbs.param[1] = 10; chan = 0; } else { va_start(ap, ctl); chan = va_arg(ap, int); va_end(ap); mbs.param[1] = SDPARAM(isp, chan)->isp_bus_reset_delay; if (mbs.param[1] < 2) { mbs.param[1] = 2; } mbs.param[2] = chan; } MBSINIT(&mbs, MBOX_BUS_RESET, MBLOGALL, 0); ISP_SET_SENDMARKER(isp, chan, 1); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { break; } isp_prt(isp, ISP_LOGINFO, "driver initiated bus reset of bus %d", chan); return (0); case ISPCTL_RESET_DEV: va_start(ap, ctl); chan = va_arg(ap, int); tgt = va_arg(ap, int); va_end(ap); if (IS_24XX(isp)) { uint8_t local[QENTRY_LEN]; isp24xx_tmf_t *tmf; isp24xx_statusreq_t *sp; fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; if (tgt < 0 || tgt >= MAX_FC_TARG) { isp_prt(isp, ISP_LOGWARN, "Chan %d trying to reset bad target %d", chan, tgt); break; } lp = &fcp->portdb[tgt]; if (lp->is_target == 0 || lp->state != FC_PORTDB_STATE_VALID) { isp_prt(isp, ISP_LOGWARN, "Chan %d abort of no longer valid target %d", chan, tgt); break; } tmf = (isp24xx_tmf_t *) local; ISP_MEMZERO(tmf, QENTRY_LEN); tmf->tmf_header.rqs_entry_type = RQSTYPE_TSK_MGMT; tmf->tmf_header.rqs_entry_count = 1; tmf->tmf_nphdl = lp->handle; tmf->tmf_delay = 2; tmf->tmf_timeout = 4; tmf->tmf_flags = ISP24XX_TMF_TARGET_RESET; tmf->tmf_tidlo = lp->portid; tmf->tmf_tidhi = lp->portid >> 16; tmf->tmf_vpidx = ISP_GET_VPIDX(isp, chan); isp_put_24xx_tmf(isp, tmf, isp->isp_iocb); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "TMF IOCB request", QENTRY_LEN, isp->isp_iocb); MEMORYBARRIER(isp, SYNC_IFORDEV, 0, QENTRY_LEN, chan); fcp->sendmarker = 1; isp_prt(isp, ISP_LOGALL, "Chan %d Reset N-Port Handle 0x%04x @ Port 0x%06x", chan, lp->handle, lp->portid); MBSINIT(&mbs, MBOX_EXEC_COMMAND_IOCB_A64, MBLOGALL, MBCMD_DEFAULT_TIMEOUT + tmf->tmf_timeout * 1000000); mbs.param[1] = QENTRY_LEN; mbs.param[2] = DMA_WD1(isp->isp_iocb_dma); mbs.param[3] = DMA_WD0(isp->isp_iocb_dma); mbs.param[6] = DMA_WD3(isp->isp_iocb_dma); mbs.param[7] = DMA_WD2(isp->isp_iocb_dma); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) break; MEMORYBARRIER(isp, SYNC_IFORCPU, QENTRY_LEN, QENTRY_LEN, chan); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "TMF IOCB response", QENTRY_LEN, &((isp24xx_statusreq_t *)isp->isp_iocb)[1]); sp = (isp24xx_statusreq_t *) local; isp_get_24xx_response(isp, &((isp24xx_statusreq_t *)isp->isp_iocb)[1], sp); if (sp->req_completion_status == 0) { return (0); } isp_prt(isp, ISP_LOGWARN, "Chan %d reset of target %d returned 0x%x", chan, tgt, sp->req_completion_status); break; } else if (IS_FC(isp)) { if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = tgt; mbs.ibits = (1 << 10); } else { mbs.param[1] = (tgt << 8); } } else { mbs.param[1] = (chan << 15) | (tgt << 8); } MBSINIT(&mbs, MBOX_ABORT_TARGET, MBLOGALL, 0); mbs.param[2] = 3; /* 'delay', in seconds */ isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { break; } isp_prt(isp, ISP_LOGINFO, "Target %d on Bus %d Reset Succeeded", tgt, chan); ISP_SET_SENDMARKER(isp, chan, 1); return (0); case ISPCTL_ABORT_CMD: va_start(ap, ctl); xs = va_arg(ap, XS_T *); va_end(ap); tgt = XS_TGT(xs); chan = XS_CHANNEL(xs); handle = isp_find_handle(isp, xs); if (handle == 0) { isp_prt(isp, ISP_LOGWARN, "cannot find handle for command to abort"); break; } if (IS_24XX(isp)) { isp24xx_abrt_t local, *ab = &local; fcparam *fcp; fcportdb_t *lp; fcp = FCPARAM(isp, chan); if (tgt < 0 || tgt >= MAX_FC_TARG) { isp_prt(isp, ISP_LOGWARN, "Chan %d trying to abort bad target %d", chan, tgt); break; } lp = &fcp->portdb[tgt]; if (lp->is_target == 0 || lp->state != FC_PORTDB_STATE_VALID) { isp_prt(isp, ISP_LOGWARN, "Chan %d abort of no longer valid target %d", chan, tgt); break; } isp_prt(isp, ISP_LOGALL, "Chan %d Abort Cmd for N-Port 0x%04x @ Port 0x%06x", chan, lp->handle, lp->portid); ISP_MEMZERO(ab, QENTRY_LEN); ab->abrt_header.rqs_entry_type = RQSTYPE_ABORT_IO; ab->abrt_header.rqs_entry_count = 1; ab->abrt_handle = lp->handle; ab->abrt_cmd_handle = handle; ab->abrt_tidlo = lp->portid; ab->abrt_tidhi = lp->portid >> 16; ab->abrt_vpidx = ISP_GET_VPIDX(isp, chan); isp_put_24xx_abrt(isp, ab, isp->isp_iocb); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "AB IOCB quest", QENTRY_LEN, isp->isp_iocb); MEMORYBARRIER(isp, SYNC_IFORDEV, 0, 2 * QENTRY_LEN, chan); ISP_MEMZERO(&mbs, sizeof (mbs)); MBSINIT(&mbs, MBOX_EXEC_COMMAND_IOCB_A64, MBLOGALL, 5000000); mbs.param[1] = QENTRY_LEN; mbs.param[2] = DMA_WD1(isp->isp_iocb_dma); mbs.param[3] = DMA_WD0(isp->isp_iocb_dma); mbs.param[6] = DMA_WD3(isp->isp_iocb_dma); mbs.param[7] = DMA_WD2(isp->isp_iocb_dma); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) break; MEMORYBARRIER(isp, SYNC_IFORCPU, QENTRY_LEN, QENTRY_LEN, chan); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "AB IOCB response", QENTRY_LEN, &((isp24xx_abrt_t *)isp->isp_iocb)[1]); isp_get_24xx_abrt(isp, &((isp24xx_abrt_t *)isp->isp_iocb)[1], ab); if (ab->abrt_nphdl == ISP24XX_ABRT_OKAY) { return (0); } isp_prt(isp, ISP_LOGWARN, "Chan %d handle %d abort returned 0x%x", chan, tgt, ab->abrt_nphdl); break; } else if (IS_FC(isp)) { if (ISP_CAP_SCCFW(isp)) { if (ISP_CAP_2KLOGIN(isp)) { mbs.param[1] = tgt; } else { mbs.param[1] = tgt << 8; } mbs.param[6] = XS_LUN(xs); } else { mbs.param[1] = tgt << 8 | XS_LUN(xs); } } else { mbs.param[1] = (chan << 15) | (tgt << 8) | XS_LUN(xs); } MBSINIT(&mbs, MBOX_ABORT, MBLOGALL & ~MBLOGMASK(MBOX_COMMAND_ERROR), 0); mbs.param[2] = handle; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { break; } return (0); case ISPCTL_UPDATE_PARAMS: va_start(ap, ctl); chan = va_arg(ap, int); va_end(ap); isp_spi_update(isp, chan); return (0); case ISPCTL_FCLINK_TEST: if (IS_FC(isp)) { int usdelay; va_start(ap, ctl); chan = va_arg(ap, int); usdelay = va_arg(ap, int); va_end(ap); if (usdelay == 0) { usdelay = 250000; } return (isp_fclink_test(isp, chan, usdelay)); } break; case ISPCTL_SCAN_FABRIC: if (IS_FC(isp)) { va_start(ap, ctl); chan = va_arg(ap, int); va_end(ap); return (isp_scan_fabric(isp, chan)); } break; case ISPCTL_SCAN_LOOP: if (IS_FC(isp)) { va_start(ap, ctl); chan = va_arg(ap, int); va_end(ap); return (isp_scan_loop(isp, chan)); } break; case ISPCTL_PDB_SYNC: if (IS_FC(isp)) { va_start(ap, ctl); chan = va_arg(ap, int); va_end(ap); return (isp_pdb_sync(isp, chan)); } break; case ISPCTL_SEND_LIP: if (IS_FC(isp) && !IS_24XX(isp)) { MBSINIT(&mbs, MBOX_INIT_LIP, MBLOGALL, 0); if (ISP_CAP_2KLOGIN(isp)) { mbs.ibits = (1 << 10); } isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { return (0); } } break; case ISPCTL_GET_PDB: if (IS_FC(isp)) { isp_pdb_t *pdb; va_start(ap, ctl); chan = va_arg(ap, int); tgt = va_arg(ap, int); pdb = va_arg(ap, isp_pdb_t *); va_end(ap); return (isp_getpdb(isp, chan, tgt, pdb)); } break; case ISPCTL_GET_NAMES: { uint64_t *wwnn, *wwnp; va_start(ap, ctl); chan = va_arg(ap, int); tgt = va_arg(ap, int); wwnn = va_arg(ap, uint64_t *); wwnp = va_arg(ap, uint64_t *); va_end(ap); if (wwnn == NULL && wwnp == NULL) { break; } if (wwnn) { *wwnn = isp_get_wwn(isp, chan, tgt, 1); if (*wwnn == INI_NONE) { break; } } if (wwnp) { *wwnp = isp_get_wwn(isp, chan, tgt, 0); if (*wwnp == INI_NONE) { break; } } return (0); } case ISPCTL_RUN_MBOXCMD: { va_start(ap, ctl); mbr = va_arg(ap, mbreg_t *); va_end(ap); isp_mboxcmd(isp, mbr); return (0); } case ISPCTL_PLOGX: { isp_plcmd_t *p; int r; va_start(ap, ctl); p = va_arg(ap, isp_plcmd_t *); va_end(ap); if ((p->flags & PLOGX_FLG_CMD_MASK) != PLOGX_FLG_CMD_PLOGI || (p->handle != NIL_HANDLE)) { return (isp_plogx(isp, p->channel, p->handle, p->portid, p->flags)); } do { isp_next_handle(isp, &p->handle); r = isp_plogx(isp, p->channel, p->handle, p->portid, p->flags); if ((r & 0xffff) == MBOX_PORT_ID_USED) { p->handle = r >> 16; r = 0; break; } } while ((r & 0xffff) == MBOX_LOOP_ID_USED); return (r); } case ISPCTL_CHANGE_ROLE: if (IS_FC(isp)) { int role, r; va_start(ap, ctl); chan = va_arg(ap, int); role = va_arg(ap, int); va_end(ap); r = isp_fc_change_role(isp, chan, role); return (r); } break; default: isp_prt(isp, ISP_LOGERR, "Unknown Control Opcode 0x%x", ctl); break; } return (-1); } /* * Interrupt Service Routine(s). * * External (OS) framework has done the appropriate locking, * and the locking will be held throughout this function. */ #ifdef ISP_TARGET_MODE void isp_intr_atioq(ispsoftc_t *isp) { uint8_t qe[QENTRY_LEN]; isphdr_t *hp; void *addr; uint32_t iptr, optr, oop; iptr = ISP_READ(isp, BIU2400_ATIO_RSPINP); optr = isp->isp_atioodx; while (optr != iptr) { oop = optr; MEMORYBARRIER(isp, SYNC_ATIOQ, oop, QENTRY_LEN, -1); addr = ISP_QUEUE_ENTRY(isp->isp_atioq, oop); isp_get_hdr(isp, addr, (isphdr_t *)qe); hp = (isphdr_t *)qe; switch (hp->rqs_entry_type) { case RQSTYPE_NOTIFY: case RQSTYPE_ATIO: (void) isp_target_notify(isp, addr, &oop); break; default: isp_print_qentry(isp, "?ATIOQ entry?", oop, addr); break; } optr = ISP_NXT_QENTRY(oop, RESULT_QUEUE_LEN(isp)); } if (isp->isp_atioodx != optr) { ISP_WRITE(isp, BIU2400_ATIO_RSPOUTP, optr); isp->isp_atioodx = optr; } } #endif void isp_intr_async(ispsoftc_t *isp, uint16_t event) { if (IS_FC(isp)) isp_parse_async_fc(isp, event); else isp_parse_async(isp, event); } void isp_intr_mbox(ispsoftc_t *isp, uint16_t mbox0) { int i, obits; if (!isp->isp_mboxbsy) { isp_prt(isp, ISP_LOGWARN, "mailbox 0x%x with no waiters", mbox0); return; } obits = isp->isp_obits; isp->isp_mboxtmp[0] = mbox0; for (i = 1; i < ISP_NMBOX(isp); i++) { if ((obits & (1 << i)) == 0) continue; isp->isp_mboxtmp[i] = ISP_READ(isp, MBOX_OFF(i)); } MBOX_NOTIFY_COMPLETE(isp); } void isp_intr_respq(ispsoftc_t *isp) { XS_T *xs, *cont_xs; uint8_t qe[QENTRY_LEN]; ispstatusreq_t *sp = (ispstatusreq_t *)qe; isp24xx_statusreq_t *sp2 = (isp24xx_statusreq_t *)qe; isphdr_t *hp; uint8_t *resp, *snsp; int buddaboom, completion_status, cont = 0, etype, i; int req_status_flags, req_state_flags, scsi_status; uint32_t iptr, junk, cptr, optr, rlen, slen, sptr, totslen, resid; /* * We can't be getting this now. */ if (isp->isp_state != ISP_RUNSTATE) { isp_prt(isp, ISP_LOGINFO, "respq interrupt when not ready"); return; } iptr = ISP_READ(isp, isp->isp_respinrp); /* Debounce the 2300 if revision less than 2. */ if (IS_2100(isp) || (IS_2300(isp) && isp->isp_revision < 2)) { do { junk = iptr; iptr = ISP_READ(isp, isp->isp_respinrp); } while (junk != iptr); } isp->isp_residx = iptr; optr = isp->isp_resodx; while (optr != iptr) { sptr = cptr = optr; hp = (isphdr_t *) ISP_QUEUE_ENTRY(isp->isp_result, cptr); optr = ISP_NXT_QENTRY(optr, RESULT_QUEUE_LEN(isp)); /* * Synchronize our view of this response queue entry. */ MEMORYBARRIER(isp, SYNC_RESULT, cptr, QENTRY_LEN, -1); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_qentry(isp, "Response Queue Entry", cptr, hp); isp_get_hdr(isp, hp, &sp->req_header); etype = sp->req_header.rqs_entry_type; /* We expected Status Continuation, but got different IOCB. */ if (cont > 0 && etype != RQSTYPE_STATUS_CONT) { cont = 0; isp_done(cont_xs); } if (IS_24XX(isp) && etype == RQSTYPE_RESPONSE) { isp_get_24xx_response(isp, (isp24xx_statusreq_t *)hp, sp2); scsi_status = sp2->req_scsi_status; completion_status = sp2->req_completion_status; req_status_flags = 0; if ((scsi_status & 0xff) != 0) req_state_flags = RQSF_GOT_STATUS; else req_state_flags = 0; resid = sp2->req_resid; } else if (etype == RQSTYPE_RESPONSE) { isp_get_response(isp, (ispstatusreq_t *) hp, sp); scsi_status = sp->req_scsi_status; completion_status = sp->req_completion_status; req_status_flags = sp->req_status_flags; req_state_flags = sp->req_state_flags; resid = sp->req_resid; } else if (etype == RQSTYPE_RIO1) { isp_rio1_t *rio = (isp_rio1_t *) qe; isp_get_rio1(isp, (isp_rio1_t *) hp, rio); for (i = 0; i < rio->req_header.rqs_seqno; i++) { isp_fastpost_complete(isp, rio->req_handles[i]); } ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ continue; } else if (etype == RQSTYPE_RIO2) { isp_prt(isp, ISP_LOGERR, "dropping RIO2 response"); ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ continue; } else if (etype == RQSTYPE_STATUS_CONT) { ispstatus_cont_t *scp = (ispstatus_cont_t *)qe; isp_get_cont_response(isp, (ispstatus_cont_t *)hp, scp); if (cont > 0) { i = min(cont, sizeof(scp->req_sense_data)); XS_SENSE_APPEND(cont_xs, scp->req_sense_data, i); cont -= i; if (cont == 0) { isp_done(cont_xs); } else { isp_prt(isp, ISP_LOGDEBUG0|ISP_LOG_CWARN, "Expecting Status Continuations for %u bytes", cont); } } else { isp_prt(isp, ISP_LOG_WARN1, "Ignored Continuation Response"); } ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ continue; } else if (isp_handle_other_response(isp, etype, hp, &cptr)) { /* More then one IOCB could be consumed. */ while (sptr != cptr) { ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ sptr = ISP_NXT_QENTRY(sptr, RESULT_QUEUE_LEN(isp)); hp = (isphdr_t *)ISP_QUEUE_ENTRY(isp->isp_result, sptr); } ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ optr = ISP_NXT_QENTRY(cptr, RESULT_QUEUE_LEN(isp)); continue; } else { /* We don't know what was this -- log and skip. */ isp_prt(isp, ISP_LOGERR, notresp, etype, cptr, optr); ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ continue; } buddaboom = 0; if (sp->req_header.rqs_flags & RQSFLAG_MASK) { if (sp->req_header.rqs_flags & RQSFLAG_CONTINUATION) { isp_print_qentry(isp, "unexpected continuation segment", cptr, hp); continue; } if (sp->req_header.rqs_flags & RQSFLAG_FULL) { isp_prt(isp, ISP_LOG_WARN1, "internal queues full"); /* * We'll synthesize a QUEUE FULL message below. */ } if (sp->req_header.rqs_flags & RQSFLAG_BADHEADER) { isp_print_qentry(isp, "bad header flag", cptr, hp); buddaboom++; } if (sp->req_header.rqs_flags & RQSFLAG_BADPACKET) { isp_print_qentry(isp, "bad request packet", cptr, hp); buddaboom++; } if (sp->req_header.rqs_flags & RQSFLAG_BADCOUNT) { isp_print_qentry(isp, "invalid entry count", cptr, hp); buddaboom++; } if (sp->req_header.rqs_flags & RQSFLAG_BADORDER) { isp_print_qentry(isp, "invalid IOCB ordering", cptr, hp); continue; } } xs = isp_find_xs(isp, sp->req_handle); if (xs == NULL) { uint8_t ts = completion_status & 0xff; /* * Only whine if this isn't the expected fallout of * aborting the command or resetting the target. */ if (etype != RQSTYPE_RESPONSE) { isp_prt(isp, ISP_LOGERR, "cannot find handle 0x%x (type 0x%x)", sp->req_handle, etype); } else if (ts != RQCS_ABORTED && ts != RQCS_RESET_OCCURRED) { isp_prt(isp, ISP_LOGERR, "cannot find handle 0x%x (status 0x%x)", sp->req_handle, ts); } ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ continue; } if (req_status_flags & RQSTF_BUS_RESET) { isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx bus was reset", XS_CHANNEL(xs), XS_TGT(xs), (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_BUSRESET); ISP_SET_SENDMARKER(isp, XS_CHANNEL(xs), 1); } if (buddaboom) { isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx buddaboom", XS_CHANNEL(xs), XS_TGT(xs), (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_BOTCH); } resp = snsp = NULL; rlen = slen = totslen = 0; if (IS_24XX(isp) && (scsi_status & (RQCS_RV|RQCS_SV)) != 0) { resp = sp2->req_rsp_sense; rlen = sp2->req_response_len; } else if (IS_FC(isp) && (scsi_status & RQCS_RV) != 0) { resp = sp->req_response; rlen = sp->req_response_len; } if (IS_FC(isp) && (scsi_status & RQCS_SV) != 0) { /* * Fibre Channel F/W doesn't say we got status * if there's Sense Data instead. I guess they * think it goes w/o saying. */ req_state_flags |= RQSF_GOT_STATUS|RQSF_GOT_SENSE; if (IS_24XX(isp)) { snsp = sp2->req_rsp_sense; snsp += rlen; totslen = sp2->req_sense_len; slen = sizeof(sp2->req_rsp_sense) - rlen; } else { snsp = sp->req_sense_data; totslen = sp->req_sense_len; slen = sizeof(sp->req_sense_data); } } else if (IS_SCSI(isp) && (req_state_flags & RQSF_GOT_SENSE)) { snsp = sp->req_sense_data; totslen = sp->req_sense_len; slen = sizeof (sp->req_sense_data); } if (slen > totslen) slen = totslen; if (req_state_flags & RQSF_GOT_STATUS) *XS_STSP(xs) = scsi_status & 0xff; if (rlen >= 4 && resp[FCP_RSPNS_CODE_OFFSET] != 0) { const char *ptr; char lb[64]; const char *rnames[10] = { "Task Management function complete", "FCP_DATA length different than FCP_BURST_LEN", "FCP_CMND fields invalid", "FCP_DATA parameter mismatch with FCP_DATA_RO", "Task Management function rejected", "Task Management function failed", NULL, NULL, "Task Management function succeeded", "Task Management function incorrect logical unit number", }; uint8_t code = resp[FCP_RSPNS_CODE_OFFSET]; if (code >= 10 || rnames[code] == NULL) { ISP_SNPRINTF(lb, sizeof(lb), "Unknown FCP Response Code 0x%x", code); ptr = lb; } else { ptr = rnames[code]; } isp_xs_prt(isp, xs, ISP_LOGWARN, "FCP RESPONSE, LENGTH %u: %s CDB0=0x%02x", rlen, ptr, XS_CDBP(xs)[0] & 0xff); if (code != 0 && code != 8) XS_SETERR(xs, HBA_BOTCH); } if (IS_24XX(isp)) isp_parse_status_24xx(isp, sp2, xs, &resid); else isp_parse_status(isp, sp, xs, &resid); if ((XS_NOERR(xs) || XS_ERR(xs) == HBA_NOERROR) && (*XS_STSP(xs) == SCSI_BUSY)) XS_SETERR(xs, HBA_TGTBSY); if (IS_SCSI(isp)) { XS_SET_RESID(xs, resid); /* * A new synchronous rate was negotiated for * this target. Mark state such that we'll go * look up that which has changed later. */ if (req_status_flags & RQSTF_NEGOTIATION) { int t = XS_TGT(xs); sdparam *sdp = SDPARAM(isp, XS_CHANNEL(xs)); sdp->isp_devparam[t].dev_refresh = 1; sdp->update = 1; } } else { if (req_status_flags & RQSF_XFER_COMPLETE) { XS_SET_RESID(xs, 0); } else if (scsi_status & RQCS_RESID) { XS_SET_RESID(xs, resid); } else { XS_SET_RESID(xs, 0); } } if (slen > 0) { XS_SAVE_SENSE(xs, snsp, slen); if (totslen > slen) { cont = totslen - slen; cont_xs = xs; isp_prt(isp, ISP_LOGDEBUG0|ISP_LOG_CWARN, "Expecting Status Continuations for %u bytes", cont); } } isp_prt(isp, ISP_LOGDEBUG2, "asked for %lu got raw resid %lu settled for %lu", (u_long)XS_XFRLEN(xs), (u_long)resid, (u_long)XS_GET_RESID(xs)); if (XS_XFRLEN(xs)) ISP_DMAFREE(isp, xs, sp->req_handle); isp_destroy_handle(isp, sp->req_handle); ISP_MEMZERO(hp, QENTRY_LEN); /* PERF */ /* Complete command if we expect no Status Continuations. */ if (cont == 0) isp_done(xs); } /* We haven't received all Status Continuations, but that is it. */ if (cont > 0) isp_done(cont_xs); /* If we processed any IOCBs, let ISP know about it. */ if (optr != isp->isp_resodx) { ISP_WRITE(isp, isp->isp_respoutrp, optr); isp->isp_resodx = optr; } } /* * Parse an ASYNC mailbox complete */ static void isp_parse_async(ispsoftc_t *isp, uint16_t mbox) { uint32_t h1 = 0, h2 = 0; uint16_t chan = 0; /* * Pick up the channel, but not if this is a ASYNC_RIO32_2, * where Mailboxes 6/7 have the second handle. */ if (mbox != ASYNC_RIO32_2) { if (IS_DUALBUS(isp)) { chan = ISP_READ(isp, OUTMAILBOX6); } } isp_prt(isp, ISP_LOGDEBUG2, "Async Mbox 0x%x", mbox); switch (mbox) { case ASYNC_BUS_RESET: ISP_SET_SENDMARKER(isp, chan, 1); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif isp_async(isp, ISPASYNC_BUS_RESET, chan); break; case ASYNC_SYSTEM_ERROR: isp->isp_state = ISP_CRASHED; /* * Were we waiting for a mailbox command to complete? * If so, it's dead, so wake up the waiter. */ if (isp->isp_mboxbsy) { isp->isp_obits = 1; isp->isp_mboxtmp[0] = MBOX_HOST_INTERFACE_ERROR; MBOX_NOTIFY_COMPLETE(isp); } /* * It's up to the handler for isp_async to reinit stuff and * restart the firmware */ isp_async(isp, ISPASYNC_FW_CRASH); break; case ASYNC_RQS_XFER_ERR: isp_prt(isp, ISP_LOGERR, "Request Queue Transfer Error"); break; case ASYNC_RSP_XFER_ERR: isp_prt(isp, ISP_LOGERR, "Response Queue Transfer Error"); break; case ASYNC_QWAKEUP: /* * We've just been notified that the Queue has woken up. * We don't need to be chatty about this- just unlatch things * and move on. */ mbox = ISP_READ(isp, isp->isp_rqstoutrp); break; case ASYNC_TIMEOUT_RESET: isp_prt(isp, ISP_LOGWARN, "timeout initiated SCSI bus reset of chan %d", chan); ISP_SET_SENDMARKER(isp, chan, 1); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif break; case ASYNC_DEVICE_RESET: isp_prt(isp, ISP_LOGINFO, "device reset on chan %d", chan); ISP_SET_SENDMARKER(isp, chan, 1); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif break; case ASYNC_EXTMSG_UNDERRUN: isp_prt(isp, ISP_LOGWARN, "extended message underrun"); break; case ASYNC_SCAM_INT: isp_prt(isp, ISP_LOGINFO, "SCAM interrupt"); break; case ASYNC_HUNG_SCSI: isp_prt(isp, ISP_LOGERR, "stalled SCSI Bus after DATA Overrun"); /* XXX: Need to issue SCSI reset at this point */ break; case ASYNC_KILLED_BUS: isp_prt(isp, ISP_LOGERR, "SCSI Bus reset after DATA Overrun"); break; case ASYNC_BUS_TRANSIT: mbox = ISP_READ(isp, OUTMAILBOX2); switch (mbox & SXP_PINS_MODE_MASK) { case SXP_PINS_LVD_MODE: isp_prt(isp, ISP_LOGINFO, "Transition to LVD mode"); SDPARAM(isp, chan)->isp_diffmode = 0; SDPARAM(isp, chan)->isp_ultramode = 0; SDPARAM(isp, chan)->isp_lvdmode = 1; break; case SXP_PINS_HVD_MODE: isp_prt(isp, ISP_LOGINFO, "Transition to Differential mode"); SDPARAM(isp, chan)->isp_diffmode = 1; SDPARAM(isp, chan)->isp_ultramode = 0; SDPARAM(isp, chan)->isp_lvdmode = 0; break; case SXP_PINS_SE_MODE: isp_prt(isp, ISP_LOGINFO, "Transition to Single Ended mode"); SDPARAM(isp, chan)->isp_diffmode = 0; SDPARAM(isp, chan)->isp_ultramode = 1; SDPARAM(isp, chan)->isp_lvdmode = 0; break; default: isp_prt(isp, ISP_LOGWARN, "Transition to Unknown Mode 0x%x", mbox); break; } /* * XXX: Set up to renegotiate again! */ /* Can only be for a 1080... */ ISP_SET_SENDMARKER(isp, chan, 1); break; case ASYNC_CMD_CMPLT: case ASYNC_RIO32_1: if (!IS_ULTRA3(isp)) { isp_prt(isp, ISP_LOGERR, "unexpected fast posting completion"); break; } /* FALLTHROUGH */ h1 = (ISP_READ(isp, OUTMAILBOX2) << 16) | ISP_READ(isp, OUTMAILBOX1); break; case ASYNC_RIO32_2: h1 = (ISP_READ(isp, OUTMAILBOX2) << 16) | ISP_READ(isp, OUTMAILBOX1); h2 = (ISP_READ(isp, OUTMAILBOX7) << 16) | ISP_READ(isp, OUTMAILBOX6); break; case ASYNC_RIO16_5: case ASYNC_RIO16_4: case ASYNC_RIO16_3: case ASYNC_RIO16_2: case ASYNC_RIO16_1: isp_prt(isp, ISP_LOGERR, "unexpected 16 bit RIO handle"); break; default: isp_prt(isp, ISP_LOGWARN, "%s: unhandled async code 0x%x", __func__, mbox); break; } if (h1 || h2) { isp_prt(isp, ISP_LOGDEBUG3, "fast post/rio completion of 0x%08x", h1); isp_fastpost_complete(isp, h1); if (h2) { isp_prt(isp, ISP_LOGDEBUG3, "fast post/rio completion of 0x%08x", h2); isp_fastpost_complete(isp, h2); } } } static void isp_parse_async_fc(ispsoftc_t *isp, uint16_t mbox) { fcparam *fcp; uint16_t chan; if (IS_DUALBUS(isp)) { chan = ISP_READ(isp, OUTMAILBOX6); } else { chan = 0; } isp_prt(isp, ISP_LOGDEBUG2, "Async Mbox 0x%x", mbox); switch (mbox) { case ASYNC_SYSTEM_ERROR: isp->isp_state = ISP_CRASHED; FCPARAM(isp, chan)->isp_loopstate = LOOP_NIL; isp_change_fw_state(isp, chan, FW_CONFIG_WAIT); /* * Were we waiting for a mailbox command to complete? * If so, it's dead, so wake up the waiter. */ if (isp->isp_mboxbsy) { isp->isp_obits = 1; isp->isp_mboxtmp[0] = MBOX_HOST_INTERFACE_ERROR; MBOX_NOTIFY_COMPLETE(isp); } /* * It's up to the handler for isp_async to reinit stuff and * restart the firmware */ isp_async(isp, ISPASYNC_FW_CRASH); break; case ASYNC_RQS_XFER_ERR: isp_prt(isp, ISP_LOGERR, "Request Queue Transfer Error"); break; case ASYNC_RSP_XFER_ERR: isp_prt(isp, ISP_LOGERR, "Response Queue Transfer Error"); break; case ASYNC_QWAKEUP: #ifdef ISP_TARGET_MODE if (IS_24XX(isp)) { isp_prt(isp, ISP_LOGERR, "ATIO Queue Transfer Error"); break; } #endif isp_prt(isp, ISP_LOGERR, "%s: unexpected ASYNC_QWAKEUP code", __func__); break; case ASYNC_CMD_CMPLT: isp_fastpost_complete(isp, (ISP_READ(isp, OUTMAILBOX2) << 16) | ISP_READ(isp, OUTMAILBOX1)); break; case ASYNC_RIOZIO_STALL: isp_intr_respq(isp); break; case ASYNC_CTIO_DONE: #ifdef ISP_TARGET_MODE isp_target_async(isp, (ISP_READ(isp, OUTMAILBOX2) << 16) | ISP_READ(isp, OUTMAILBOX1), mbox); #else isp_prt(isp, ISP_LOGWARN, "unexpected ASYNC CTIO done"); #endif break; case ASYNC_LIP_ERROR: case ASYNC_LIP_NOS_OLS_RECV: case ASYNC_LIP_OCCURRED: case ASYNC_PTPMODE: /* * These are broadcast events that have to be sent across * all active channels. */ for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); int topo = fcp->isp_topo; if (fcp->role == ISP_ROLE_NONE) continue; if (fcp->isp_loopstate > LOOP_HAVE_LINK) fcp->isp_loopstate = LOOP_HAVE_LINK; ISP_SET_SENDMARKER(isp, chan, 1); isp_async(isp, ISPASYNC_LIP, chan); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif /* * We've had problems with data corruption occurring on * commands that complete (with no apparent error) after * we receive a LIP. This has been observed mostly on * Local Loop topologies. To be safe, let's just mark * all active initiator commands as dead. */ if (topo == TOPO_NL_PORT || topo == TOPO_FL_PORT) { int i, j; for (i = j = 0; i < isp->isp_maxcmds; i++) { XS_T *xs; isp_hdl_t *hdp; hdp = &isp->isp_xflist[i]; if (ISP_H2HT(hdp->handle) != ISP_HANDLE_INITIATOR) { continue; } xs = hdp->cmd; if (XS_CHANNEL(xs) != chan) { continue; } j++; isp_prt(isp, ISP_LOG_WARN1, "%d.%d.%jx bus reset set at %s:%u", XS_CHANNEL(xs), XS_TGT(xs), (uintmax_t)XS_LUN(xs), __func__, __LINE__); XS_SETERR(xs, HBA_BUSRESET); } if (j) { isp_prt(isp, ISP_LOGERR, lipd, chan, j); } } } break; case ASYNC_LOOP_UP: /* * This is a broadcast event that has to be sent across * all active channels. */ for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; fcp->isp_linkstate = 1; if (fcp->isp_loopstate < LOOP_HAVE_LINK) fcp->isp_loopstate = LOOP_HAVE_LINK; ISP_SET_SENDMARKER(isp, chan, 1); isp_async(isp, ISPASYNC_LOOP_UP, chan); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif } break; case ASYNC_LOOP_DOWN: /* * This is a broadcast event that has to be sent across * all active channels. */ for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; ISP_SET_SENDMARKER(isp, chan, 1); fcp->isp_linkstate = 0; fcp->isp_loopstate = LOOP_NIL; isp_async(isp, ISPASYNC_LOOP_DOWN, chan); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif } break; case ASYNC_LOOP_RESET: /* * This is a broadcast event that has to be sent across * all active channels. */ for (chan = 0; chan < isp->isp_nchan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; ISP_SET_SENDMARKER(isp, chan, 1); if (fcp->isp_loopstate > LOOP_HAVE_LINK) fcp->isp_loopstate = LOOP_HAVE_LINK; isp_async(isp, ISPASYNC_LOOP_RESET, chan); #ifdef ISP_TARGET_MODE isp_target_async(isp, chan, mbox); #endif } break; case ASYNC_PDB_CHANGED: { int echan, nphdl, nlstate, reason; if (IS_23XX(isp) || IS_24XX(isp)) { nphdl = ISP_READ(isp, OUTMAILBOX1); nlstate = ISP_READ(isp, OUTMAILBOX2); } else { nphdl = nlstate = 0xffff; } if (IS_24XX(isp)) reason = ISP_READ(isp, OUTMAILBOX3) >> 8; else reason = 0xff; if (ISP_CAP_MULTI_ID(isp)) { chan = ISP_READ(isp, OUTMAILBOX3) & 0xff; if (chan == 0xff || nphdl == NIL_HANDLE) { chan = 0; echan = isp->isp_nchan - 1; } else if (chan >= isp->isp_nchan) { break; } else { echan = chan; } } else { chan = echan = 0; } for (; chan <= echan; chan++) { fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; if (fcp->isp_loopstate > LOOP_LTEST_DONE) { if (nphdl != NIL_HANDLE && nphdl == fcp->isp_login_hdl && reason == PDB24XX_AE_OPN_2) continue; fcp->isp_loopstate = LOOP_LTEST_DONE; } else if (fcp->isp_loopstate < LOOP_HAVE_LINK) fcp->isp_loopstate = LOOP_HAVE_LINK; isp_async(isp, ISPASYNC_CHANGE_NOTIFY, chan, ISPASYNC_CHANGE_PDB, nphdl, nlstate, reason); } break; } case ASYNC_CHANGE_NOTIFY: { int portid; portid = ((ISP_READ(isp, OUTMAILBOX1) & 0xff) << 16) | ISP_READ(isp, OUTMAILBOX2); if (ISP_CAP_MULTI_ID(isp)) { chan = ISP_READ(isp, OUTMAILBOX3) & 0xff; if (chan >= isp->isp_nchan) break; } else { chan = 0; } fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) break; if (fcp->isp_loopstate > LOOP_LTEST_DONE) fcp->isp_loopstate = LOOP_LTEST_DONE; else if (fcp->isp_loopstate < LOOP_HAVE_LINK) fcp->isp_loopstate = LOOP_HAVE_LINK; isp_async(isp, ISPASYNC_CHANGE_NOTIFY, chan, ISPASYNC_CHANGE_SNS, portid); break; } case ASYNC_ERR_LOGGING_DISABLED: isp_prt(isp, ISP_LOGWARN, "Error logging disabled (reason 0x%x)", ISP_READ(isp, OUTMAILBOX1)); break; case ASYNC_CONNMODE: /* * This only applies to 2100 amd 2200 cards */ if (!IS_2200(isp) && !IS_2100(isp)) { isp_prt(isp, ISP_LOGWARN, "bad card for ASYNC_CONNMODE event"); break; } chan = 0; mbox = ISP_READ(isp, OUTMAILBOX1); switch (mbox) { case ISP_CONN_LOOP: isp_prt(isp, ISP_LOGINFO, "Point-to-Point -> Loop mode"); break; case ISP_CONN_PTP: isp_prt(isp, ISP_LOGINFO, "Loop -> Point-to-Point mode"); break; case ISP_CONN_BADLIP: isp_prt(isp, ISP_LOGWARN, "Point-to-Point -> Loop mode (BAD LIP)"); break; case ISP_CONN_FATAL: isp->isp_state = ISP_CRASHED; isp_prt(isp, ISP_LOGERR, "FATAL CONNECTION ERROR"); isp_async(isp, ISPASYNC_FW_CRASH); return; case ISP_CONN_LOOPBACK: isp_prt(isp, ISP_LOGWARN, "Looped Back in Point-to-Point mode"); break; default: isp_prt(isp, ISP_LOGWARN, "Unknown connection mode (0x%x)", mbox); break; } ISP_SET_SENDMARKER(isp, chan, 1); FCPARAM(isp, chan)->isp_loopstate = LOOP_HAVE_LINK; isp_async(isp, ISPASYNC_CHANGE_NOTIFY, chan, ISPASYNC_CHANGE_OTHER); break; case ASYNC_P2P_INIT_ERR: isp_prt(isp, ISP_LOGWARN, "P2P init error (reason 0x%x)", ISP_READ(isp, OUTMAILBOX1)); break; case ASYNC_RCV_ERR: if (IS_24XX(isp)) { isp_prt(isp, ISP_LOGWARN, "Receive Error"); } else { isp_prt(isp, ISP_LOGWARN, "unexpected ASYNC_RCV_ERR"); } break; case ASYNC_RJT_SENT: /* same as ASYNC_QFULL_SENT */ if (IS_24XX(isp)) { isp_prt(isp, ISP_LOGTDEBUG0, "LS_RJT sent"); break; } else { isp_prt(isp, ISP_LOGTDEBUG0, "QFULL sent"); break; } case ASYNC_FW_RESTART_COMPLETE: isp_prt(isp, ISP_LOGDEBUG0, "FW restart complete"); break; case ASYNC_TEMPERATURE_ALERT: isp_prt(isp, ISP_LOGERR, "Temperature alert (subcode 0x%x)", ISP_READ(isp, OUTMAILBOX1)); break; case ASYNC_TRANSCEIVER_INSERTION: isp_prt(isp, ISP_LOGDEBUG0, "Transceiver insertion (0x%x)", ISP_READ(isp, OUTMAILBOX1)); break; case ASYNC_TRANSCEIVER_REMOVAL: isp_prt(isp, ISP_LOGDEBUG0, "Transceiver removal"); break; case ASYNC_AUTOLOAD_FW_COMPLETE: isp_prt(isp, ISP_LOGDEBUG0, "Autoload FW init complete"); break; case ASYNC_AUTOLOAD_FW_FAILURE: isp_prt(isp, ISP_LOGERR, "Autoload FW init failure"); break; default: isp_prt(isp, ISP_LOGWARN, "Unknown Async Code 0x%x", mbox); break; } } /* * Handle other response entries. A pointer to the request queue output * index is here in case we want to eat several entries at once, although * this is not used currently. */ static int isp_handle_other_response(ispsoftc_t *isp, int type, isphdr_t *hp, uint32_t *optrp) { isp_ridacq_t rid; int chan, c; uint32_t hdl, portid; void *ptr; switch (type) { case RQSTYPE_MARKER: isp_prt(isp, ISP_LOG_WARN1, "Marker Response"); return (1); case RQSTYPE_RPT_ID_ACQ: isp_get_ridacq(isp, (isp_ridacq_t *)hp, &rid); portid = (uint32_t)rid.ridacq_vp_port_hi << 16 | rid.ridacq_vp_port_lo; if (rid.ridacq_format == 0) { for (chan = 0; chan < isp->isp_nchan; chan++) { fcparam *fcp = FCPARAM(isp, chan); if (fcp->role == ISP_ROLE_NONE) continue; c = (chan == 0) ? 127 : (chan - 1); if (rid.ridacq_map[c / 16] & (1 << (c % 16)) || chan == 0) { fcp->isp_loopstate = LOOP_HAVE_LINK; isp_async(isp, ISPASYNC_CHANGE_NOTIFY, chan, ISPASYNC_CHANGE_OTHER); } else { fcp->isp_loopstate = LOOP_NIL; isp_async(isp, ISPASYNC_LOOP_DOWN, chan); } } } else { fcparam *fcp = FCPARAM(isp, rid.ridacq_vp_index); if (rid.ridacq_vp_status == RIDACQ_STS_COMPLETE || rid.ridacq_vp_status == RIDACQ_STS_CHANGED) { fcp->isp_topo = (rid.ridacq_map[0] >> 9) & 0x7; fcp->isp_portid = portid; fcp->isp_loopstate = LOOP_HAVE_ADDR; isp_async(isp, ISPASYNC_CHANGE_NOTIFY, rid.ridacq_vp_index, ISPASYNC_CHANGE_OTHER); } else { fcp->isp_loopstate = LOOP_NIL; isp_async(isp, ISPASYNC_LOOP_DOWN, rid.ridacq_vp_index); } } return (1); case RQSTYPE_CT_PASSTHRU: case RQSTYPE_VP_MODIFY: case RQSTYPE_VP_CTRL: case RQSTYPE_LOGIN: ISP_IOXGET_32(isp, (uint32_t *)(hp + 1), hdl); ptr = isp_find_xs(isp, hdl); if (ptr != NULL) { isp_destroy_handle(isp, hdl); memcpy(ptr, hp, QENTRY_LEN); wakeup(ptr); } return (1); case RQSTYPE_ATIO: case RQSTYPE_CTIO: case RQSTYPE_NOTIFY: case RQSTYPE_NOTIFY_ACK: case RQSTYPE_CTIO1: case RQSTYPE_ATIO2: case RQSTYPE_CTIO2: case RQSTYPE_CTIO3: case RQSTYPE_CTIO7: case RQSTYPE_ABTS_RCVD: case RQSTYPE_ABTS_RSP: #ifdef ISP_TARGET_MODE return (isp_target_notify(isp, (ispstatusreq_t *) hp, optrp)); #endif /* FALLTHROUGH */ case RQSTYPE_REQUEST: default: return (0); } } static void isp_parse_status(ispsoftc_t *isp, ispstatusreq_t *sp, XS_T *xs, uint32_t *rp) { switch (sp->req_completion_status & 0xff) { case RQCS_COMPLETE: if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); } return; case RQCS_INCOMPLETE: if ((sp->req_state_flags & RQSF_GOT_TARGET) == 0) { isp_xs_prt(isp, xs, ISP_LOG_WARN1, "Selection Timeout @ %s:%d", __func__, __LINE__); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_SELTIMEOUT); *rp = XS_XFRLEN(xs); } return; } isp_xs_prt(isp, xs, ISP_LOGERR, "Command Incomplete, state 0x%x", sp->req_state_flags); break; case RQCS_DMA_ERROR: isp_xs_prt(isp, xs, ISP_LOGERR, "DMA Error"); *rp = XS_XFRLEN(xs); break; case RQCS_TRANSPORT_ERROR: { char buf[172]; ISP_SNPRINTF(buf, sizeof (buf), "states=>"); if (sp->req_state_flags & RQSF_GOT_BUS) { ISP_SNPRINTF(buf, sizeof (buf), "%s GOT_BUS", buf); } if (sp->req_state_flags & RQSF_GOT_TARGET) { ISP_SNPRINTF(buf, sizeof (buf), "%s GOT_TGT", buf); } if (sp->req_state_flags & RQSF_SENT_CDB) { ISP_SNPRINTF(buf, sizeof (buf), "%s SENT_CDB", buf); } if (sp->req_state_flags & RQSF_XFRD_DATA) { ISP_SNPRINTF(buf, sizeof (buf), "%s XFRD_DATA", buf); } if (sp->req_state_flags & RQSF_GOT_STATUS) { ISP_SNPRINTF(buf, sizeof (buf), "%s GOT_STS", buf); } if (sp->req_state_flags & RQSF_GOT_SENSE) { ISP_SNPRINTF(buf, sizeof (buf), "%s GOT_SNS", buf); } if (sp->req_state_flags & RQSF_XFER_COMPLETE) { ISP_SNPRINTF(buf, sizeof (buf), "%s XFR_CMPLT", buf); } ISP_SNPRINTF(buf, sizeof (buf), "%s\nstatus=>", buf); if (sp->req_status_flags & RQSTF_DISCONNECT) { ISP_SNPRINTF(buf, sizeof (buf), "%s Disconnect", buf); } if (sp->req_status_flags & RQSTF_SYNCHRONOUS) { ISP_SNPRINTF(buf, sizeof (buf), "%s Sync_xfr", buf); } if (sp->req_status_flags & RQSTF_PARITY_ERROR) { ISP_SNPRINTF(buf, sizeof (buf), "%s Parity", buf); } if (sp->req_status_flags & RQSTF_BUS_RESET) { ISP_SNPRINTF(buf, sizeof (buf), "%s Bus_Reset", buf); } if (sp->req_status_flags & RQSTF_DEVICE_RESET) { ISP_SNPRINTF(buf, sizeof (buf), "%s Device_Reset", buf); } if (sp->req_status_flags & RQSTF_ABORTED) { ISP_SNPRINTF(buf, sizeof (buf), "%s Aborted", buf); } if (sp->req_status_flags & RQSTF_TIMEOUT) { ISP_SNPRINTF(buf, sizeof (buf), "%s Timeout", buf); } if (sp->req_status_flags & RQSTF_NEGOTIATION) { ISP_SNPRINTF(buf, sizeof (buf), "%s Negotiation", buf); } isp_xs_prt(isp, xs, ISP_LOGERR, "Transport Error: %s", buf); *rp = XS_XFRLEN(xs); break; } case RQCS_RESET_OCCURRED: { int chan; isp_xs_prt(isp, xs, ISP_LOGWARN, "Bus Reset destroyed command"); for (chan = 0; chan < isp->isp_nchan; chan++) { FCPARAM(isp, chan)->sendmarker = 1; } if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BUSRESET); } *rp = XS_XFRLEN(xs); return; } case RQCS_ABORTED: isp_xs_prt(isp, xs, ISP_LOGERR, "Command Aborted"); ISP_SET_SENDMARKER(isp, XS_CHANNEL(xs), 1); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_ABORTED); } return; case RQCS_TIMEOUT: isp_xs_prt(isp, xs, ISP_LOGWARN, "Command timed out"); /* * XXX: Check to see if we logged out of the device. */ if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_CMDTIMEOUT); } return; case RQCS_DATA_OVERRUN: XS_SET_RESID(xs, sp->req_resid); isp_xs_prt(isp, xs, ISP_LOGERR, "data overrun (%ld)", (long) XS_GET_RESID(xs)); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_DATAOVR); } return; case RQCS_COMMAND_OVERRUN: isp_xs_prt(isp, xs, ISP_LOGERR, "command overrun"); break; case RQCS_STATUS_OVERRUN: isp_xs_prt(isp, xs, ISP_LOGERR, "status overrun"); break; case RQCS_BAD_MESSAGE: isp_xs_prt(isp, xs, ISP_LOGERR, "msg not COMMAND COMPLETE after status"); break; case RQCS_NO_MESSAGE_OUT: isp_xs_prt(isp, xs, ISP_LOGERR, "No MESSAGE OUT phase after selection"); break; case RQCS_EXT_ID_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "EXTENDED IDENTIFY failed"); break; case RQCS_IDE_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "INITIATOR DETECTED ERROR rejected"); break; case RQCS_ABORT_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "ABORT OPERATION rejected"); break; case RQCS_REJECT_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "MESSAGE REJECT rejected"); break; case RQCS_NOP_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "NOP rejected"); break; case RQCS_PARITY_ERROR_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "MESSAGE PARITY ERROR rejected"); break; case RQCS_DEVICE_RESET_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGWARN, "BUS DEVICE RESET rejected"); break; case RQCS_ID_MSG_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "IDENTIFY rejected"); break; case RQCS_UNEXP_BUS_FREE: isp_xs_prt(isp, xs, ISP_LOGERR, "Unexpected Bus Free"); break; case RQCS_DATA_UNDERRUN: { if (IS_FC(isp)) { int ru_marked = (sp->req_scsi_status & RQCS_RU) != 0; if (!ru_marked || sp->req_resid > XS_XFRLEN(xs)) { isp_xs_prt(isp, xs, ISP_LOGWARN, bun, XS_XFRLEN(xs), sp->req_resid, (ru_marked)? "marked" : "not marked"); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BOTCH); } return; } } XS_SET_RESID(xs, sp->req_resid); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); } return; } case RQCS_XACT_ERR1: isp_xs_prt(isp, xs, ISP_LOGERR, "HBA attempted queued transaction with disconnect not set"); break; case RQCS_XACT_ERR2: isp_xs_prt(isp, xs, ISP_LOGERR, "HBA attempted queued transaction to target routine %jx", (uintmax_t)XS_LUN(xs)); break; case RQCS_XACT_ERR3: isp_xs_prt(isp, xs, ISP_LOGERR, "HBA attempted queued cmd when queueing disabled"); break; case RQCS_BAD_ENTRY: isp_prt(isp, ISP_LOGERR, "Invalid IOCB entry type detected"); break; case RQCS_QUEUE_FULL: isp_xs_prt(isp, xs, ISP_LOG_WARN1, "internal queues full status 0x%x", *XS_STSP(xs)); /* * If QFULL or some other status byte is set, then this * isn't an error, per se. * * Unfortunately, some QLogic f/w writers have, in * some cases, omitted to *set* status to QFULL. */ #if 0 if (*XS_STSP(xs) != SCSI_GOOD && XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); return; } #endif *XS_STSP(xs) = SCSI_QFULL; XS_SETERR(xs, HBA_NOERROR); return; case RQCS_PHASE_SKIPPED: isp_xs_prt(isp, xs, ISP_LOGERR, "SCSI phase skipped"); break; case RQCS_ARQS_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "Auto Request Sense Failed"); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_ARQFAIL); } return; case RQCS_WIDE_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "Wide Negotiation Failed"); if (IS_SCSI(isp)) { sdparam *sdp = SDPARAM(isp, XS_CHANNEL(xs)); sdp->isp_devparam[XS_TGT(xs)].goal_flags &= ~DPARM_WIDE; sdp->isp_devparam[XS_TGT(xs)].dev_update = 1; sdp->update = 1; } if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); } return; case RQCS_SYNCXFER_FAILED: isp_xs_prt(isp, xs, ISP_LOGERR, "SDTR Message Failed"); if (IS_SCSI(isp)) { sdparam *sdp = SDPARAM(isp, XS_CHANNEL(xs)); sdp += XS_CHANNEL(xs); sdp->isp_devparam[XS_TGT(xs)].goal_flags &= ~DPARM_SYNC; sdp->isp_devparam[XS_TGT(xs)].dev_update = 1; sdp->update = 1; } break; case RQCS_LVD_BUSERR: isp_xs_prt(isp, xs, ISP_LOGERR, "Bad LVD condition"); break; case RQCS_PORT_UNAVAILABLE: /* * No such port on the loop. Moral equivalent of SELTIMEO */ case RQCS_PORT_LOGGED_OUT: { const char *reason; uint8_t sts = sp->req_completion_status & 0xff; fcparam *fcp = FCPARAM(isp, 0); fcportdb_t *lp; /* * It was there (maybe)- treat as a selection timeout. */ if (sts == RQCS_PORT_UNAVAILABLE) { reason = "unavailable"; } else { reason = "logout"; } isp_prt(isp, ISP_LOGINFO, "port %s for target %d", reason, XS_TGT(xs)); /* XXX: Should we trigger rescan or FW announce change? */ if (XS_NOERR(xs)) { lp = &fcp->portdb[XS_TGT(xs)]; if (lp->state == FC_PORTDB_STATE_ZOMBIE) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } else XS_SETERR(xs, HBA_SELTIMEOUT); } return; } case RQCS_PORT_CHANGED: isp_prt(isp, ISP_LOGWARN, "port changed for target %d", XS_TGT(xs)); if (XS_NOERR(xs)) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } return; case RQCS_PORT_BUSY: isp_prt(isp, ISP_LOGWARN, "port busy for target %d", XS_TGT(xs)); if (XS_NOERR(xs)) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } return; default: isp_prt(isp, ISP_LOGERR, "Unknown Completion Status 0x%x", sp->req_completion_status); break; } if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BOTCH); } } static void isp_parse_status_24xx(ispsoftc_t *isp, isp24xx_statusreq_t *sp, XS_T *xs, uint32_t *rp) { int ru_marked, sv_marked; int chan = XS_CHANNEL(xs); switch (sp->req_completion_status) { case RQCS_COMPLETE: if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); } return; case RQCS_DMA_ERROR: isp_xs_prt(isp, xs, ISP_LOGERR, "DMA error"); break; case RQCS_TRANSPORT_ERROR: isp_xs_prt(isp, xs, ISP_LOGERR, "Transport Error"); break; case RQCS_RESET_OCCURRED: isp_xs_prt(isp, xs, ISP_LOGWARN, "reset destroyed command"); FCPARAM(isp, chan)->sendmarker = 1; if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BUSRESET); } return; case RQCS_ABORTED: isp_xs_prt(isp, xs, ISP_LOGERR, "Command Aborted"); FCPARAM(isp, chan)->sendmarker = 1; if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_ABORTED); } return; case RQCS_TIMEOUT: isp_xs_prt(isp, xs, ISP_LOGWARN, "Command Timed Out"); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_CMDTIMEOUT); } return; case RQCS_DATA_OVERRUN: XS_SET_RESID(xs, sp->req_resid); isp_xs_prt(isp, xs, ISP_LOGERR, "Data Overrun"); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_DATAOVR); } return; case RQCS_24XX_DRE: /* data reassembly error */ isp_prt(isp, ISP_LOGERR, "Chan %d data reassembly error for target %d", chan, XS_TGT(xs)); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_ABORTED); } *rp = XS_XFRLEN(xs); return; case RQCS_24XX_TABORT: /* aborted by target */ isp_prt(isp, ISP_LOGERR, "Chan %d target %d sent ABTS", chan, XS_TGT(xs)); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_ABORTED); } return; case RQCS_DATA_UNDERRUN: ru_marked = (sp->req_scsi_status & RQCS_RU) != 0; /* * We can get an underrun w/o things being marked * if we got a non-zero status. */ sv_marked = (sp->req_scsi_status & (RQCS_SV|RQCS_RV)) != 0; if ((ru_marked == 0 && sv_marked == 0) || (sp->req_resid > XS_XFRLEN(xs))) { isp_xs_prt(isp, xs, ISP_LOGWARN, bun, XS_XFRLEN(xs), sp->req_resid, (ru_marked)? "marked" : "not marked"); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BOTCH); } return; } XS_SET_RESID(xs, sp->req_resid); isp_xs_prt(isp, xs, ISP_LOG_WARN1, "Data Underrun (%d) for command 0x%x", sp->req_resid, XS_CDBP(xs)[0] & 0xff); if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_NOERROR); } return; case RQCS_PORT_UNAVAILABLE: /* * No such port on the loop. Moral equivalent of SELTIMEO */ case RQCS_PORT_LOGGED_OUT: { const char *reason; uint8_t sts = sp->req_completion_status & 0xff; fcparam *fcp = FCPARAM(isp, XS_CHANNEL(xs)); fcportdb_t *lp; /* * It was there (maybe)- treat as a selection timeout. */ if (sts == RQCS_PORT_UNAVAILABLE) { reason = "unavailable"; } else { reason = "logout"; } isp_prt(isp, ISP_LOGINFO, "Chan %d port %s for target %d", chan, reason, XS_TGT(xs)); /* XXX: Should we trigger rescan or FW announce change? */ if (XS_NOERR(xs)) { lp = &fcp->portdb[XS_TGT(xs)]; if (lp->state == FC_PORTDB_STATE_ZOMBIE) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } else XS_SETERR(xs, HBA_SELTIMEOUT); } return; } case RQCS_PORT_CHANGED: isp_prt(isp, ISP_LOGWARN, "port changed for target %d chan %d", XS_TGT(xs), chan); if (XS_NOERR(xs)) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } return; case RQCS_24XX_ENOMEM: /* f/w resource unavailable */ isp_prt(isp, ISP_LOGWARN, "f/w resource unavailable for target %d chan %d", XS_TGT(xs), chan); if (XS_NOERR(xs)) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } return; case RQCS_24XX_TMO: /* task management overrun */ isp_prt(isp, ISP_LOGWARN, "command for target %d overlapped task management for chan %d", XS_TGT(xs), chan); if (XS_NOERR(xs)) { *XS_STSP(xs) = SCSI_BUSY; XS_SETERR(xs, HBA_TGTBSY); } return; default: isp_prt(isp, ISP_LOGERR, "Unknown Completion Status 0x%x on chan %d", sp->req_completion_status, chan); break; } if (XS_NOERR(xs)) { XS_SETERR(xs, HBA_BOTCH); } } static void isp_fastpost_complete(ispsoftc_t *isp, uint32_t fph) { XS_T *xs; if (fph == 0) { return; } xs = isp_find_xs(isp, fph); if (xs == NULL) { isp_prt(isp, ISP_LOGWARN, "Command for fast post handle 0x%x not found", fph); return; } isp_destroy_handle(isp, fph); /* * Since we don't have a result queue entry item, * we must believe that SCSI status is zero and * that all data transferred. */ XS_SET_RESID(xs, 0); *XS_STSP(xs) = SCSI_GOOD; if (XS_XFRLEN(xs)) { ISP_DMAFREE(isp, xs, fph); } isp_done(xs); } #define ISP_SCSI_IBITS(op) (mbpscsi[((op)<<1)]) #define ISP_SCSI_OBITS(op) (mbpscsi[((op)<<1) + 1]) #define ISP_SCSI_OPMAP(in, out) in, out static const uint8_t mbpscsi[] = { ISP_SCSI_OPMAP(0x01, 0x01), /* 0x00: MBOX_NO_OP */ ISP_SCSI_OPMAP(0x1f, 0x01), /* 0x01: MBOX_LOAD_RAM */ ISP_SCSI_OPMAP(0x03, 0x01), /* 0x02: MBOX_EXEC_FIRMWARE */ ISP_SCSI_OPMAP(0x1f, 0x01), /* 0x03: MBOX_DUMP_RAM */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x04: MBOX_WRITE_RAM_WORD */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x05: MBOX_READ_RAM_WORD */ ISP_SCSI_OPMAP(0x3f, 0x3f), /* 0x06: MBOX_MAILBOX_REG_TEST */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x07: MBOX_VERIFY_CHECKSUM */ ISP_SCSI_OPMAP(0x01, 0x0f), /* 0x08: MBOX_ABOUT_FIRMWARE */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x09: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x0a: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x0b: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x0c: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x0d: */ ISP_SCSI_OPMAP(0x01, 0x05), /* 0x0e: MBOX_CHECK_FIRMWARE */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x0f: */ ISP_SCSI_OPMAP(0x1f, 0x1f), /* 0x10: MBOX_INIT_REQ_QUEUE */ ISP_SCSI_OPMAP(0x3f, 0x3f), /* 0x11: MBOX_INIT_RES_QUEUE */ ISP_SCSI_OPMAP(0x0f, 0x0f), /* 0x12: MBOX_EXECUTE_IOCB */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x13: MBOX_WAKE_UP */ ISP_SCSI_OPMAP(0x01, 0x3f), /* 0x14: MBOX_STOP_FIRMWARE */ ISP_SCSI_OPMAP(0x0f, 0x0f), /* 0x15: MBOX_ABORT */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x16: MBOX_ABORT_DEVICE */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x17: MBOX_ABORT_TARGET */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x18: MBOX_BUS_RESET */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x19: MBOX_STOP_QUEUE */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x1a: MBOX_START_QUEUE */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x1b: MBOX_SINGLE_STEP_QUEUE */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x1c: MBOX_ABORT_QUEUE */ ISP_SCSI_OPMAP(0x03, 0x4f), /* 0x1d: MBOX_GET_DEV_QUEUE_STATUS */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x1e: */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x1f: MBOX_GET_FIRMWARE_STATUS */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x20: MBOX_GET_INIT_SCSI_ID */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x21: MBOX_GET_SELECT_TIMEOUT */ ISP_SCSI_OPMAP(0x01, 0xc7), /* 0x22: MBOX_GET_RETRY_COUNT */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x23: MBOX_GET_TAG_AGE_LIMIT */ ISP_SCSI_OPMAP(0x01, 0x03), /* 0x24: MBOX_GET_CLOCK_RATE */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x25: MBOX_GET_ACT_NEG_STATE */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x26: MBOX_GET_ASYNC_DATA_SETUP_TIME */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x27: MBOX_GET_PCI_PARAMS */ ISP_SCSI_OPMAP(0x03, 0x4f), /* 0x28: MBOX_GET_TARGET_PARAMS */ ISP_SCSI_OPMAP(0x03, 0x0f), /* 0x29: MBOX_GET_DEV_QUEUE_PARAMS */ ISP_SCSI_OPMAP(0x01, 0x07), /* 0x2a: MBOX_GET_RESET_DELAY_PARAMS */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x2b: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x2c: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x2d: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x2e: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x2f: */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x30: MBOX_SET_INIT_SCSI_ID */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x31: MBOX_SET_SELECT_TIMEOUT */ ISP_SCSI_OPMAP(0xc7, 0xc7), /* 0x32: MBOX_SET_RETRY_COUNT */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x33: MBOX_SET_TAG_AGE_LIMIT */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x34: MBOX_SET_CLOCK_RATE */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x35: MBOX_SET_ACT_NEG_STATE */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x36: MBOX_SET_ASYNC_DATA_SETUP_TIME */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x37: MBOX_SET_PCI_CONTROL_PARAMS */ ISP_SCSI_OPMAP(0x4f, 0x4f), /* 0x38: MBOX_SET_TARGET_PARAMS */ ISP_SCSI_OPMAP(0x0f, 0x0f), /* 0x39: MBOX_SET_DEV_QUEUE_PARAMS */ ISP_SCSI_OPMAP(0x07, 0x07), /* 0x3a: MBOX_SET_RESET_DELAY_PARAMS */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x3b: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x3c: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x3d: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x3e: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x3f: */ ISP_SCSI_OPMAP(0x01, 0x03), /* 0x40: MBOX_RETURN_BIOS_BLOCK_ADDR */ ISP_SCSI_OPMAP(0x3f, 0x01), /* 0x41: MBOX_WRITE_FOUR_RAM_WORDS */ ISP_SCSI_OPMAP(0x03, 0x07), /* 0x42: MBOX_EXEC_BIOS_IOCB */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x43: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x44: */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x45: SET SYSTEM PARAMETER */ ISP_SCSI_OPMAP(0x01, 0x03), /* 0x46: GET SYSTEM PARAMETER */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x47: */ ISP_SCSI_OPMAP(0x01, 0xcf), /* 0x48: GET SCAM CONFIGURATION */ ISP_SCSI_OPMAP(0xcf, 0xcf), /* 0x49: SET SCAM CONFIGURATION */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x4a: MBOX_SET_FIRMWARE_FEATURES */ ISP_SCSI_OPMAP(0x01, 0x03), /* 0x4b: MBOX_GET_FIRMWARE_FEATURES */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x4c: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x4d: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x4e: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x4f: */ ISP_SCSI_OPMAP(0xdf, 0xdf), /* 0x50: LOAD RAM A64 */ ISP_SCSI_OPMAP(0xdf, 0xdf), /* 0x51: DUMP RAM A64 */ ISP_SCSI_OPMAP(0xdf, 0xff), /* 0x52: INITIALIZE REQUEST QUEUE A64 */ ISP_SCSI_OPMAP(0xef, 0xff), /* 0x53: INITIALIZE RESPONSE QUEUE A64 */ ISP_SCSI_OPMAP(0xcf, 0x01), /* 0x54: EXECUCUTE COMMAND IOCB A64 */ ISP_SCSI_OPMAP(0x07, 0x01), /* 0x55: ENABLE TARGET MODE */ ISP_SCSI_OPMAP(0x03, 0x0f), /* 0x56: GET TARGET STATUS */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x57: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x58: */ ISP_SCSI_OPMAP(0x00, 0x00), /* 0x59: */ ISP_SCSI_OPMAP(0x03, 0x03), /* 0x5a: SET DATA OVERRUN RECOVERY MODE */ ISP_SCSI_OPMAP(0x01, 0x03), /* 0x5b: GET DATA OVERRUN RECOVERY MODE */ ISP_SCSI_OPMAP(0x0f, 0x0f), /* 0x5c: SET HOST DATA */ ISP_SCSI_OPMAP(0x01, 0x01) /* 0x5d: GET NOST DATA */ }; #define MAX_SCSI_OPCODE 0x5d static const char *scsi_mbcmd_names[] = { "NO-OP", "LOAD RAM", "EXEC FIRMWARE", "DUMP RAM", "WRITE RAM WORD", "READ RAM WORD", "MAILBOX REG TEST", "VERIFY CHECKSUM", "ABOUT FIRMWARE", NULL, NULL, NULL, NULL, NULL, "CHECK FIRMWARE", NULL, "INIT REQUEST QUEUE", "INIT RESULT QUEUE", "EXECUTE IOCB", "WAKE UP", "STOP FIRMWARE", "ABORT", "ABORT DEVICE", "ABORT TARGET", "BUS RESET", "STOP QUEUE", "START QUEUE", "SINGLE STEP QUEUE", "ABORT QUEUE", "GET DEV QUEUE STATUS", NULL, "GET FIRMWARE STATUS", "GET INIT SCSI ID", "GET SELECT TIMEOUT", "GET RETRY COUNT", "GET TAG AGE LIMIT", "GET CLOCK RATE", "GET ACT NEG STATE", "GET ASYNC DATA SETUP TIME", "GET PCI PARAMS", "GET TARGET PARAMS", "GET DEV QUEUE PARAMS", "GET RESET DELAY PARAMS", NULL, NULL, NULL, NULL, NULL, "SET INIT SCSI ID", "SET SELECT TIMEOUT", "SET RETRY COUNT", "SET TAG AGE LIMIT", "SET CLOCK RATE", "SET ACT NEG STATE", "SET ASYNC DATA SETUP TIME", "SET PCI CONTROL PARAMS", "SET TARGET PARAMS", "SET DEV QUEUE PARAMS", "SET RESET DELAY PARAMS", NULL, NULL, NULL, NULL, NULL, "RETURN BIOS BLOCK ADDR", "WRITE FOUR RAM WORDS", "EXEC BIOS IOCB", NULL, NULL, "SET SYSTEM PARAMETER", "GET SYSTEM PARAMETER", NULL, "GET SCAM CONFIGURATION", "SET SCAM CONFIGURATION", "SET FIRMWARE FEATURES", "GET FIRMWARE FEATURES", NULL, NULL, NULL, NULL, "LOAD RAM A64", "DUMP RAM A64", "INITIALIZE REQUEST QUEUE A64", "INITIALIZE RESPONSE QUEUE A64", "EXECUTE IOCB A64", "ENABLE TARGET MODE", "GET TARGET MODE STATE", NULL, NULL, NULL, "SET DATA OVERRUN RECOVERY MODE", "GET DATA OVERRUN RECOVERY MODE", "SET HOST DATA", "GET NOST DATA", }; #define ISP_FC_IBITS(op) ((mbpfc[((op)<<3) + 0] << 24) | (mbpfc[((op)<<3) + 1] << 16) | (mbpfc[((op)<<3) + 2] << 8) | (mbpfc[((op)<<3) + 3])) #define ISP_FC_OBITS(op) ((mbpfc[((op)<<3) + 4] << 24) | (mbpfc[((op)<<3) + 5] << 16) | (mbpfc[((op)<<3) + 6] << 8) | (mbpfc[((op)<<3) + 7])) #define ISP_FC_OPMAP(in0, out0) 0, 0, 0, in0, 0, 0, 0, out0 #define ISP_FC_OPMAP_HALF(in1, in0, out1, out0) 0, 0, in1, in0, 0, 0, out1, out0 #define ISP_FC_OPMAP_FULL(in3, in2, in1, in0, out3, out2, out1, out0) in3, in2, in1, in0, out3, out2, out1, out0 static const uint32_t mbpfc[] = { ISP_FC_OPMAP(0x01, 0x01), /* 0x00: MBOX_NO_OP */ ISP_FC_OPMAP(0x1f, 0x01), /* 0x01: MBOX_LOAD_RAM */ ISP_FC_OPMAP_HALF(0x07, 0xff, 0x00, 0x1f), /* 0x02: MBOX_EXEC_FIRMWARE */ ISP_FC_OPMAP(0xdf, 0x01), /* 0x03: MBOX_DUMP_RAM */ ISP_FC_OPMAP(0x07, 0x07), /* 0x04: MBOX_WRITE_RAM_WORD */ ISP_FC_OPMAP(0x03, 0x07), /* 0x05: MBOX_READ_RAM_WORD */ ISP_FC_OPMAP_FULL(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff), /* 0x06: MBOX_MAILBOX_REG_TEST */ ISP_FC_OPMAP(0x07, 0x07), /* 0x07: MBOX_VERIFY_CHECKSUM */ ISP_FC_OPMAP_FULL(0x0, 0x0, 0x0, 0x01, 0x0, 0x3, 0x80, 0x7f), /* 0x08: MBOX_ABOUT_FIRMWARE */ ISP_FC_OPMAP(0xdf, 0x01), /* 0x09: MBOX_LOAD_RISC_RAM_2100 */ ISP_FC_OPMAP(0xdf, 0x01), /* 0x0a: DUMP RAM */ ISP_FC_OPMAP_HALF(0x1, 0xff, 0x0, 0x01), /* 0x0b: MBOX_LOAD_RISC_RAM */ ISP_FC_OPMAP(0x00, 0x00), /* 0x0c: */ ISP_FC_OPMAP_HALF(0x1, 0x0f, 0x0, 0x01), /* 0x0d: MBOX_WRITE_RAM_WORD_EXTENDED */ ISP_FC_OPMAP(0x01, 0x05), /* 0x0e: MBOX_CHECK_FIRMWARE */ ISP_FC_OPMAP_HALF(0x1, 0x03, 0x0, 0x0d), /* 0x0f: MBOX_READ_RAM_WORD_EXTENDED */ ISP_FC_OPMAP(0x1f, 0x11), /* 0x10: MBOX_INIT_REQ_QUEUE */ ISP_FC_OPMAP(0x2f, 0x21), /* 0x11: MBOX_INIT_RES_QUEUE */ ISP_FC_OPMAP(0x0f, 0x01), /* 0x12: MBOX_EXECUTE_IOCB */ ISP_FC_OPMAP(0x03, 0x03), /* 0x13: MBOX_WAKE_UP */ ISP_FC_OPMAP_HALF(0x1, 0xff, 0x0, 0x03), /* 0x14: MBOX_STOP_FIRMWARE */ ISP_FC_OPMAP(0x4f, 0x01), /* 0x15: MBOX_ABORT */ ISP_FC_OPMAP(0x07, 0x01), /* 0x16: MBOX_ABORT_DEVICE */ ISP_FC_OPMAP(0x07, 0x01), /* 0x17: MBOX_ABORT_TARGET */ ISP_FC_OPMAP(0x03, 0x03), /* 0x18: MBOX_BUS_RESET */ ISP_FC_OPMAP(0x07, 0x05), /* 0x19: MBOX_STOP_QUEUE */ ISP_FC_OPMAP(0x07, 0x05), /* 0x1a: MBOX_START_QUEUE */ ISP_FC_OPMAP(0x07, 0x05), /* 0x1b: MBOX_SINGLE_STEP_QUEUE */ ISP_FC_OPMAP(0x07, 0x05), /* 0x1c: MBOX_ABORT_QUEUE */ ISP_FC_OPMAP(0x07, 0x03), /* 0x1d: MBOX_GET_DEV_QUEUE_STATUS */ ISP_FC_OPMAP(0x00, 0x00), /* 0x1e: */ ISP_FC_OPMAP(0x01, 0x07), /* 0x1f: MBOX_GET_FIRMWARE_STATUS */ ISP_FC_OPMAP_HALF(0x2, 0x01, 0x7e, 0xcf), /* 0x20: MBOX_GET_LOOP_ID */ ISP_FC_OPMAP(0x00, 0x00), /* 0x21: */ ISP_FC_OPMAP(0x03, 0x4b), /* 0x22: MBOX_GET_TIMEOUT_PARAMS */ ISP_FC_OPMAP(0x00, 0x00), /* 0x23: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x24: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x25: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x26: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x27: */ ISP_FC_OPMAP(0x01, 0x03), /* 0x28: MBOX_GET_FIRMWARE_OPTIONS */ ISP_FC_OPMAP(0x03, 0x07), /* 0x29: MBOX_GET_PORT_QUEUE_PARAMS */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2a: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2b: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2c: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2d: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2e: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x2f: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x30: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x31: */ ISP_FC_OPMAP(0x4b, 0x4b), /* 0x32: MBOX_SET_TIMEOUT_PARAMS */ ISP_FC_OPMAP(0x00, 0x00), /* 0x33: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x34: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x35: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x36: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x37: */ ISP_FC_OPMAP(0x0f, 0x01), /* 0x38: MBOX_SET_FIRMWARE_OPTIONS */ ISP_FC_OPMAP(0x0f, 0x07), /* 0x39: MBOX_SET_PORT_QUEUE_PARAMS */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3a: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3b: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3c: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3d: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3e: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x3f: */ ISP_FC_OPMAP(0x03, 0x01), /* 0x40: MBOX_LOOP_PORT_BYPASS */ ISP_FC_OPMAP(0x03, 0x01), /* 0x41: MBOX_LOOP_PORT_ENABLE */ ISP_FC_OPMAP_HALF(0x0, 0x01, 0x1f, 0xcf), /* 0x42: MBOX_GET_RESOURCE_COUNT */ ISP_FC_OPMAP(0x01, 0x01), /* 0x43: MBOX_REQUEST_OFFLINE_MODE */ ISP_FC_OPMAP(0x00, 0x00), /* 0x44: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x45: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x46: */ ISP_FC_OPMAP(0xcf, 0x03), /* 0x47: GET PORT_DATABASE ENHANCED */ ISP_FC_OPMAP(0xcf, 0x0f), /* 0x48: MBOX_INIT_FIRMWARE_MULTI_ID */ ISP_FC_OPMAP(0xcd, 0x01), /* 0x49: MBOX_GET_VP_DATABASE */ ISP_FC_OPMAP_HALF(0x2, 0xcd, 0x0, 0x01), /* 0x4a: MBOX_GET_VP_DATABASE_ENTRY */ ISP_FC_OPMAP(0x00, 0x00), /* 0x4b: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x4c: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x4d: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x4e: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x4f: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x50: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x51: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x52: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x53: */ ISP_FC_OPMAP(0xcf, 0x01), /* 0x54: EXECUTE IOCB A64 */ ISP_FC_OPMAP(0x00, 0x00), /* 0x55: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x56: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x57: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x58: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x59: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x5a: */ ISP_FC_OPMAP(0x03, 0x01), /* 0x5b: MBOX_DRIVER_HEARTBEAT */ ISP_FC_OPMAP(0xcf, 0x01), /* 0x5c: MBOX_FW_HEARTBEAT */ ISP_FC_OPMAP(0x07, 0x1f), /* 0x5d: MBOX_GET_SET_DATA_RATE */ ISP_FC_OPMAP(0x00, 0x00), /* 0x5e: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x5f: */ ISP_FC_OPMAP(0xcf, 0x0f), /* 0x60: MBOX_INIT_FIRMWARE */ ISP_FC_OPMAP(0x00, 0x00), /* 0x61: */ ISP_FC_OPMAP(0x01, 0x01), /* 0x62: MBOX_INIT_LIP */ ISP_FC_OPMAP(0xcd, 0x03), /* 0x63: MBOX_GET_FC_AL_POSITION_MAP */ ISP_FC_OPMAP(0xcf, 0x01), /* 0x64: MBOX_GET_PORT_DB */ ISP_FC_OPMAP(0x07, 0x01), /* 0x65: MBOX_CLEAR_ACA */ ISP_FC_OPMAP(0x07, 0x01), /* 0x66: MBOX_TARGET_RESET */ ISP_FC_OPMAP(0x07, 0x01), /* 0x67: MBOX_CLEAR_TASK_SET */ ISP_FC_OPMAP(0x07, 0x01), /* 0x68: MBOX_ABORT_TASK_SET */ ISP_FC_OPMAP_HALF(0x00, 0x01, 0x0f, 0x1f), /* 0x69: MBOX_GET_FW_STATE */ ISP_FC_OPMAP_HALF(0x6, 0x03, 0x0, 0xcf), /* 0x6a: MBOX_GET_PORT_NAME */ ISP_FC_OPMAP(0xcf, 0x01), /* 0x6b: MBOX_GET_LINK_STATUS */ ISP_FC_OPMAP(0x0f, 0x01), /* 0x6c: MBOX_INIT_LIP_RESET */ ISP_FC_OPMAP(0x00, 0x00), /* 0x6d: */ ISP_FC_OPMAP(0xcf, 0x03), /* 0x6e: MBOX_SEND_SNS */ ISP_FC_OPMAP(0x0f, 0x07), /* 0x6f: MBOX_FABRIC_LOGIN */ ISP_FC_OPMAP_HALF(0x02, 0x03, 0x00, 0x03), /* 0x70: MBOX_SEND_CHANGE_REQUEST */ ISP_FC_OPMAP(0x03, 0x03), /* 0x71: MBOX_FABRIC_LOGOUT */ ISP_FC_OPMAP(0x0f, 0x0f), /* 0x72: MBOX_INIT_LIP_LOGIN */ ISP_FC_OPMAP(0x00, 0x00), /* 0x73: */ ISP_FC_OPMAP(0x07, 0x01), /* 0x74: LOGIN LOOP PORT */ ISP_FC_OPMAP_HALF(0x03, 0xcf, 0x00, 0x07), /* 0x75: GET PORT/NODE NAME LIST */ ISP_FC_OPMAP(0x4f, 0x01), /* 0x76: SET VENDOR ID */ ISP_FC_OPMAP(0xcd, 0x01), /* 0x77: INITIALIZE IP MAILBOX */ ISP_FC_OPMAP(0x00, 0x00), /* 0x78: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x79: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x7a: */ ISP_FC_OPMAP(0x00, 0x00), /* 0x7b: */ ISP_FC_OPMAP_HALF(0x03, 0x4f, 0x00, 0x07), /* 0x7c: Get ID List */ ISP_FC_OPMAP(0xcf, 0x01), /* 0x7d: SEND LFA */ ISP_FC_OPMAP(0x0f, 0x01) /* 0x7e: LUN RESET */ }; #define MAX_FC_OPCODE 0x7e /* * Footnotes * * (1): this sets bits 21..16 in mailbox register #8, which we nominally * do not access at this time in the core driver. The caller is * responsible for setting this register first (Gross!). The assumption * is that we won't overflow. */ static const char *fc_mbcmd_names[] = { "NO-OP", /* 00h */ "LOAD RAM", "EXEC FIRMWARE", "DUMP RAM", "WRITE RAM WORD", "READ RAM WORD", "MAILBOX REG TEST", "VERIFY CHECKSUM", "ABOUT FIRMWARE", "LOAD RAM (2100)", "DUMP RAM", "LOAD RISC RAM", "DUMP RISC RAM", "WRITE RAM WORD EXTENDED", "CHECK FIRMWARE", "READ RAM WORD EXTENDED", "INIT REQUEST QUEUE", /* 10h */ "INIT RESULT QUEUE", "EXECUTE IOCB", "WAKE UP", "STOP FIRMWARE", "ABORT", "ABORT DEVICE", "ABORT TARGET", "BUS RESET", "STOP QUEUE", "START QUEUE", "SINGLE STEP QUEUE", "ABORT QUEUE", "GET DEV QUEUE STATUS", NULL, "GET FIRMWARE STATUS", "GET LOOP ID", /* 20h */ NULL, "GET TIMEOUT PARAMS", NULL, NULL, NULL, NULL, NULL, "GET FIRMWARE OPTIONS", "GET PORT QUEUE PARAMS", "GENERATE SYSTEM ERROR", NULL, NULL, NULL, NULL, NULL, "WRITE SFP", /* 30h */ "READ SFP", "SET TIMEOUT PARAMS", NULL, NULL, NULL, NULL, NULL, "SET FIRMWARE OPTIONS", "SET PORT QUEUE PARAMS", NULL, "SET FC LED CONF", NULL, "RESTART NIC FIRMWARE", "ACCESS CONTROL", NULL, "LOOP PORT BYPASS", /* 40h */ "LOOP PORT ENABLE", "GET RESOURCE COUNT", "REQUEST NON PARTICIPATING MODE", "DIAGNOSTIC ECHO TEST", "DIAGNOSTIC LOOPBACK", NULL, "GET PORT DATABASE ENHANCED", "INIT FIRMWARE MULTI ID", "GET VP DATABASE", "GET VP DATABASE ENTRY", NULL, NULL, NULL, NULL, NULL, "GET FCF LIST", /* 50h */ "GET DCBX PARAMETERS", NULL, "HOST MEMORY COPY", "EXECUTE IOCB A64", NULL, NULL, "SEND RNID", NULL, "SET PARAMETERS", "GET PARAMETERS", "DRIVER HEARTBEAT", "FIRMWARE HEARTBEAT", "GET/SET DATA RATE", "SEND RNFT", NULL, "INIT FIRMWARE", /* 60h */ "GET INIT CONTROL BLOCK", "INIT LIP", "GET FC-AL POSITION MAP", "GET PORT DATABASE", "CLEAR ACA", "TARGET RESET", "CLEAR TASK SET", "ABORT TASK SET", "GET FW STATE", "GET PORT NAME", "GET LINK STATUS", "INIT LIP RESET", "GET LINK STATS & PRIVATE DATA CNTS", "SEND SNS", "FABRIC LOGIN", "SEND CHANGE REQUEST", /* 70h */ "FABRIC LOGOUT", "INIT LIP LOGIN", NULL, "LOGIN LOOP PORT", "GET PORT/NODE NAME LIST", "SET VENDOR ID", "INITIALIZE IP MAILBOX", NULL, NULL, "GET XGMAC STATS", NULL, "GET ID LIST", "SEND LFA", "LUN RESET" }; static void isp_mboxcmd(ispsoftc_t *isp, mbreg_t *mbp) { const char *cname, *xname, *sname; char tname[16], mname[16]; unsigned int ibits, obits, box, opcode; opcode = mbp->param[0]; if (IS_FC(isp)) { if (opcode > MAX_FC_OPCODE) { mbp->param[0] = MBOX_INVALID_COMMAND; isp_prt(isp, ISP_LOGERR, "Unknown Command 0x%x", opcode); return; } cname = fc_mbcmd_names[opcode]; ibits = ISP_FC_IBITS(opcode); obits = ISP_FC_OBITS(opcode); } else { if (opcode > MAX_SCSI_OPCODE) { mbp->param[0] = MBOX_INVALID_COMMAND; isp_prt(isp, ISP_LOGERR, "Unknown Command 0x%x", opcode); return; } cname = scsi_mbcmd_names[opcode]; ibits = ISP_SCSI_IBITS(opcode); obits = ISP_SCSI_OBITS(opcode); } if (cname == NULL) { cname = tname; ISP_SNPRINTF(tname, sizeof tname, "opcode %x", opcode); } isp_prt(isp, ISP_LOGDEBUG3, "Mailbox Command '%s'", cname); /* * Pick up any additional bits that the caller might have set. */ ibits |= mbp->ibits; obits |= mbp->obits; /* * Mask any bits that the caller wants us to mask */ ibits &= mbp->ibitm; obits &= mbp->obitm; if (ibits == 0 && obits == 0) { mbp->param[0] = MBOX_COMMAND_PARAM_ERROR; isp_prt(isp, ISP_LOGERR, "no parameters for 0x%x", opcode); return; } /* * Get exclusive usage of mailbox registers. */ if (MBOX_ACQUIRE(isp)) { mbp->param[0] = MBOX_REGS_BUSY; goto out; } for (box = 0; box < ISP_NMBOX(isp); box++) { if (ibits & (1 << box)) { isp_prt(isp, ISP_LOGDEBUG3, "IN mbox %d = 0x%04x", box, mbp->param[box]); ISP_WRITE(isp, MBOX_OFF(box), mbp->param[box]); } isp->isp_mboxtmp[box] = mbp->param[box] = 0; } isp->isp_lastmbxcmd = opcode; /* * We assume that we can't overwrite a previous command. */ isp->isp_obits = obits; isp->isp_mboxbsy = 1; /* * Set Host Interrupt condition so that RISC will pick up mailbox regs. */ if (IS_24XX(isp)) { ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_SET_HOST_INT); } else { ISP_WRITE(isp, HCCR, HCCR_CMD_SET_HOST_INT); } /* * While we haven't finished the command, spin our wheels here. */ MBOX_WAIT_COMPLETE(isp, mbp); /* * Did the command time out? */ if (mbp->param[0] == MBOX_TIMEOUT) { isp->isp_mboxbsy = 0; MBOX_RELEASE(isp); goto out; } /* * Copy back output registers. */ for (box = 0; box < ISP_NMBOX(isp); box++) { if (obits & (1 << box)) { mbp->param[box] = isp->isp_mboxtmp[box]; isp_prt(isp, ISP_LOGDEBUG3, "OUT mbox %d = 0x%04x", box, mbp->param[box]); } } isp->isp_mboxbsy = 0; MBOX_RELEASE(isp); out: if (mbp->logval == 0 || mbp->param[0] == MBOX_COMMAND_COMPLETE) return; if ((mbp->param[0] & 0xbfe0) == 0 && (mbp->logval & MBLOGMASK(mbp->param[0])) == 0) return; xname = NULL; sname = ""; switch (mbp->param[0]) { case MBOX_INVALID_COMMAND: xname = "INVALID COMMAND"; break; case MBOX_HOST_INTERFACE_ERROR: xname = "HOST INTERFACE ERROR"; break; case MBOX_TEST_FAILED: xname = "TEST FAILED"; break; case MBOX_COMMAND_ERROR: xname = "COMMAND ERROR"; ISP_SNPRINTF(mname, sizeof(mname), " subcode 0x%x", mbp->param[1]); sname = mname; break; case MBOX_COMMAND_PARAM_ERROR: xname = "COMMAND PARAMETER ERROR"; break; case MBOX_PORT_ID_USED: xname = "PORT ID ALREADY IN USE"; break; case MBOX_LOOP_ID_USED: xname = "LOOP ID ALREADY IN USE"; break; case MBOX_ALL_IDS_USED: xname = "ALL LOOP IDS IN USE"; break; case MBOX_NOT_LOGGED_IN: xname = "NOT LOGGED IN"; break; case MBOX_LINK_DOWN_ERROR: xname = "LINK DOWN ERROR"; break; case MBOX_LOOPBACK_ERROR: xname = "LOOPBACK ERROR"; break; case MBOX_CHECKSUM_ERROR: xname = "CHECKSUM ERROR"; break; case MBOX_INVALID_PRODUCT_KEY: xname = "INVALID PRODUCT KEY"; break; case MBOX_REGS_BUSY: xname = "REGISTERS BUSY"; break; case MBOX_TIMEOUT: xname = "TIMEOUT"; break; default: ISP_SNPRINTF(mname, sizeof mname, "error 0x%x", mbp->param[0]); xname = mname; break; } if (xname) { isp_prt(isp, ISP_LOGALL, "Mailbox Command '%s' failed (%s%s)", cname, xname, sname); } } static int isp_fw_state(ispsoftc_t *isp, int chan) { if (IS_FC(isp)) { mbreg_t mbs; MBSINIT(&mbs, MBOX_GET_FW_STATE, MBLOGALL, 0); isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { return (mbs.param[1]); } } return (FW_ERROR); } static void isp_spi_update(ispsoftc_t *isp, int chan) { int tgt; mbreg_t mbs; sdparam *sdp; if (IS_FC(isp)) { /* * There are no 'per-bus' settings for Fibre Channel. */ return; } sdp = SDPARAM(isp, chan); sdp->update = 0; for (tgt = 0; tgt < MAX_TARGETS; tgt++) { uint16_t flags, period, offset; int get; if (sdp->isp_devparam[tgt].dev_enable == 0) { sdp->isp_devparam[tgt].dev_update = 0; sdp->isp_devparam[tgt].dev_refresh = 0; isp_prt(isp, ISP_LOGDEBUG0, "skipping target %d bus %d update", tgt, chan); continue; } /* * If the goal is to update the status of the device, * take what's in goal_flags and try and set the device * toward that. Otherwise, if we're just refreshing the * current device state, get the current parameters. */ MBSINIT(&mbs, 0, MBLOGALL, 0); /* * Refresh overrides set */ if (sdp->isp_devparam[tgt].dev_refresh) { mbs.param[0] = MBOX_GET_TARGET_PARAMS; get = 1; } else if (sdp->isp_devparam[tgt].dev_update) { mbs.param[0] = MBOX_SET_TARGET_PARAMS; /* * Make sure goal_flags has "Renegotiate on Error" * on and "Freeze Queue on Error" off. */ sdp->isp_devparam[tgt].goal_flags |= DPARM_RENEG; sdp->isp_devparam[tgt].goal_flags &= ~DPARM_QFRZ; mbs.param[2] = sdp->isp_devparam[tgt].goal_flags; /* * Insist that PARITY must be enabled * if SYNC or WIDE is enabled. */ if ((mbs.param[2] & (DPARM_SYNC|DPARM_WIDE)) != 0) { mbs.param[2] |= DPARM_PARITY; } if (mbs.param[2] & DPARM_SYNC) { mbs.param[3] = (sdp->isp_devparam[tgt].goal_offset << 8) | (sdp->isp_devparam[tgt].goal_period); } /* * A command completion later that has * RQSTF_NEGOTIATION set can cause * the dev_refresh/announce cycle also. * * Note: It is really important to update our current * flags with at least the state of TAG capabilities- * otherwise we might try and send a tagged command * when we have it all turned off. So change it here * to say that current already matches goal. */ sdp->isp_devparam[tgt].actv_flags &= ~DPARM_TQING; sdp->isp_devparam[tgt].actv_flags |= (sdp->isp_devparam[tgt].goal_flags & DPARM_TQING); isp_prt(isp, ISP_LOGDEBUG0, "bus %d set tgt %d flags 0x%x off 0x%x period 0x%x", chan, tgt, mbs.param[2], mbs.param[3] >> 8, mbs.param[3] & 0xff); get = 0; } else { continue; } mbs.param[1] = (chan << 15) | (tgt << 8); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { continue; } if (get == 0) { sdp->sendmarker = 1; sdp->isp_devparam[tgt].dev_update = 0; sdp->isp_devparam[tgt].dev_refresh = 1; } else { sdp->isp_devparam[tgt].dev_refresh = 0; flags = mbs.param[2]; period = mbs.param[3] & 0xff; offset = mbs.param[3] >> 8; sdp->isp_devparam[tgt].actv_flags = flags; sdp->isp_devparam[tgt].actv_period = period; sdp->isp_devparam[tgt].actv_offset = offset; isp_async(isp, ISPASYNC_NEW_TGT_PARAMS, chan, tgt); } } for (tgt = 0; tgt < MAX_TARGETS; tgt++) { if (sdp->isp_devparam[tgt].dev_update || sdp->isp_devparam[tgt].dev_refresh) { sdp->update = 1; break; } } } static void isp_setdfltsdparm(ispsoftc_t *isp) { int tgt; sdparam *sdp, *sdp1; sdp = SDPARAM(isp, 0); if (IS_DUALBUS(isp)) sdp1 = sdp + 1; else sdp1 = NULL; /* * Establish some default parameters. */ sdp->isp_cmd_dma_burst_enable = 0; sdp->isp_data_dma_burst_enabl = 1; sdp->isp_fifo_threshold = 0; sdp->isp_initiator_id = DEFAULT_IID(isp, 0); if (isp->isp_type >= ISP_HA_SCSI_1040) { sdp->isp_async_data_setup = 9; } else { sdp->isp_async_data_setup = 6; } sdp->isp_selection_timeout = 250; sdp->isp_max_queue_depth = MAXISPREQUEST(isp); sdp->isp_tag_aging = 8; sdp->isp_bus_reset_delay = 5; /* * Don't retry selection, busy or queue full automatically- reflect * these back to us. */ sdp->isp_retry_count = 0; sdp->isp_retry_delay = 0; for (tgt = 0; tgt < MAX_TARGETS; tgt++) { sdp->isp_devparam[tgt].exc_throttle = ISP_EXEC_THROTTLE; sdp->isp_devparam[tgt].dev_enable = 1; } /* * The trick here is to establish a default for the default (honk!) * state (goal_flags). Then try and get the current status from * the card to fill in the current state. We don't, in fact, set * the default to the SAFE default state- that's not the goal state. */ for (tgt = 0; tgt < MAX_TARGETS; tgt++) { uint8_t off, per; sdp->isp_devparam[tgt].actv_offset = 0; sdp->isp_devparam[tgt].actv_period = 0; sdp->isp_devparam[tgt].actv_flags = 0; sdp->isp_devparam[tgt].goal_flags = sdp->isp_devparam[tgt].nvrm_flags = DPARM_DEFAULT; /* * We default to Wide/Fast for versions less than a 1040 * (unless it's SBus). */ if (IS_ULTRA3(isp)) { off = ISP_80M_SYNCPARMS >> 8; per = ISP_80M_SYNCPARMS & 0xff; } else if (IS_ULTRA2(isp)) { off = ISP_40M_SYNCPARMS >> 8; per = ISP_40M_SYNCPARMS & 0xff; } else if (IS_1240(isp)) { off = ISP_20M_SYNCPARMS >> 8; per = ISP_20M_SYNCPARMS & 0xff; } else if ((isp->isp_bustype == ISP_BT_SBUS && isp->isp_type < ISP_HA_SCSI_1020A) || (isp->isp_bustype == ISP_BT_PCI && isp->isp_type < ISP_HA_SCSI_1040) || (isp->isp_clock && isp->isp_clock < 60) || (sdp->isp_ultramode == 0)) { off = ISP_10M_SYNCPARMS >> 8; per = ISP_10M_SYNCPARMS & 0xff; } else { off = ISP_20M_SYNCPARMS_1040 >> 8; per = ISP_20M_SYNCPARMS_1040 & 0xff; } sdp->isp_devparam[tgt].goal_offset = sdp->isp_devparam[tgt].nvrm_offset = off; sdp->isp_devparam[tgt].goal_period = sdp->isp_devparam[tgt].nvrm_period = per; } /* * If we're a dual bus card, just copy the data over */ if (sdp1) { *sdp1 = *sdp; sdp1->isp_initiator_id = DEFAULT_IID(isp, 1); } /* * If we've not been told to avoid reading NVRAM, try and read it. * If we're successful reading it, we can then return because NVRAM * will tell us what the desired settings are. Otherwise, we establish * some reasonable 'fake' nvram and goal defaults. */ if ((isp->isp_confopts & ISP_CFG_NONVRAM) == 0) { mbreg_t mbs; if (isp_read_nvram(isp, 0) == 0) { if (IS_DUALBUS(isp)) { if (isp_read_nvram(isp, 1) == 0) { return; } } } MBSINIT(&mbs, MBOX_GET_ACT_NEG_STATE, MBLOGNONE, 0); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { sdp->isp_req_ack_active_neg = 1; sdp->isp_data_line_active_neg = 1; if (sdp1) { sdp1->isp_req_ack_active_neg = 1; sdp1->isp_data_line_active_neg = 1; } } else { sdp->isp_req_ack_active_neg = (mbs.param[1] >> 4) & 0x1; sdp->isp_data_line_active_neg = (mbs.param[1] >> 5) & 0x1; if (sdp1) { sdp1->isp_req_ack_active_neg = (mbs.param[2] >> 4) & 0x1; sdp1->isp_data_line_active_neg = (mbs.param[2] >> 5) & 0x1; } } } } static void isp_setdfltfcparm(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); /* * Establish some default parameters. */ fcp->role = DEFAULT_ROLE(isp, chan); fcp->isp_maxalloc = ICB_DFLT_ALLOC; fcp->isp_retry_delay = ICB_DFLT_RDELAY; fcp->isp_retry_count = ICB_DFLT_RCOUNT; fcp->isp_loopid = DEFAULT_LOOPID(isp, chan); fcp->isp_wwnn_nvram = DEFAULT_NODEWWN(isp, chan); fcp->isp_wwpn_nvram = DEFAULT_PORTWWN(isp, chan); fcp->isp_fwoptions = 0; fcp->isp_xfwoptions = 0; fcp->isp_zfwoptions = 0; fcp->isp_lasthdl = NIL_HANDLE; fcp->isp_login_hdl = NIL_HANDLE; if (IS_24XX(isp)) { fcp->isp_fwoptions |= ICB2400_OPT1_FAIRNESS; fcp->isp_fwoptions |= ICB2400_OPT1_HARD_ADDRESS; if (isp->isp_confopts & ISP_CFG_FULL_DUPLEX) fcp->isp_fwoptions |= ICB2400_OPT1_FULL_DUPLEX; fcp->isp_fwoptions |= ICB2400_OPT1_BOTH_WWNS; fcp->isp_xfwoptions |= ICB2400_OPT2_LOOP_2_PTP; fcp->isp_zfwoptions |= ICB2400_OPT3_RATE_AUTO; } else { fcp->isp_fwoptions |= ICBOPT_FAIRNESS; fcp->isp_fwoptions |= ICBOPT_PDBCHANGE_AE; fcp->isp_fwoptions |= ICBOPT_HARD_ADDRESS; if (isp->isp_confopts & ISP_CFG_FULL_DUPLEX) fcp->isp_fwoptions |= ICBOPT_FULL_DUPLEX; /* * Make sure this is turned off now until we get * extended options from NVRAM */ fcp->isp_fwoptions &= ~ICBOPT_EXTENDED; fcp->isp_xfwoptions |= ICBXOPT_LOOP_2_PTP; fcp->isp_zfwoptions |= ICBZOPT_RATE_AUTO; } /* * Now try and read NVRAM unless told to not do so. * This will set fcparam's isp_wwnn_nvram && isp_wwpn_nvram. */ if ((isp->isp_confopts & ISP_CFG_NONVRAM) == 0) { int i, j = 0; /* * Give a couple of tries at reading NVRAM. */ for (i = 0; i < 2; i++) { j = isp_read_nvram(isp, chan); if (j == 0) { break; } } if (j) { isp->isp_confopts |= ISP_CFG_NONVRAM; } } fcp->isp_wwnn = ACTIVE_NODEWWN(isp, chan); fcp->isp_wwpn = ACTIVE_PORTWWN(isp, chan); isp_prt(isp, ISP_LOGCONFIG, "Chan %d 0x%08x%08x/0x%08x%08x Role %s", chan, (uint32_t) (fcp->isp_wwnn >> 32), (uint32_t) (fcp->isp_wwnn), (uint32_t) (fcp->isp_wwpn >> 32), (uint32_t) (fcp->isp_wwpn), isp_class3_roles[fcp->role]); } /* * Re-initialize the ISP and complete all orphaned commands * with a 'botched' notice. The reset/init routines should * not disturb an already active list of commands. */ int isp_reinit(ispsoftc_t *isp, int do_load_defaults) { int i, res = 0; if (isp->isp_state > ISP_RESETSTATE) isp_stop(isp); if (isp->isp_state != ISP_RESETSTATE) isp_reset(isp, do_load_defaults); if (isp->isp_state != ISP_RESETSTATE) { res = EIO; isp_prt(isp, ISP_LOGERR, "%s: cannot reset card", __func__); goto cleanup; } isp_init(isp); if (isp->isp_state > ISP_RESETSTATE && isp->isp_state != ISP_RUNSTATE) { res = EIO; isp_prt(isp, ISP_LOGERR, "%s: cannot init card", __func__); ISP_DISABLE_INTS(isp); if (IS_FC(isp)) { /* * If we're in ISP_ROLE_NONE, turn off the lasers. */ if (!IS_24XX(isp)) { ISP_WRITE(isp, BIU2100_CSR, BIU2100_FPM0_REGS); ISP_WRITE(isp, FPM_DIAG_CONFIG, FPM_SOFT_RESET); ISP_WRITE(isp, BIU2100_CSR, BIU2100_FB_REGS); ISP_WRITE(isp, FBM_CMD, FBMCMD_FIFO_RESET_ALL); ISP_WRITE(isp, BIU2100_CSR, BIU2100_RISC_REGS); } } } cleanup: isp_clear_commands(isp); if (IS_FC(isp)) { for (i = 0; i < isp->isp_nchan; i++) isp_clear_portdb(isp, i); } return (res); } /* * NVRAM Routines */ static int isp_read_nvram(ispsoftc_t *isp, int bus) { int i, amt, retval; uint8_t csum, minversion; union { uint8_t _x[ISP2400_NVRAM_SIZE]; uint16_t _s[ISP2400_NVRAM_SIZE>>1]; } _n; #define nvram_data _n._x #define nvram_words _n._s if (IS_24XX(isp)) { return (isp_read_nvram_2400(isp, nvram_data)); } else if (IS_FC(isp)) { amt = ISP2100_NVRAM_SIZE; minversion = 1; } else if (IS_ULTRA2(isp)) { amt = ISP1080_NVRAM_SIZE; minversion = 0; } else { amt = ISP_NVRAM_SIZE; minversion = 2; } for (i = 0; i < amt>>1; i++) { isp_rdnvram_word(isp, i, &nvram_words[i]); } if (nvram_data[0] != 'I' || nvram_data[1] != 'S' || nvram_data[2] != 'P') { if (isp->isp_bustype != ISP_BT_SBUS) { isp_prt(isp, ISP_LOGWARN, "invalid NVRAM header"); isp_prt(isp, ISP_LOGDEBUG0, "%x %x %x", nvram_data[0], nvram_data[1], nvram_data[2]); } retval = -1; goto out; } for (csum = 0, i = 0; i < amt; i++) { csum += nvram_data[i]; } if (csum != 0) { isp_prt(isp, ISP_LOGWARN, "invalid NVRAM checksum"); retval = -1; goto out; } if (ISP_NVRAM_VERSION(nvram_data) < minversion) { isp_prt(isp, ISP_LOGWARN, "version %d NVRAM not understood", ISP_NVRAM_VERSION(nvram_data)); retval = -1; goto out; } if (IS_ULTRA3(isp)) { isp_parse_nvram_12160(isp, bus, nvram_data); } else if (IS_1080(isp)) { isp_parse_nvram_1080(isp, bus, nvram_data); } else if (IS_1280(isp) || IS_1240(isp)) { isp_parse_nvram_1080(isp, bus, nvram_data); } else if (IS_SCSI(isp)) { isp_parse_nvram_1020(isp, nvram_data); } else { isp_parse_nvram_2100(isp, nvram_data); } retval = 0; out: return (retval); #undef nvram_data #undef nvram_words } static int isp_read_nvram_2400(ispsoftc_t *isp, uint8_t *nvram_data) { int retval = 0; uint32_t addr, csum, lwrds, *dptr; if (isp->isp_port) { addr = ISP2400_NVRAM_PORT1_ADDR; } else { addr = ISP2400_NVRAM_PORT0_ADDR; } dptr = (uint32_t *) nvram_data; for (lwrds = 0; lwrds < ISP2400_NVRAM_SIZE >> 2; lwrds++) { isp_rd_2400_nvram(isp, addr++, dptr++); } if (nvram_data[0] != 'I' || nvram_data[1] != 'S' || nvram_data[2] != 'P') { isp_prt(isp, ISP_LOGWARN, "invalid NVRAM header (%x %x %x)", nvram_data[0], nvram_data[1], nvram_data[2]); retval = -1; goto out; } dptr = (uint32_t *) nvram_data; for (csum = 0, lwrds = 0; lwrds < ISP2400_NVRAM_SIZE >> 2; lwrds++) { uint32_t tmp; ISP_IOXGET_32(isp, &dptr[lwrds], tmp); csum += tmp; } if (csum != 0) { isp_prt(isp, ISP_LOGWARN, "invalid NVRAM checksum"); retval = -1; goto out; } isp_parse_nvram_2400(isp, nvram_data); out: return (retval); } static void isp_rdnvram_word(ispsoftc_t *isp, int wo, uint16_t *rp) { int i, cbits; uint16_t bit, rqst, junk; ISP_WRITE(isp, BIU_NVRAM, BIU_NVRAM_SELECT); ISP_DELAY(10); ISP_WRITE(isp, BIU_NVRAM, BIU_NVRAM_SELECT|BIU_NVRAM_CLOCK); ISP_DELAY(10); if (IS_FC(isp)) { wo &= ((ISP2100_NVRAM_SIZE >> 1) - 1); if (IS_2312(isp) && isp->isp_port) { wo += 128; } rqst = (ISP_NVRAM_READ << 8) | wo; cbits = 10; } else if (IS_ULTRA2(isp)) { wo &= ((ISP1080_NVRAM_SIZE >> 1) - 1); rqst = (ISP_NVRAM_READ << 8) | wo; cbits = 10; } else { wo &= ((ISP_NVRAM_SIZE >> 1) - 1); rqst = (ISP_NVRAM_READ << 6) | wo; cbits = 8; } /* * Clock the word select request out... */ for (i = cbits; i >= 0; i--) { if ((rqst >> i) & 1) { bit = BIU_NVRAM_SELECT | BIU_NVRAM_DATAOUT; } else { bit = BIU_NVRAM_SELECT; } ISP_WRITE(isp, BIU_NVRAM, bit); ISP_DELAY(10); junk = ISP_READ(isp, BIU_NVRAM); /* force PCI flush */ ISP_WRITE(isp, BIU_NVRAM, bit | BIU_NVRAM_CLOCK); ISP_DELAY(10); junk = ISP_READ(isp, BIU_NVRAM); /* force PCI flush */ ISP_WRITE(isp, BIU_NVRAM, bit); ISP_DELAY(10); junk = ISP_READ(isp, BIU_NVRAM); /* force PCI flush */ } /* * Now read the result back in (bits come back in MSB format). */ *rp = 0; for (i = 0; i < 16; i++) { uint16_t rv; *rp <<= 1; ISP_WRITE(isp, BIU_NVRAM, BIU_NVRAM_SELECT|BIU_NVRAM_CLOCK); ISP_DELAY(10); rv = ISP_READ(isp, BIU_NVRAM); if (rv & BIU_NVRAM_DATAIN) { *rp |= 1; } ISP_DELAY(10); ISP_WRITE(isp, BIU_NVRAM, BIU_NVRAM_SELECT); ISP_DELAY(10); junk = ISP_READ(isp, BIU_NVRAM); /* force PCI flush */ } ISP_WRITE(isp, BIU_NVRAM, 0); ISP_DELAY(10); junk = ISP_READ(isp, BIU_NVRAM); /* force PCI flush */ ISP_SWIZZLE_NVRAM_WORD(isp, rp); } static void isp_rd_2400_nvram(ispsoftc_t *isp, uint32_t addr, uint32_t *rp) { int loops = 0; uint32_t base = 0x7ffe0000; uint32_t tmp = 0; if (IS_26XX(isp)) { base = 0x7fe7c000; /* XXX: Observation, may be wrong. */ } else if (IS_25XX(isp)) { base = 0x7ff00000 | 0x48000; } ISP_WRITE(isp, BIU2400_FLASH_ADDR, base | addr); for (loops = 0; loops < 5000; loops++) { ISP_DELAY(10); tmp = ISP_READ(isp, BIU2400_FLASH_ADDR); if ((tmp & (1U << 31)) != 0) { break; } } if (tmp & (1U << 31)) { *rp = ISP_READ(isp, BIU2400_FLASH_DATA); ISP_SWIZZLE_NVRAM_LONG(isp, rp); } else { *rp = 0xffffffff; } } static void isp_parse_nvram_1020(ispsoftc_t *isp, uint8_t *nvram_data) { sdparam *sdp = SDPARAM(isp, 0); int tgt; sdp->isp_fifo_threshold = ISP_NVRAM_FIFO_THRESHOLD(nvram_data) | (ISP_NVRAM_FIFO_THRESHOLD_128(nvram_data) << 2); if ((isp->isp_confopts & ISP_CFG_OWNLOOPID) == 0) sdp->isp_initiator_id = ISP_NVRAM_INITIATOR_ID(nvram_data); sdp->isp_bus_reset_delay = ISP_NVRAM_BUS_RESET_DELAY(nvram_data); sdp->isp_retry_count = ISP_NVRAM_BUS_RETRY_COUNT(nvram_data); sdp->isp_retry_delay = ISP_NVRAM_BUS_RETRY_DELAY(nvram_data); sdp->isp_async_data_setup = ISP_NVRAM_ASYNC_DATA_SETUP_TIME(nvram_data); if (isp->isp_type >= ISP_HA_SCSI_1040) { if (sdp->isp_async_data_setup < 9) { sdp->isp_async_data_setup = 9; } } else { if (sdp->isp_async_data_setup != 6) { sdp->isp_async_data_setup = 6; } } sdp->isp_req_ack_active_neg = ISP_NVRAM_REQ_ACK_ACTIVE_NEGATION(nvram_data); sdp->isp_data_line_active_neg = ISP_NVRAM_DATA_LINE_ACTIVE_NEGATION(nvram_data); sdp->isp_data_dma_burst_enabl = ISP_NVRAM_DATA_DMA_BURST_ENABLE(nvram_data); sdp->isp_cmd_dma_burst_enable = ISP_NVRAM_CMD_DMA_BURST_ENABLE(nvram_data); sdp->isp_tag_aging = ISP_NVRAM_TAG_AGE_LIMIT(nvram_data); sdp->isp_selection_timeout = ISP_NVRAM_SELECTION_TIMEOUT(nvram_data); sdp->isp_max_queue_depth = ISP_NVRAM_MAX_QUEUE_DEPTH(nvram_data); sdp->isp_fast_mttr = ISP_NVRAM_FAST_MTTR_ENABLE(nvram_data); for (tgt = 0; tgt < MAX_TARGETS; tgt++) { sdp->isp_devparam[tgt].dev_enable = ISP_NVRAM_TGT_DEVICE_ENABLE(nvram_data, tgt); sdp->isp_devparam[tgt].exc_throttle = ISP_NVRAM_TGT_EXEC_THROTTLE(nvram_data, tgt); sdp->isp_devparam[tgt].nvrm_offset = ISP_NVRAM_TGT_SYNC_OFFSET(nvram_data, tgt); sdp->isp_devparam[tgt].nvrm_period = ISP_NVRAM_TGT_SYNC_PERIOD(nvram_data, tgt); /* * We probably shouldn't lie about this, but it * it makes it much safer if we limit NVRAM values * to sanity. */ if (isp->isp_type < ISP_HA_SCSI_1040) { /* * If we're not ultra, we can't possibly * be a shorter period than this. */ if (sdp->isp_devparam[tgt].nvrm_period < 0x19) { sdp->isp_devparam[tgt].nvrm_period = 0x19; } if (sdp->isp_devparam[tgt].nvrm_offset > 0xc) { sdp->isp_devparam[tgt].nvrm_offset = 0x0c; } } else { if (sdp->isp_devparam[tgt].nvrm_offset > 0x8) { sdp->isp_devparam[tgt].nvrm_offset = 0x8; } } sdp->isp_devparam[tgt].nvrm_flags = 0; if (ISP_NVRAM_TGT_RENEG(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_RENEG; sdp->isp_devparam[tgt].nvrm_flags |= DPARM_ARQ; if (ISP_NVRAM_TGT_TQING(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_TQING; if (ISP_NVRAM_TGT_SYNC(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_SYNC; if (ISP_NVRAM_TGT_WIDE(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_WIDE; if (ISP_NVRAM_TGT_PARITY(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_PARITY; if (ISP_NVRAM_TGT_DISC(nvram_data, tgt)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_DISC; sdp->isp_devparam[tgt].actv_flags = 0; /* we don't know */ sdp->isp_devparam[tgt].goal_offset = sdp->isp_devparam[tgt].nvrm_offset; sdp->isp_devparam[tgt].goal_period = sdp->isp_devparam[tgt].nvrm_period; sdp->isp_devparam[tgt].goal_flags = sdp->isp_devparam[tgt].nvrm_flags; } } static void isp_parse_nvram_1080(ispsoftc_t *isp, int bus, uint8_t *nvram_data) { sdparam *sdp = SDPARAM(isp, bus); int tgt; sdp->isp_fifo_threshold = ISP1080_NVRAM_FIFO_THRESHOLD(nvram_data); if ((isp->isp_confopts & ISP_CFG_OWNLOOPID) == 0) sdp->isp_initiator_id = ISP1080_NVRAM_INITIATOR_ID(nvram_data, bus); sdp->isp_bus_reset_delay = ISP1080_NVRAM_BUS_RESET_DELAY(nvram_data, bus); sdp->isp_retry_count = ISP1080_NVRAM_BUS_RETRY_COUNT(nvram_data, bus); sdp->isp_retry_delay = ISP1080_NVRAM_BUS_RETRY_DELAY(nvram_data, bus); sdp->isp_async_data_setup = ISP1080_NVRAM_ASYNC_DATA_SETUP_TIME(nvram_data, bus); sdp->isp_req_ack_active_neg = ISP1080_NVRAM_REQ_ACK_ACTIVE_NEGATION(nvram_data, bus); sdp->isp_data_line_active_neg = ISP1080_NVRAM_DATA_LINE_ACTIVE_NEGATION(nvram_data, bus); sdp->isp_data_dma_burst_enabl = ISP1080_NVRAM_BURST_ENABLE(nvram_data); sdp->isp_cmd_dma_burst_enable = ISP1080_NVRAM_BURST_ENABLE(nvram_data); sdp->isp_selection_timeout = ISP1080_NVRAM_SELECTION_TIMEOUT(nvram_data, bus); sdp->isp_max_queue_depth = ISP1080_NVRAM_MAX_QUEUE_DEPTH(nvram_data, bus); for (tgt = 0; tgt < MAX_TARGETS; tgt++) { sdp->isp_devparam[tgt].dev_enable = ISP1080_NVRAM_TGT_DEVICE_ENABLE(nvram_data, tgt, bus); sdp->isp_devparam[tgt].exc_throttle = ISP1080_NVRAM_TGT_EXEC_THROTTLE(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_offset = ISP1080_NVRAM_TGT_SYNC_OFFSET(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_period = ISP1080_NVRAM_TGT_SYNC_PERIOD(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_flags = 0; if (ISP1080_NVRAM_TGT_RENEG(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_RENEG; sdp->isp_devparam[tgt].nvrm_flags |= DPARM_ARQ; if (ISP1080_NVRAM_TGT_TQING(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_TQING; if (ISP1080_NVRAM_TGT_SYNC(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_SYNC; if (ISP1080_NVRAM_TGT_WIDE(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_WIDE; if (ISP1080_NVRAM_TGT_PARITY(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_PARITY; if (ISP1080_NVRAM_TGT_DISC(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_DISC; sdp->isp_devparam[tgt].actv_flags = 0; sdp->isp_devparam[tgt].goal_offset = sdp->isp_devparam[tgt].nvrm_offset; sdp->isp_devparam[tgt].goal_period = sdp->isp_devparam[tgt].nvrm_period; sdp->isp_devparam[tgt].goal_flags = sdp->isp_devparam[tgt].nvrm_flags; } } static void isp_parse_nvram_12160(ispsoftc_t *isp, int bus, uint8_t *nvram_data) { sdparam *sdp = SDPARAM(isp, bus); int tgt; sdp->isp_fifo_threshold = ISP12160_NVRAM_FIFO_THRESHOLD(nvram_data); if ((isp->isp_confopts & ISP_CFG_OWNLOOPID) == 0) sdp->isp_initiator_id = ISP12160_NVRAM_INITIATOR_ID(nvram_data, bus); sdp->isp_bus_reset_delay = ISP12160_NVRAM_BUS_RESET_DELAY(nvram_data, bus); sdp->isp_retry_count = ISP12160_NVRAM_BUS_RETRY_COUNT(nvram_data, bus); sdp->isp_retry_delay = ISP12160_NVRAM_BUS_RETRY_DELAY(nvram_data, bus); sdp->isp_async_data_setup = ISP12160_NVRAM_ASYNC_DATA_SETUP_TIME(nvram_data, bus); sdp->isp_req_ack_active_neg = ISP12160_NVRAM_REQ_ACK_ACTIVE_NEGATION(nvram_data, bus); sdp->isp_data_line_active_neg = ISP12160_NVRAM_DATA_LINE_ACTIVE_NEGATION(nvram_data, bus); sdp->isp_data_dma_burst_enabl = ISP12160_NVRAM_BURST_ENABLE(nvram_data); sdp->isp_cmd_dma_burst_enable = ISP12160_NVRAM_BURST_ENABLE(nvram_data); sdp->isp_selection_timeout = ISP12160_NVRAM_SELECTION_TIMEOUT(nvram_data, bus); sdp->isp_max_queue_depth = ISP12160_NVRAM_MAX_QUEUE_DEPTH(nvram_data, bus); for (tgt = 0; tgt < MAX_TARGETS; tgt++) { sdp->isp_devparam[tgt].dev_enable = ISP12160_NVRAM_TGT_DEVICE_ENABLE(nvram_data, tgt, bus); sdp->isp_devparam[tgt].exc_throttle = ISP12160_NVRAM_TGT_EXEC_THROTTLE(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_offset = ISP12160_NVRAM_TGT_SYNC_OFFSET(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_period = ISP12160_NVRAM_TGT_SYNC_PERIOD(nvram_data, tgt, bus); sdp->isp_devparam[tgt].nvrm_flags = 0; if (ISP12160_NVRAM_TGT_RENEG(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_RENEG; sdp->isp_devparam[tgt].nvrm_flags |= DPARM_ARQ; if (ISP12160_NVRAM_TGT_TQING(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_TQING; if (ISP12160_NVRAM_TGT_SYNC(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_SYNC; if (ISP12160_NVRAM_TGT_WIDE(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_WIDE; if (ISP12160_NVRAM_TGT_PARITY(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_PARITY; if (ISP12160_NVRAM_TGT_DISC(nvram_data, tgt, bus)) sdp->isp_devparam[tgt].nvrm_flags |= DPARM_DISC; sdp->isp_devparam[tgt].actv_flags = 0; sdp->isp_devparam[tgt].goal_offset = sdp->isp_devparam[tgt].nvrm_offset; sdp->isp_devparam[tgt].goal_period = sdp->isp_devparam[tgt].nvrm_period; sdp->isp_devparam[tgt].goal_flags = sdp->isp_devparam[tgt].nvrm_flags; } } static void isp_parse_nvram_2100(ispsoftc_t *isp, uint8_t *nvram_data) { fcparam *fcp = FCPARAM(isp, 0); uint64_t wwn; /* * There is NVRAM storage for both Port and Node entities- * but the Node entity appears to be unused on all the cards * I can find. However, we should account for this being set * at some point in the future. * * Qlogic WWNs have an NAA of 2, but usually nothing shows up in * bits 48..60. In the case of the 2202, it appears that they do * use bit 48 to distinguish between the two instances on the card. * The 2204, which I've never seen, *probably* extends this method. */ wwn = ISP2100_NVRAM_PORT_NAME(nvram_data); if (wwn) { isp_prt(isp, ISP_LOGCONFIG, "NVRAM Port WWN 0x%08x%08x", (uint32_t) (wwn >> 32), (uint32_t) (wwn)); if ((wwn >> 60) == 0) { wwn |= (((uint64_t) 2)<< 60); } } fcp->isp_wwpn_nvram = wwn; if (IS_2200(isp) || IS_23XX(isp)) { wwn = ISP2100_NVRAM_NODE_NAME(nvram_data); if (wwn) { isp_prt(isp, ISP_LOGCONFIG, "NVRAM Node WWN 0x%08x%08x", (uint32_t) (wwn >> 32), (uint32_t) (wwn)); if ((wwn >> 60) == 0) { wwn |= (((uint64_t) 2)<< 60); } } else { wwn = fcp->isp_wwpn_nvram & ~((uint64_t) 0xfff << 48); } } else { wwn &= ~((uint64_t) 0xfff << 48); } fcp->isp_wwnn_nvram = wwn; fcp->isp_maxalloc = ISP2100_NVRAM_MAXIOCBALLOCATION(nvram_data); if ((isp->isp_confopts & ISP_CFG_OWNFSZ) == 0) { DEFAULT_FRAMESIZE(isp) = ISP2100_NVRAM_MAXFRAMELENGTH(nvram_data); } fcp->isp_retry_delay = ISP2100_NVRAM_RETRY_DELAY(nvram_data); fcp->isp_retry_count = ISP2100_NVRAM_RETRY_COUNT(nvram_data); if ((isp->isp_confopts & ISP_CFG_OWNLOOPID) == 0) { fcp->isp_loopid = ISP2100_NVRAM_HARDLOOPID(nvram_data); } if ((isp->isp_confopts & ISP_CFG_OWNEXCTHROTTLE) == 0) { DEFAULT_EXEC_THROTTLE(isp) = ISP2100_NVRAM_EXECUTION_THROTTLE(nvram_data); } fcp->isp_fwoptions = ISP2100_NVRAM_OPTIONS(nvram_data); isp_prt(isp, ISP_LOGDEBUG0, "NVRAM 0x%08x%08x 0x%08x%08x maxalloc %d maxframelen %d", (uint32_t) (fcp->isp_wwnn_nvram >> 32), (uint32_t) fcp->isp_wwnn_nvram, (uint32_t) (fcp->isp_wwpn_nvram >> 32), (uint32_t) fcp->isp_wwpn_nvram, ISP2100_NVRAM_MAXIOCBALLOCATION(nvram_data), ISP2100_NVRAM_MAXFRAMELENGTH(nvram_data)); isp_prt(isp, ISP_LOGDEBUG0, "execthrottle %d fwoptions 0x%x hardloop %d tov %d", ISP2100_NVRAM_EXECUTION_THROTTLE(nvram_data), ISP2100_NVRAM_OPTIONS(nvram_data), ISP2100_NVRAM_HARDLOOPID(nvram_data), ISP2100_NVRAM_TOV(nvram_data)); fcp->isp_xfwoptions = ISP2100_XFW_OPTIONS(nvram_data); fcp->isp_zfwoptions = ISP2100_ZFW_OPTIONS(nvram_data); isp_prt(isp, ISP_LOGDEBUG0, "xfwoptions 0x%x zfw options 0x%x", ISP2100_XFW_OPTIONS(nvram_data), ISP2100_ZFW_OPTIONS(nvram_data)); } static void isp_parse_nvram_2400(ispsoftc_t *isp, uint8_t *nvram_data) { fcparam *fcp = FCPARAM(isp, 0); uint64_t wwn; isp_prt(isp, ISP_LOGDEBUG0, "NVRAM 0x%08x%08x 0x%08x%08x exchg_cnt %d maxframelen %d", (uint32_t) (ISP2400_NVRAM_NODE_NAME(nvram_data) >> 32), (uint32_t) (ISP2400_NVRAM_NODE_NAME(nvram_data)), (uint32_t) (ISP2400_NVRAM_PORT_NAME(nvram_data) >> 32), (uint32_t) (ISP2400_NVRAM_PORT_NAME(nvram_data)), ISP2400_NVRAM_EXCHANGE_COUNT(nvram_data), ISP2400_NVRAM_MAXFRAMELENGTH(nvram_data)); isp_prt(isp, ISP_LOGDEBUG0, "NVRAM execthr %d loopid %d fwopt1 0x%x fwopt2 0x%x fwopt3 0x%x", ISP2400_NVRAM_EXECUTION_THROTTLE(nvram_data), ISP2400_NVRAM_HARDLOOPID(nvram_data), ISP2400_NVRAM_FIRMWARE_OPTIONS1(nvram_data), ISP2400_NVRAM_FIRMWARE_OPTIONS2(nvram_data), ISP2400_NVRAM_FIRMWARE_OPTIONS3(nvram_data)); wwn = ISP2400_NVRAM_PORT_NAME(nvram_data); fcp->isp_wwpn_nvram = wwn; wwn = ISP2400_NVRAM_NODE_NAME(nvram_data); if (wwn) { if ((wwn >> 60) != 2 && (wwn >> 60) != 5) { wwn = 0; } } if (wwn == 0 && (fcp->isp_wwpn_nvram >> 60) == 2) { wwn = fcp->isp_wwpn_nvram; wwn &= ~((uint64_t) 0xfff << 48); } fcp->isp_wwnn_nvram = wwn; if (ISP2400_NVRAM_EXCHANGE_COUNT(nvram_data)) { fcp->isp_maxalloc = ISP2400_NVRAM_EXCHANGE_COUNT(nvram_data); } if ((isp->isp_confopts & ISP_CFG_OWNFSZ) == 0) { DEFAULT_FRAMESIZE(isp) = ISP2400_NVRAM_MAXFRAMELENGTH(nvram_data); } if ((isp->isp_confopts & ISP_CFG_OWNLOOPID) == 0) { fcp->isp_loopid = ISP2400_NVRAM_HARDLOOPID(nvram_data); } if ((isp->isp_confopts & ISP_CFG_OWNEXCTHROTTLE) == 0) { DEFAULT_EXEC_THROTTLE(isp) = ISP2400_NVRAM_EXECUTION_THROTTLE(nvram_data); } fcp->isp_fwoptions = ISP2400_NVRAM_FIRMWARE_OPTIONS1(nvram_data); fcp->isp_xfwoptions = ISP2400_NVRAM_FIRMWARE_OPTIONS2(nvram_data); fcp->isp_zfwoptions = ISP2400_NVRAM_FIRMWARE_OPTIONS3(nvram_data); } Index: projects/import-googletest-1.8.1/sys/dev/isp/isp_freebsd.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/isp/isp_freebsd.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/isp/isp_freebsd.c (revision 345026) @@ -1,4314 +1,4328 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2017 Alexander Motin * Copyright (c) 1997-2009 by Matthew Jacob * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice immediately at the beginning of the file, without modification, * this list of conditions, and the following disclaimer. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Platform (FreeBSD) dependent common attachment code for Qlogic adapters. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include MODULE_VERSION(isp, 1); MODULE_DEPEND(isp, cam, 1, 1, 1); int isp_announced = 0; int isp_loop_down_limit = 60; /* default loop down limit */ int isp_quickboot_time = 7; /* don't wait more than N secs for loop up */ int isp_gone_device_time = 30; /* grace time before reporting device lost */ static const char prom3[] = "Chan %d [%u] PortID 0x%06x Departed because of %s"; static void isp_freeze_loopdown(ispsoftc_t *, int); static void isp_loop_changed(ispsoftc_t *isp, int chan); static d_ioctl_t ispioctl; static void isp_cam_async(void *, uint32_t, struct cam_path *, void *); static void isp_poll(struct cam_sim *); static timeout_t isp_watchdog; static timeout_t isp_gdt; static task_fn_t isp_gdt_task; static void isp_kthread(void *); static void isp_action(struct cam_sim *, union ccb *); static int isp_timer_count; static void isp_timer(void *); static struct cdevsw isp_cdevsw = { .d_version = D_VERSION, .d_ioctl = ispioctl, .d_name = "isp", }; static int isp_role_sysctl(SYSCTL_HANDLER_ARGS) { ispsoftc_t *isp = (ispsoftc_t *)arg1; int chan = arg2; int error, old, value; value = FCPARAM(isp, chan)->role; error = sysctl_handle_int(oidp, &value, 0, req); if ((error != 0) || (req->newptr == NULL)) return (error); if (value < ISP_ROLE_NONE || value > ISP_ROLE_BOTH) return (EINVAL); ISP_LOCK(isp); old = FCPARAM(isp, chan)->role; /* We don't allow target mode switch from here. */ value = (old & ISP_ROLE_TARGET) | (value & ISP_ROLE_INITIATOR); /* If nothing has changed -- we are done. */ if (value == old) { ISP_UNLOCK(isp); return (0); } /* Actually change the role. */ error = isp_control(isp, ISPCTL_CHANGE_ROLE, chan, value); ISP_UNLOCK(isp); return (error); } static int isp_attach_chan(ispsoftc_t *isp, struct cam_devq *devq, int chan) { struct ccb_setasync csa; struct cam_sim *sim; struct cam_path *path; #ifdef ISP_TARGET_MODE int i; #endif sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp, device_get_unit(isp->isp_dev), &isp->isp_lock, isp->isp_maxcmds, isp->isp_maxcmds, devq); if (sim == NULL) return (ENOMEM); ISP_LOCK(isp); if (xpt_bus_register(sim, isp->isp_dev, chan) != CAM_SUCCESS) { ISP_UNLOCK(isp); cam_sim_free(sim, FALSE); return (EIO); } ISP_UNLOCK(isp); if (xpt_create_path(&path, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { ISP_LOCK(isp); xpt_bus_deregister(cam_sim_path(sim)); ISP_UNLOCK(isp); cam_sim_free(sim, FALSE); return (ENXIO); } xpt_setup_ccb(&csa.ccb_h, path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = AC_LOST_DEVICE; csa.callback = isp_cam_async; csa.callback_arg = sim; ISP_LOCK(isp); xpt_action((union ccb *)&csa); ISP_UNLOCK(isp); if (IS_SCSI(isp)) { struct isp_spi *spi = ISP_SPI_PC(isp, chan); spi->sim = sim; spi->path = path; #ifdef ISP_TARGET_MODE TAILQ_INIT(&spi->waitq); STAILQ_INIT(&spi->ntfree); for (i = 0; i < ATPDPSIZE; i++) STAILQ_INSERT_TAIL(&spi->ntfree, &spi->ntpool[i], next); LIST_INIT(&spi->atfree); for (i = ATPDPSIZE-1; i >= 0; i--) LIST_INSERT_HEAD(&spi->atfree, &spi->atpool[i], next); for (i = 0; i < ATPDPHASHSIZE; i++) LIST_INIT(&spi->atused[i]); #endif } else { fcparam *fcp = FCPARAM(isp, chan); struct isp_fc *fc = ISP_FC_PC(isp, chan); struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(isp->isp_osinfo.dev); struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev); char name[16]; ISP_LOCK(isp); fc->sim = sim; fc->path = path; fc->isp = isp; fc->ready = 1; fcp->isp_use_gft_id = 1; fcp->isp_use_gff_id = 1; callout_init_mtx(&fc->gdt, &isp->isp_lock, 0); TASK_INIT(&fc->gtask, 1, isp_gdt_task, fc); #ifdef ISP_TARGET_MODE TAILQ_INIT(&fc->waitq); STAILQ_INIT(&fc->ntfree); for (i = 0; i < ATPDPSIZE; i++) STAILQ_INSERT_TAIL(&fc->ntfree, &fc->ntpool[i], next); LIST_INIT(&fc->atfree); for (i = ATPDPSIZE-1; i >= 0; i--) LIST_INSERT_HEAD(&fc->atfree, &fc->atpool[i], next); for (i = 0; i < ATPDPHASHSIZE; i++) LIST_INIT(&fc->atused[i]); #endif isp_loop_changed(isp, chan); ISP_UNLOCK(isp); if (kproc_create(isp_kthread, fc, &fc->kproc, 0, 0, "%s_%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) { xpt_free_path(fc->path); ISP_LOCK(isp); xpt_bus_deregister(cam_sim_path(fc->sim)); ISP_UNLOCK(isp); cam_sim_free(fc->sim, FALSE); return (ENOMEM); } fc->num_threads += 1; if (chan > 0) { snprintf(name, sizeof(name), "chan%d", chan); tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, name, CTLFLAG_RW, 0, "Virtual channel"); } SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwnn", CTLFLAG_RD, &fcp->isp_wwnn, "World Wide Node Name"); SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwpn", CTLFLAG_RD, &fcp->isp_wwpn, "World Wide Port Name"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "loop_down_limit", CTLFLAG_RW, &fc->loop_down_limit, 0, "Loop Down Limit"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "gone_device_time", CTLFLAG_RW, &fc->gone_device_time, 0, "Gone Device Time"); #if defined(ISP_TARGET_MODE) && defined(DEBUG) SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "inject_lost_data_frame", CTLFLAG_RW, &fc->inject_lost_data_frame, 0, "Cause a Lost Frame on a Read"); #endif SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "role", CTLTYPE_INT | CTLFLAG_RW, isp, chan, isp_role_sysctl, "I", "Current role"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "speed", CTLFLAG_RD, &fcp->isp_gbspeed, 0, "Connection speed in gigabits"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "linkstate", CTLFLAG_RD, &fcp->isp_linkstate, 0, "Link state"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "fwstate", CTLFLAG_RD, &fcp->isp_fwstate, 0, "Firmware state"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "loopstate", CTLFLAG_RD, &fcp->isp_loopstate, 0, "Loop state"); SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "topo", CTLFLAG_RD, &fcp->isp_topo, 0, "Connection topology"); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "use_gft_id", CTLFLAG_RWTUN, &fcp->isp_use_gft_id, 0, "Use GFT_ID during fabric scan"); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "use_gff_id", CTLFLAG_RWTUN, &fcp->isp_use_gff_id, 0, "Use GFF_ID during fabric scan"); } return (0); } static void isp_detach_chan(ispsoftc_t *isp, int chan) { struct cam_sim *sim; struct cam_path *path; struct ccb_setasync csa; int *num_threads; ISP_GET_PC(isp, chan, sim, sim); ISP_GET_PC(isp, chan, path, path); ISP_GET_PC_ADDR(isp, chan, num_threads, num_threads); xpt_setup_ccb(&csa.ccb_h, path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = 0; csa.callback = isp_cam_async; csa.callback_arg = sim; xpt_action((union ccb *)&csa); xpt_free_path(path); xpt_bus_deregister(cam_sim_path(sim)); cam_sim_free(sim, FALSE); /* Wait for the channel's spawned threads to exit. */ wakeup(isp->isp_osinfo.pc.ptr); while (*num_threads != 0) mtx_sleep(isp, &isp->isp_lock, PRIBIO, "isp_reap", 100); } int isp_attach(ispsoftc_t *isp) { const char *nu = device_get_nameunit(isp->isp_osinfo.dev); int du = device_get_unit(isp->isp_dev); int chan; /* * Create the device queue for our SIM(s). */ isp->isp_osinfo.devq = cam_simq_alloc(isp->isp_maxcmds); if (isp->isp_osinfo.devq == NULL) { return (EIO); } for (chan = 0; chan < isp->isp_nchan; chan++) { if (isp_attach_chan(isp, isp->isp_osinfo.devq, chan)) { goto unwind; } } callout_init_mtx(&isp->isp_osinfo.tmo, &isp->isp_lock, 0); isp_timer_count = hz >> 2; callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp); isp->isp_osinfo.cdev = make_dev(&isp_cdevsw, du, UID_ROOT, GID_OPERATOR, 0600, "%s", nu); if (isp->isp_osinfo.cdev) { isp->isp_osinfo.cdev->si_drv1 = isp; } return (0); unwind: while (--chan >= 0) { struct cam_sim *sim; struct cam_path *path; ISP_GET_PC(isp, chan, sim, sim); ISP_GET_PC(isp, chan, path, path); xpt_free_path(path); ISP_LOCK(isp); xpt_bus_deregister(cam_sim_path(sim)); ISP_UNLOCK(isp); cam_sim_free(sim, FALSE); } cam_simq_free(isp->isp_osinfo.devq); isp->isp_osinfo.devq = NULL; return (-1); } int isp_detach(ispsoftc_t *isp) { int chan; if (isp->isp_osinfo.cdev) { destroy_dev(isp->isp_osinfo.cdev); isp->isp_osinfo.cdev = NULL; } ISP_LOCK(isp); /* Tell spawned threads that we're exiting. */ isp->isp_osinfo.is_exiting = 1; for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1) isp_detach_chan(isp, chan); ISP_UNLOCK(isp); callout_drain(&isp->isp_osinfo.tmo); cam_simq_free(isp->isp_osinfo.devq); return (0); } static void isp_freeze_loopdown(ispsoftc_t *isp, int chan) { struct isp_fc *fc = ISP_FC_PC(isp, chan); if (fc->sim == NULL) return; if (fc->simqfrozen == 0) { isp_prt(isp, ISP_LOGDEBUG0, "Chan %d Freeze simq (loopdown)", chan); fc->simqfrozen = SIMQFRZ_LOOPDOWN; xpt_hold_boot(); xpt_freeze_simq(fc->sim, 1); } else { isp_prt(isp, ISP_LOGDEBUG0, "Chan %d Mark simq frozen (loopdown)", chan); fc->simqfrozen |= SIMQFRZ_LOOPDOWN; } } static void isp_unfreeze_loopdown(ispsoftc_t *isp, int chan) { struct isp_fc *fc = ISP_FC_PC(isp, chan); if (fc->sim == NULL) return; int wasfrozen = fc->simqfrozen & SIMQFRZ_LOOPDOWN; fc->simqfrozen &= ~SIMQFRZ_LOOPDOWN; if (wasfrozen && fc->simqfrozen == 0) { isp_prt(isp, ISP_LOGDEBUG0, "Chan %d Release simq", chan); xpt_release_simq(fc->sim, 1); xpt_release_boot(); } } static int ispioctl(struct cdev *dev, u_long c, caddr_t addr, int flags, struct thread *td) { ispsoftc_t *isp; int nr, chan, retval = ENOTTY; isp = dev->si_drv1; switch (c) { case ISP_SDBLEV: { int olddblev = isp->isp_dblev; isp->isp_dblev = *(int *)addr; *(int *)addr = olddblev; retval = 0; break; } case ISP_GETROLE: chan = *(int *)addr; if (chan < 0 || chan >= isp->isp_nchan) { retval = -ENXIO; break; } if (IS_FC(isp)) { *(int *)addr = FCPARAM(isp, chan)->role; } else { *(int *)addr = ISP_ROLE_INITIATOR; } retval = 0; break; case ISP_SETROLE: if (IS_SCSI(isp)) break; nr = *(int *)addr; chan = nr >> 8; if (chan < 0 || chan >= isp->isp_nchan) { retval = -ENXIO; break; } nr &= 0xff; if (nr & ~(ISP_ROLE_INITIATOR|ISP_ROLE_TARGET)) { retval = EINVAL; break; } ISP_LOCK(isp); *(int *)addr = FCPARAM(isp, chan)->role; retval = isp_control(isp, ISPCTL_CHANGE_ROLE, chan, nr); ISP_UNLOCK(isp); retval = 0; break; case ISP_RESETHBA: ISP_LOCK(isp); isp_reinit(isp, 0); ISP_UNLOCK(isp); retval = 0; break; case ISP_RESCAN: if (IS_FC(isp)) { chan = *(intptr_t *)addr; if (chan < 0 || chan >= isp->isp_nchan) { retval = -ENXIO; break; } ISP_LOCK(isp); if (isp_fc_runstate(isp, chan, 5 * 1000000) != LOOP_READY) { retval = EIO; } else { retval = 0; } ISP_UNLOCK(isp); } break; case ISP_FC_LIP: if (IS_FC(isp)) { chan = *(intptr_t *)addr; if (chan < 0 || chan >= isp->isp_nchan) { retval = -ENXIO; break; } ISP_LOCK(isp); if (isp_control(isp, ISPCTL_SEND_LIP, chan)) { retval = EIO; } else { retval = 0; } ISP_UNLOCK(isp); } break; case ISP_FC_GETDINFO: { struct isp_fc_device *ifc = (struct isp_fc_device *) addr; fcportdb_t *lp; if (IS_SCSI(isp)) { break; } if (ifc->loopid >= MAX_FC_TARG) { retval = EINVAL; break; } lp = &FCPARAM(isp, ifc->chan)->portdb[ifc->loopid]; if (lp->state != FC_PORTDB_STATE_NIL) { ifc->role = (lp->prli_word3 & SVC3_ROLE_MASK) >> SVC3_ROLE_SHIFT; ifc->loopid = lp->handle; ifc->portid = lp->portid; ifc->node_wwn = lp->node_wwn; ifc->port_wwn = lp->port_wwn; retval = 0; } else { retval = ENODEV; } break; } case ISP_FC_GETHINFO: { struct isp_hba_device *hba = (struct isp_hba_device *) addr; int chan = hba->fc_channel; if (chan < 0 || chan >= isp->isp_nchan) { retval = ENXIO; break; } hba->fc_fw_major = ISP_FW_MAJORX(isp->isp_fwrev); hba->fc_fw_minor = ISP_FW_MINORX(isp->isp_fwrev); hba->fc_fw_micro = ISP_FW_MICROX(isp->isp_fwrev); hba->fc_nchannels = isp->isp_nchan; if (IS_FC(isp)) { hba->fc_nports = MAX_FC_TARG; hba->fc_speed = FCPARAM(isp, hba->fc_channel)->isp_gbspeed; hba->fc_topology = FCPARAM(isp, chan)->isp_topo + 1; hba->fc_loopid = FCPARAM(isp, chan)->isp_loopid; hba->nvram_node_wwn = FCPARAM(isp, chan)->isp_wwnn_nvram; hba->nvram_port_wwn = FCPARAM(isp, chan)->isp_wwpn_nvram; hba->active_node_wwn = FCPARAM(isp, chan)->isp_wwnn; hba->active_port_wwn = FCPARAM(isp, chan)->isp_wwpn; } else { hba->fc_nports = MAX_TARGETS; hba->fc_speed = 0; hba->fc_topology = 0; hba->nvram_node_wwn = 0ull; hba->nvram_port_wwn = 0ull; hba->active_node_wwn = 0ull; hba->active_port_wwn = 0ull; } retval = 0; break; } case ISP_TSK_MGMT: { int needmarker; struct isp_fc_tsk_mgmt *fct = (struct isp_fc_tsk_mgmt *) addr; uint16_t nphdl; mbreg_t mbs; if (IS_SCSI(isp)) { break; } chan = fct->chan; if (chan < 0 || chan >= isp->isp_nchan) { retval = -ENXIO; break; } needmarker = retval = 0; nphdl = fct->loopid; ISP_LOCK(isp); if (IS_24XX(isp)) { void *reqp; uint8_t resp[QENTRY_LEN]; isp24xx_tmf_t tmf; isp24xx_statusreq_t sp; fcparam *fcp = FCPARAM(isp, chan); fcportdb_t *lp; int i; for (i = 0; i < MAX_FC_TARG; i++) { lp = &fcp->portdb[i]; if (lp->handle == nphdl) { break; } } if (i == MAX_FC_TARG) { retval = ENXIO; ISP_UNLOCK(isp); break; } ISP_MEMZERO(&tmf, sizeof(tmf)); tmf.tmf_header.rqs_entry_type = RQSTYPE_TSK_MGMT; tmf.tmf_header.rqs_entry_count = 1; tmf.tmf_nphdl = lp->handle; tmf.tmf_delay = 2; tmf.tmf_timeout = 4; tmf.tmf_tidlo = lp->portid; tmf.tmf_tidhi = lp->portid >> 16; tmf.tmf_vpidx = ISP_GET_VPIDX(isp, chan); tmf.tmf_lun[1] = fct->lun & 0xff; if (fct->lun >= 256) { tmf.tmf_lun[0] = 0x40 | (fct->lun >> 8); } switch (fct->action) { case IPT_CLEAR_ACA: tmf.tmf_flags = ISP24XX_TMF_CLEAR_ACA; break; case IPT_TARGET_RESET: tmf.tmf_flags = ISP24XX_TMF_TARGET_RESET; needmarker = 1; break; case IPT_LUN_RESET: tmf.tmf_flags = ISP24XX_TMF_LUN_RESET; needmarker = 1; break; case IPT_CLEAR_TASK_SET: tmf.tmf_flags = ISP24XX_TMF_CLEAR_TASK_SET; needmarker = 1; break; case IPT_ABORT_TASK_SET: tmf.tmf_flags = ISP24XX_TMF_ABORT_TASK_SET; needmarker = 1; break; default: retval = EINVAL; break; } if (retval) { ISP_UNLOCK(isp); break; } /* Prepare space for response in memory */ memset(resp, 0xff, sizeof(resp)); tmf.tmf_handle = isp_allocate_handle(isp, resp, ISP_HANDLE_CTRL); if (tmf.tmf_handle == 0) { isp_prt(isp, ISP_LOGERR, "%s: TMF of Chan %d out of handles", __func__, chan); ISP_UNLOCK(isp); retval = ENOMEM; break; } /* Send request and wait for response. */ reqp = isp_getrqentry(isp); if (reqp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: TMF of Chan %d out of rqent", __func__, chan); isp_destroy_handle(isp, tmf.tmf_handle); ISP_UNLOCK(isp); retval = EIO; break; } isp_put_24xx_tmf(isp, &tmf, (isp24xx_tmf_t *)reqp); if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB TMF", QENTRY_LEN, reqp); ISP_SYNC_REQUEST(isp); if (msleep(resp, &isp->isp_lock, 0, "TMF", 5*hz) == EWOULDBLOCK) { isp_prt(isp, ISP_LOGERR, "%s: TMF of Chan %d timed out", __func__, chan); isp_destroy_handle(isp, tmf.tmf_handle); ISP_UNLOCK(isp); retval = EIO; break; } if (isp->isp_dblev & ISP_LOGDEBUG1) isp_print_bytes(isp, "IOCB TMF response", QENTRY_LEN, resp); isp_get_24xx_response(isp, (isp24xx_statusreq_t *)resp, &sp); if (sp.req_completion_status != 0) retval = EIO; else if (needmarker) fcp->sendmarker = 1; } else { MBSINIT(&mbs, 0, MBLOGALL, 0); if (ISP_CAP_2KLOGIN(isp) == 0) { nphdl <<= 8; } switch (fct->action) { case IPT_CLEAR_ACA: mbs.param[0] = MBOX_CLEAR_ACA; mbs.param[1] = nphdl; mbs.param[2] = fct->lun; break; case IPT_TARGET_RESET: mbs.param[0] = MBOX_TARGET_RESET; mbs.param[1] = nphdl; needmarker = 1; break; case IPT_LUN_RESET: mbs.param[0] = MBOX_LUN_RESET; mbs.param[1] = nphdl; mbs.param[2] = fct->lun; needmarker = 1; break; case IPT_CLEAR_TASK_SET: mbs.param[0] = MBOX_CLEAR_TASK_SET; mbs.param[1] = nphdl; mbs.param[2] = fct->lun; needmarker = 1; break; case IPT_ABORT_TASK_SET: mbs.param[0] = MBOX_ABORT_TASK_SET; mbs.param[1] = nphdl; mbs.param[2] = fct->lun; needmarker = 1; break; default: retval = EINVAL; break; } if (retval == 0) { if (needmarker) { FCPARAM(isp, chan)->sendmarker = 1; } retval = isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs); if (retval) { retval = EIO; } } } ISP_UNLOCK(isp); break; } default: break; } return (retval); } /* * Local Inlines */ static ISP_INLINE int isp_get_pcmd(ispsoftc_t *, union ccb *); static ISP_INLINE void isp_free_pcmd(ispsoftc_t *, union ccb *); static ISP_INLINE int isp_get_pcmd(ispsoftc_t *isp, union ccb *ccb) { ISP_PCMD(ccb) = isp->isp_osinfo.pcmd_free; if (ISP_PCMD(ccb) == NULL) { return (-1); } isp->isp_osinfo.pcmd_free = ((struct isp_pcmd *)ISP_PCMD(ccb))->next; return (0); } static ISP_INLINE void isp_free_pcmd(ispsoftc_t *isp, union ccb *ccb) { if (ISP_PCMD(ccb)) { #ifdef ISP_TARGET_MODE PISP_PCMD(ccb)->datalen = 0; #endif PISP_PCMD(ccb)->next = isp->isp_osinfo.pcmd_free; isp->isp_osinfo.pcmd_free = ISP_PCMD(ccb); ISP_PCMD(ccb) = NULL; } } /* * Put the target mode functions here, because some are inlines */ #ifdef ISP_TARGET_MODE static ISP_INLINE tstate_t *get_lun_statep(ispsoftc_t *, int, lun_id_t); static atio_private_data_t *isp_get_atpd(ispsoftc_t *, int, uint32_t); static atio_private_data_t *isp_find_atpd(ispsoftc_t *, int, uint32_t); static void isp_put_atpd(ispsoftc_t *, int, atio_private_data_t *); static inot_private_data_t *isp_get_ntpd(ispsoftc_t *, int); static inot_private_data_t *isp_find_ntpd(ispsoftc_t *, int, uint32_t, uint32_t); static void isp_put_ntpd(ispsoftc_t *, int, inot_private_data_t *); static cam_status create_lun_state(ispsoftc_t *, int, struct cam_path *, tstate_t **); static void destroy_lun_state(ispsoftc_t *, int, tstate_t *); static void isp_enable_lun(ispsoftc_t *, union ccb *); static void isp_disable_lun(ispsoftc_t *, union ccb *); static timeout_t isp_refire_putback_atio; static timeout_t isp_refire_notify_ack; static void isp_complete_ctio(union ccb *); static void isp_target_putback_atio(union ccb *); enum Start_Ctio_How { FROM_CAM, FROM_TIMER, FROM_SRR, FROM_CTIO_DONE }; static void isp_target_start_ctio(ispsoftc_t *, union ccb *, enum Start_Ctio_How); static void isp_handle_platform_atio2(ispsoftc_t *, at2_entry_t *); static void isp_handle_platform_atio7(ispsoftc_t *, at7_entry_t *); static void isp_handle_platform_ctio(ispsoftc_t *, void *); static int isp_handle_platform_target_notify_ack(ispsoftc_t *, isp_notify_t *, uint32_t rsp); static void isp_handle_platform_target_tmf(ispsoftc_t *, isp_notify_t *); static void isp_target_mark_aborted_early(ispsoftc_t *, int chan, tstate_t *, uint32_t); static ISP_INLINE tstate_t * get_lun_statep(ispsoftc_t *isp, int bus, lun_id_t lun) { tstate_t *tptr = NULL; struct tslist *lhp; if (bus < isp->isp_nchan) { ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp); SLIST_FOREACH(tptr, lhp, next) { if (tptr->ts_lun == lun) return (tptr); } } return (NULL); } static int isp_atio_restart(ispsoftc_t *isp, int bus, tstate_t *tptr) { inot_private_data_t *ntp; struct ntpdlist rq; if (STAILQ_EMPTY(&tptr->restart_queue)) return (0); STAILQ_INIT(&rq); STAILQ_CONCAT(&rq, &tptr->restart_queue); while ((ntp = STAILQ_FIRST(&rq)) != NULL) { STAILQ_REMOVE_HEAD(&rq, next); if (IS_24XX(isp)) { isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at7_entry_t *)ntp->data)->at_rxid); isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->data); } else { isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at2_entry_t *)ntp->data)->at_rxid); isp_handle_platform_atio2(isp, (at2_entry_t *) ntp->data); } isp_put_ntpd(isp, bus, ntp); if (!STAILQ_EMPTY(&tptr->restart_queue)) break; } if (!STAILQ_EMPTY(&rq)) { STAILQ_CONCAT(&rq, &tptr->restart_queue); STAILQ_CONCAT(&tptr->restart_queue, &rq); } return (!STAILQ_EMPTY(&tptr->restart_queue)); } static void isp_tmcmd_restart(ispsoftc_t *isp) { tstate_t *tptr; union ccb *ccb; struct tslist *lhp; struct isp_ccbq *waitq; int bus, i; for (bus = 0; bus < isp->isp_nchan; bus++) { for (i = 0; i < LUN_HASH_SIZE; i++) { ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp); SLIST_FOREACH(tptr, lhp, next) isp_atio_restart(isp, bus, tptr); } /* * We only need to do this once per channel. */ ISP_GET_PC_ADDR(isp, bus, waitq, waitq); ccb = (union ccb *)TAILQ_FIRST(waitq); if (ccb != NULL) { TAILQ_REMOVE(waitq, &ccb->ccb_h, sim_links.tqe); isp_target_start_ctio(isp, ccb, FROM_TIMER); } } } static atio_private_data_t * isp_get_atpd(ispsoftc_t *isp, int chan, uint32_t tag) { struct atpdlist *atfree; struct atpdlist *atused; atio_private_data_t *atp; ISP_GET_PC_ADDR(isp, chan, atfree, atfree); atp = LIST_FIRST(atfree); if (atp) { LIST_REMOVE(atp, next); atp->tag = tag; ISP_GET_PC(isp, chan, atused, atused); LIST_INSERT_HEAD(&atused[ATPDPHASH(tag)], atp, next); } return (atp); } static atio_private_data_t * isp_find_atpd(ispsoftc_t *isp, int chan, uint32_t tag) { struct atpdlist *atused; atio_private_data_t *atp; ISP_GET_PC(isp, chan, atused, atused); LIST_FOREACH(atp, &atused[ATPDPHASH(tag)], next) { if (atp->tag == tag) return (atp); } return (NULL); } static void isp_put_atpd(ispsoftc_t *isp, int chan, atio_private_data_t *atp) { struct atpdlist *atfree; if (atp->ests) { isp_put_ecmd(isp, atp->ests); } LIST_REMOVE(atp, next); memset(atp, 0, sizeof (*atp)); ISP_GET_PC_ADDR(isp, chan, atfree, atfree); LIST_INSERT_HEAD(atfree, atp, next); } static void isp_dump_atpd(ispsoftc_t *isp, int chan) { atio_private_data_t *atp, *atpool; const char *states[8] = { "Free", "ATIO", "CAM", "CTIO", "LAST_CTIO", "PDON", "?6", "7" }; ISP_GET_PC(isp, chan, atpool, atpool); for (atp = atpool; atp < &atpool[ATPDPSIZE]; atp++) { if (atp->state == ATPD_STATE_FREE) continue; isp_prt(isp, ISP_LOGALL, "Chan %d ATP [0x%x] origdlen %u bytes_xfrd %u lun %jx nphdl 0x%04x s_id 0x%06x d_id 0x%06x oxid 0x%04x state %s", chan, atp->tag, atp->orig_datalen, atp->bytes_xfered, (uintmax_t)atp->lun, atp->nphdl, atp->sid, atp->did, atp->oxid, states[atp->state & 0x7]); } } static inot_private_data_t * isp_get_ntpd(ispsoftc_t *isp, int chan) { struct ntpdlist *ntfree; inot_private_data_t *ntp; ISP_GET_PC_ADDR(isp, chan, ntfree, ntfree); ntp = STAILQ_FIRST(ntfree); if (ntp) STAILQ_REMOVE_HEAD(ntfree, next); return (ntp); } static inot_private_data_t * isp_find_ntpd(ispsoftc_t *isp, int chan, uint32_t tag_id, uint32_t seq_id) { inot_private_data_t *ntp, *ntp2; ISP_GET_PC(isp, chan, ntpool, ntp); ISP_GET_PC_ADDR(isp, chan, ntpool[ATPDPSIZE], ntp2); for (; ntp < ntp2; ntp++) { if (ntp->tag_id == tag_id && ntp->seq_id == seq_id) return (ntp); } return (NULL); } static void isp_put_ntpd(ispsoftc_t *isp, int chan, inot_private_data_t *ntp) { struct ntpdlist *ntfree; ntp->tag_id = ntp->seq_id = 0; ISP_GET_PC_ADDR(isp, chan, ntfree, ntfree); STAILQ_INSERT_HEAD(ntfree, ntp, next); } static cam_status create_lun_state(ispsoftc_t *isp, int bus, struct cam_path *path, tstate_t **rslt) { lun_id_t lun; struct tslist *lhp; tstate_t *tptr; lun = xpt_path_lun_id(path); if (lun != CAM_LUN_WILDCARD) { if (ISP_MAX_LUNS(isp) > 0 && lun >= ISP_MAX_LUNS(isp)) { return (CAM_LUN_INVALID); } } tptr = malloc(sizeof (tstate_t), M_DEVBUF, M_NOWAIT|M_ZERO); if (tptr == NULL) { return (CAM_RESRC_UNAVAIL); } tptr->ts_lun = lun; SLIST_INIT(&tptr->atios); SLIST_INIT(&tptr->inots); STAILQ_INIT(&tptr->restart_queue); ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp); SLIST_INSERT_HEAD(lhp, tptr, next); *rslt = tptr; ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, path, "created tstate\n"); return (CAM_REQ_CMP); } static void destroy_lun_state(ispsoftc_t *isp, int bus, tstate_t *tptr) { union ccb *ccb; struct tslist *lhp; inot_private_data_t *ntp; while ((ccb = (union ccb *)SLIST_FIRST(&tptr->atios)) != NULL) { SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle); ccb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(ccb); }; while ((ccb = (union ccb *)SLIST_FIRST(&tptr->inots)) != NULL) { SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle); ccb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(ccb); } while ((ntp = STAILQ_FIRST(&tptr->restart_queue)) != NULL) { isp_endcmd(isp, ntp->data, NIL_HANDLE, bus, SCSI_STATUS_BUSY, 0); STAILQ_REMOVE_HEAD(&tptr->restart_queue, next); isp_put_ntpd(isp, bus, ntp); } ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(tptr->ts_lun)], lhp); SLIST_REMOVE(lhp, tptr, tstate, next); free(tptr, M_DEVBUF); } static void isp_enable_lun(ispsoftc_t *isp, union ccb *ccb) { tstate_t *tptr; int bus; target_id_t target; lun_id_t lun; if (!IS_FC(isp) || !ISP_CAP_TMODE(isp) || !ISP_CAP_SCCFW(isp)) { xpt_print(ccb->ccb_h.path, "Target mode is not supported\n"); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; xpt_done(ccb); return; } /* * We only support either target and lun both wildcard * or target and lun both non-wildcard. */ bus = XS_CHANNEL(ccb); target = ccb->ccb_h.target_id; lun = ccb->ccb_h.target_lun; ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path, "enabling lun %jx\n", (uintmax_t)lun); if ((target == CAM_TARGET_WILDCARD) != (lun == CAM_LUN_WILDCARD)) { ccb->ccb_h.status = CAM_LUN_INVALID; xpt_done(ccb); return; } /* Create the state pointer. It should not already exist. */ tptr = get_lun_statep(isp, bus, lun); if (tptr) { ccb->ccb_h.status = CAM_LUN_ALRDY_ENA; xpt_done(ccb); return; } ccb->ccb_h.status = create_lun_state(isp, bus, ccb->ccb_h.path, &tptr); if (ccb->ccb_h.status != CAM_REQ_CMP) { xpt_done(ccb); return; } ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static void isp_disable_lun(ispsoftc_t *isp, union ccb *ccb) { tstate_t *tptr = NULL; int bus; target_id_t target; lun_id_t lun; bus = XS_CHANNEL(ccb); target = ccb->ccb_h.target_id; lun = ccb->ccb_h.target_lun; ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path, "disabling lun %jx\n", (uintmax_t)lun); if ((target == CAM_TARGET_WILDCARD) != (lun == CAM_LUN_WILDCARD)) { ccb->ccb_h.status = CAM_LUN_INVALID; xpt_done(ccb); return; } /* Find the state pointer. */ if ((tptr = get_lun_statep(isp, bus, lun)) == NULL) { ccb->ccb_h.status = CAM_PATH_INVALID; xpt_done(ccb); return; } destroy_lun_state(isp, bus, tptr); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static void isp_target_start_ctio(ispsoftc_t *isp, union ccb *ccb, enum Start_Ctio_How how) { int fctape, sendstatus, resid; fcparam *fcp; atio_private_data_t *atp; struct ccb_scsiio *cso; struct isp_ccbq *waitq; uint32_t dmaresult, handle, xfrlen, sense_length, tmp; uint8_t local[QENTRY_LEN]; isp_prt(isp, ISP_LOGTDEBUG0, "%s: ENTRY[0x%x] how %u xfrlen %u sendstatus %d sense_len %u", __func__, ccb->csio.tag_id, how, ccb->csio.dxfer_len, (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0, ((ccb->ccb_h.flags & CAM_SEND_SENSE)? ccb->csio.sense_len : 0)); ISP_GET_PC_ADDR(isp, XS_CHANNEL(ccb), waitq, waitq); switch (how) { case FROM_CAM: /* * Insert at the tail of the list, if any, waiting CTIO CCBs */ TAILQ_INSERT_TAIL(waitq, &ccb->ccb_h, sim_links.tqe); break; case FROM_TIMER: case FROM_SRR: case FROM_CTIO_DONE: TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } while ((ccb = (union ccb *) TAILQ_FIRST(waitq)) != NULL) { TAILQ_REMOVE(waitq, &ccb->ccb_h, sim_links.tqe); cso = &ccb->csio; xfrlen = cso->dxfer_len; if (xfrlen == 0) { if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) { ISP_PATH_PRT(isp, ISP_LOGERR, ccb->ccb_h.path, "a data transfer length of zero but no status to send is wrong\n"); ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); continue; } } atp = isp_find_atpd(isp, XS_CHANNEL(ccb), cso->tag_id); if (atp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: [0x%x] cannot find private data adjunct in %s", __func__, cso->tag_id, __func__); isp_dump_atpd(isp, XS_CHANNEL(ccb)); ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); continue; } /* * Is this command a dead duck? */ if (atp->dead) { isp_prt(isp, ISP_LOGERR, "%s: [0x%x] not sending a CTIO for a dead command", __func__, cso->tag_id); ccb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(ccb); continue; } /* * Check to make sure we're still in target mode. */ fcp = FCPARAM(isp, XS_CHANNEL(ccb)); if ((fcp->role & ISP_ROLE_TARGET) == 0) { isp_prt(isp, ISP_LOGERR, "%s: [0x%x] stopping sending a CTIO because we're no longer in target mode", __func__, cso->tag_id); ccb->ccb_h.status = CAM_PROVIDE_FAIL; xpt_done(ccb); continue; } /* * We're only handling ATPD_CCB_OUTSTANDING outstanding CCB at a time (one of which * could be split into two CTIOs to split data and status). */ if (atp->ctcnt >= ATPD_CCB_OUTSTANDING) { isp_prt(isp, ISP_LOGTINFO, "[0x%x] handling only %d CCBs at a time (flags for this ccb: 0x%x)", cso->tag_id, ATPD_CCB_OUTSTANDING, ccb->ccb_h.flags); TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } /* * Does the initiator expect FC-Tape style responses? */ if ((atp->word3 & PRLI_WD3_RETRY) && fcp->fctape_enabled) { fctape = 1; } else { fctape = 0; } /* * If we already did the data xfer portion of a CTIO that sends data * and status, don't do it again and do the status portion now. */ if (atp->sendst) { isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] now sending synthesized status orig_dl=%u xfered=%u bit=%u", cso->tag_id, atp->orig_datalen, atp->bytes_xfered, atp->bytes_in_transit); xfrlen = 0; /* we already did the data transfer */ atp->sendst = 0; } if (ccb->ccb_h.flags & CAM_SEND_STATUS) { sendstatus = 1; } else { sendstatus = 0; } if (ccb->ccb_h.flags & CAM_SEND_SENSE) { KASSERT((sendstatus != 0), ("how can you have CAM_SEND_SENSE w/o CAM_SEND_STATUS?")); /* * Sense length is not the entire sense data structure size. Periph * drivers don't seem to be setting sense_len to reflect the actual * size. We'll peek inside to get the right amount. */ sense_length = cso->sense_len; /* * This 'cannot' happen */ if (sense_length > (XCMD_SIZE - MIN_FCP_RESPONSE_SIZE)) { sense_length = XCMD_SIZE - MIN_FCP_RESPONSE_SIZE; } } else { sense_length = 0; } memset(local, 0, QENTRY_LEN); /* * Check for overflow */ tmp = atp->bytes_xfered + atp->bytes_in_transit; if (xfrlen > 0 && tmp > atp->orig_datalen) { isp_prt(isp, ISP_LOGERR, "%s: [0x%x] data overflow by %u bytes", __func__, cso->tag_id, tmp + xfrlen - atp->orig_datalen); ccb->ccb_h.status = CAM_DATA_RUN_ERR; xpt_done(ccb); continue; } if (xfrlen > atp->orig_datalen - tmp) { xfrlen = atp->orig_datalen - tmp; if (xfrlen == 0 && !sendstatus) { cso->resid = cso->dxfer_len; ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); continue; } } if (IS_24XX(isp)) { ct7_entry_t *cto = (ct7_entry_t *) local; cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7; cto->ct_header.rqs_entry_count = 1; cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM; ATPD_SET_SEQNO(cto, atp); cto->ct_nphdl = atp->nphdl; cto->ct_rxid = atp->tag; cto->ct_iid_lo = atp->sid; cto->ct_iid_hi = atp->sid >> 16; cto->ct_oxid = atp->oxid; cto->ct_vpidx = ISP_GET_VPIDX(isp, XS_CHANNEL(ccb)); cto->ct_timeout = XS_TIME(ccb); cto->ct_flags = atp->tattr << CT7_TASK_ATTR_SHIFT; /* * Mode 1, status, no data. Only possible when we are sending status, have * no data to transfer, and any sense data can fit into a ct7_entry_t. * * Mode 2, status, no data. We have to use this in the case that * the sense data won't fit into a ct7_entry_t. * */ if (sendstatus && xfrlen == 0) { cto->ct_flags |= CT7_SENDSTATUS | CT7_NO_DATA; resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit; if (sense_length <= MAXRESPLEN_24XX) { cto->ct_flags |= CT7_FLAG_MODE1; cto->ct_scsi_status = cso->scsi_status; if (resid < 0) { cto->ct_resid = -resid; cto->ct_scsi_status |= (FCP_RESID_OVERFLOW << 8); } else if (resid > 0) { cto->ct_resid = resid; cto->ct_scsi_status |= (FCP_RESID_UNDERFLOW << 8); } if (fctape) { cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF; } if (sense_length) { cto->ct_scsi_status |= (FCP_SNSLEN_VALID << 8); cto->rsp.m1.ct_resplen = cto->ct_senselen = sense_length; memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, sense_length); } } else { bus_addr_t addr; char buf[XCMD_SIZE]; fcp_rsp_iu_t *rp; if (atp->ests == NULL) { atp->ests = isp_get_ecmd(isp); if (atp->ests == NULL) { TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } } memset(buf, 0, sizeof (buf)); rp = (fcp_rsp_iu_t *)buf; if (fctape) { cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF; rp->fcp_rsp_bits |= FCP_CONF_REQ; } cto->ct_flags |= CT7_FLAG_MODE2; rp->fcp_rsp_scsi_status = cso->scsi_status; if (resid < 0) { rp->fcp_rsp_resid = -resid; rp->fcp_rsp_bits |= FCP_RESID_OVERFLOW; } else if (resid > 0) { rp->fcp_rsp_resid = resid; rp->fcp_rsp_bits |= FCP_RESID_UNDERFLOW; } if (sense_length) { rp->fcp_rsp_snslen = sense_length; cto->ct_senselen = sense_length; rp->fcp_rsp_bits |= FCP_SNSLEN_VALID; isp_put_fcp_rsp_iu(isp, rp, atp->ests); memcpy(((fcp_rsp_iu_t *)atp->ests)->fcp_rsp_extra, &cso->sense_data, sense_length); } else { isp_put_fcp_rsp_iu(isp, rp, atp->ests); } if (isp->isp_dblev & ISP_LOGTDEBUG1) { isp_print_bytes(isp, "FCP Response Frame After Swizzling", MIN_FCP_RESPONSE_SIZE + sense_length, atp->ests); } addr = isp->isp_osinfo.ecmd_dma; addr += ((((isp_ecmd_t *)atp->ests) - isp->isp_osinfo.ecmd_base) * XCMD_SIZE); isp_prt(isp, ISP_LOGTDEBUG0, "%s: ests base %p vaddr %p ecmd_dma %jx addr %jx len %u", __func__, isp->isp_osinfo.ecmd_base, atp->ests, (uintmax_t) isp->isp_osinfo.ecmd_dma, (uintmax_t)addr, MIN_FCP_RESPONSE_SIZE + sense_length); cto->rsp.m2.ct_datalen = MIN_FCP_RESPONSE_SIZE + sense_length; cto->rsp.m2.ct_fcp_rsp_iudata.ds_base = DMA_LO32(addr); cto->rsp.m2.ct_fcp_rsp_iudata.ds_basehi = DMA_HI32(addr); cto->rsp.m2.ct_fcp_rsp_iudata.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length; } if (sense_length) { isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d slen %u sense: %x %x/%x/%x", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid, sense_length, cso->sense_data.error_code, cso->sense_data.sense_buf[1], cso->sense_data.sense_buf[11], cso->sense_data.sense_buf[12]); } else { isp_prt(isp, ISP_LOGDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid); } atp->state = ATPD_STATE_LAST_CTIO; } /* * Mode 0 data transfers, *possibly* with status. */ if (xfrlen != 0) { cto->ct_flags |= CT7_FLAG_MODE0; if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { cto->ct_flags |= CT7_DATA_IN; } else { cto->ct_flags |= CT7_DATA_OUT; } cto->rsp.m0.reloff = atp->bytes_xfered + atp->bytes_in_transit; cto->rsp.m0.ct_xfrlen = xfrlen; #ifdef DEBUG if (ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame && xfrlen > ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame) { isp_prt(isp, ISP_LOGWARN, "%s: truncating data frame with xfrlen %d to %d", __func__, xfrlen, xfrlen - (xfrlen >> 2)); ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame = 0; cto->rsp.m0.ct_xfrlen -= xfrlen >> 2; } #endif if (sendstatus) { resid = atp->orig_datalen - atp->bytes_xfered - xfrlen; if (cso->scsi_status == SCSI_STATUS_OK && resid == 0 /* && fctape == 0 */) { cto->ct_flags |= CT7_SENDSTATUS; atp->state = ATPD_STATE_LAST_CTIO; if (fctape) { cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF; } } else { atp->sendst = 1; /* send status later */ cto->ct_header.rqs_seqno &= ~ATPD_SEQ_NOTIFY_CAM; atp->state = ATPD_STATE_CTIO; } } else { atp->state = ATPD_STATE_CTIO; } isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x xfrlen=%u off=%u", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, xfrlen, atp->bytes_xfered); } } else { ct2_entry_t *cto = (ct2_entry_t *) local; cto->ct_header.rqs_entry_type = RQSTYPE_CTIO2; cto->ct_header.rqs_entry_count = 1; cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM; ATPD_SET_SEQNO(cto, atp); if (ISP_CAP_2KLOGIN(isp)) { ((ct2e_entry_t *)cto)->ct_iid = atp->nphdl; } else { cto->ct_iid = atp->nphdl; if (ISP_CAP_SCCFW(isp) == 0) { cto->ct_lun = ccb->ccb_h.target_lun; } } cto->ct_timeout = XS_TIME(ccb); cto->ct_rxid = cso->tag_id; /* * Mode 1, status, no data. Only possible when we are sending status, have * no data to transfer, and the sense length can fit in the ct7_entry. * * Mode 2, status, no data. We have to use this in the case the response * length won't fit into a ct2_entry_t. * * We'll fill out this structure with information as if this were a * Mode 1. The hardware layer will create the Mode 2 FCP RSP IU as * needed based upon this. */ if (sendstatus && xfrlen == 0) { cto->ct_flags |= CT2_SENDSTATUS | CT2_NO_DATA; resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit; if (sense_length <= MAXRESPLEN) { if (resid < 0) { cto->ct_resid = -resid; } else if (resid > 0) { cto->ct_resid = resid; } cto->ct_flags |= CT2_FLAG_MODE1; cto->rsp.m1.ct_scsi_status = cso->scsi_status; if (resid < 0) { cto->rsp.m1.ct_scsi_status |= CT2_DATA_OVER; } else if (resid > 0) { cto->rsp.m1.ct_scsi_status |= CT2_DATA_UNDER; } if (fctape) { cto->ct_flags |= CT2_CONFIRM; } if (sense_length) { cto->rsp.m1.ct_scsi_status |= CT2_SNSLEN_VALID; cto->rsp.m1.ct_resplen = cto->rsp.m1.ct_senselen = sense_length; memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, sense_length); } } else { bus_addr_t addr; char buf[XCMD_SIZE]; fcp_rsp_iu_t *rp; if (atp->ests == NULL) { atp->ests = isp_get_ecmd(isp); if (atp->ests == NULL) { TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } } memset(buf, 0, sizeof (buf)); rp = (fcp_rsp_iu_t *)buf; if (fctape) { cto->ct_flags |= CT2_CONFIRM; rp->fcp_rsp_bits |= FCP_CONF_REQ; } cto->ct_flags |= CT2_FLAG_MODE2; rp->fcp_rsp_scsi_status = cso->scsi_status; if (resid < 0) { rp->fcp_rsp_resid = -resid; rp->fcp_rsp_bits |= FCP_RESID_OVERFLOW; } else if (resid > 0) { rp->fcp_rsp_resid = resid; rp->fcp_rsp_bits |= FCP_RESID_UNDERFLOW; } if (sense_length) { rp->fcp_rsp_snslen = sense_length; rp->fcp_rsp_bits |= FCP_SNSLEN_VALID; isp_put_fcp_rsp_iu(isp, rp, atp->ests); memcpy(((fcp_rsp_iu_t *)atp->ests)->fcp_rsp_extra, &cso->sense_data, sense_length); } else { isp_put_fcp_rsp_iu(isp, rp, atp->ests); } if (isp->isp_dblev & ISP_LOGTDEBUG1) { isp_print_bytes(isp, "FCP Response Frame After Swizzling", MIN_FCP_RESPONSE_SIZE + sense_length, atp->ests); } addr = isp->isp_osinfo.ecmd_dma; addr += ((((isp_ecmd_t *)atp->ests) - isp->isp_osinfo.ecmd_base) * XCMD_SIZE); isp_prt(isp, ISP_LOGTDEBUG0, "%s: ests base %p vaddr %p ecmd_dma %jx addr %jx len %u", __func__, isp->isp_osinfo.ecmd_base, atp->ests, (uintmax_t) isp->isp_osinfo.ecmd_dma, (uintmax_t)addr, MIN_FCP_RESPONSE_SIZE + sense_length); cto->rsp.m2.ct_datalen = MIN_FCP_RESPONSE_SIZE + sense_length; cto->rsp.m2.u.ct_fcp_rsp_iudata_32.ds_base = DMA_LO32(addr); cto->rsp.m2.u.ct_fcp_rsp_iudata_32.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length; } if (sense_length) { isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d sense: %x %x/%x/%x", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid, cso->sense_data.error_code, cso->sense_data.sense_buf[1], cso->sense_data.sense_buf[11], cso->sense_data.sense_buf[12]); } else { isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid); } atp->state = ATPD_STATE_LAST_CTIO; } if (xfrlen != 0) { cto->ct_flags |= CT2_FLAG_MODE0; if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { cto->ct_flags |= CT2_DATA_IN; } else { cto->ct_flags |= CT2_DATA_OUT; } cto->ct_reloff = atp->bytes_xfered + atp->bytes_in_transit; cto->rsp.m0.ct_xfrlen = xfrlen; if (sendstatus) { resid = atp->orig_datalen - atp->bytes_xfered - xfrlen; if (cso->scsi_status == SCSI_STATUS_OK && resid == 0 /*&& fctape == 0*/) { cto->ct_flags |= CT2_SENDSTATUS; atp->state = ATPD_STATE_LAST_CTIO; if (fctape) { cto->ct_flags |= CT2_CONFIRM; } } else { atp->sendst = 1; /* send status later */ cto->ct_header.rqs_seqno &= ~ATPD_SEQ_NOTIFY_CAM; atp->state = ATPD_STATE_CTIO; } } else { atp->state = ATPD_STATE_CTIO; } } isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[%x] seq %u nc %d CDB0=%x scsi status %x flags %x resid %d xfrlen %u offset %u", __func__, cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid, cso->dxfer_len, atp->bytes_xfered); } if (isp_get_pcmd(isp, ccb)) { ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "out of PCMDs\n"); TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } handle = isp_allocate_handle(isp, ccb, ISP_HANDLE_TARGET); if (handle == 0) { ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "No XFLIST pointers for %s\n", __func__); TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); isp_free_pcmd(isp, ccb); break; } atp->bytes_in_transit += xfrlen; PISP_PCMD(ccb)->datalen = xfrlen; /* * Call the dma setup routines for this entry (and any subsequent * CTIOs) if there's data to move, and then tell the f/w it's got * new things to play with. As with isp_start's usage of DMA setup, * any swizzling is done in the machine dependent layer. Because * of this, we put the request onto the queue area first in native * format. */ if (IS_24XX(isp)) { ct7_entry_t *cto = (ct7_entry_t *) local; cto->ct_syshandle = handle; } else { ct2_entry_t *cto = (ct2_entry_t *) local; cto->ct_syshandle = handle; } dmaresult = ISP_DMASETUP(isp, cso, (ispreq_t *) local); if (dmaresult != CMD_QUEUED) { isp_destroy_handle(isp, handle); isp_free_pcmd(isp, ccb); if (dmaresult == CMD_EAGAIN) { TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe); break; } ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); continue; } ccb->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED; if (xfrlen) { ccb->ccb_h.spriv_field0 = atp->bytes_xfered; } else { ccb->ccb_h.spriv_field0 = ~0; } atp->ctcnt++; atp->seqno++; } } static void isp_refire_putback_atio(void *arg) { union ccb *ccb = arg; ISP_ASSERT_LOCKED((ispsoftc_t *)XS_ISP(ccb)); isp_target_putback_atio(ccb); } static void isp_refire_notify_ack(void *arg) { isp_tna_t *tp = arg; ispsoftc_t *isp = tp->isp; ISP_ASSERT_LOCKED(isp); if (isp_notify_ack(isp, tp->not)) { callout_schedule(&tp->timer, 5); } else { free(tp, M_DEVBUF); } } static void isp_target_putback_atio(union ccb *ccb) { ispsoftc_t *isp = XS_ISP(ccb); struct ccb_scsiio *cso = &ccb->csio; at2_entry_t local, *at = &local; ISP_MEMZERO(at, sizeof (at2_entry_t)); at->at_header.rqs_entry_type = RQSTYPE_ATIO2; at->at_header.rqs_entry_count = 1; if (ISP_CAP_SCCFW(isp)) { at->at_scclun = (uint16_t) ccb->ccb_h.target_lun; } else { at->at_lun = (uint8_t) ccb->ccb_h.target_lun; } at->at_status = CT_OK; at->at_rxid = cso->tag_id; at->at_iid = cso->init_id; if (isp_target_put_entry(isp, at)) { callout_reset(&PISP_PCMD(ccb)->wdog, 10, isp_refire_putback_atio, ccb); } else isp_complete_ctio(ccb); } static void isp_complete_ctio(union ccb *ccb) { if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG) { ccb->ccb_h.status &= ~CAM_SIM_QUEUED; xpt_done(ccb); } } static void isp_handle_platform_atio2(ispsoftc_t *isp, at2_entry_t *aep) { fcparam *fcp; lun_id_t lun; fcportdb_t *lp; tstate_t *tptr; struct ccb_accept_tio *atiop; uint16_t nphdl; atio_private_data_t *atp; inot_private_data_t *ntp; /* * The firmware status (except for the QLTM_SVALID bit) * indicates why this ATIO was sent to us. * * If QLTM_SVALID is set, the firmware has recommended Sense Data. */ if ((aep->at_status & ~QLTM_SVALID) != AT_CDB) { isp_prt(isp, ISP_LOGWARN, "bogus atio (0x%x) leaked to platform", aep->at_status); isp_endcmd(isp, aep, NIL_HANDLE, 0, SCSI_STATUS_BUSY, 0); return; } fcp = FCPARAM(isp, 0); if (ISP_CAP_SCCFW(isp)) { lun = aep->at_scclun; } else { lun = aep->at_lun; } if (ISP_CAP_2KLOGIN(isp)) { nphdl = ((at2e_entry_t *)aep)->at_iid; } else { nphdl = aep->at_iid; } tptr = get_lun_statep(isp, 0, lun); if (tptr == NULL) { tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD); if (tptr == NULL) { isp_prt(isp, ISP_LOGWARN, "%s: [0x%x] no state pointer for lun %jx or wildcard", __func__, aep->at_rxid, (uintmax_t)lun); if (lun == 0) { isp_endcmd(isp, aep, nphdl, 0, SCSI_STATUS_BUSY, 0); } else { isp_endcmd(isp, aep, nphdl, 0, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0); } return; } } /* * Start any commands pending resources first. */ if (isp_atio_restart(isp, 0, tptr)) goto noresrc; atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios); if (atiop == NULL) { goto noresrc; } atp = isp_get_atpd(isp, 0, aep->at_rxid); if (atp == NULL) { goto noresrc; } atp->state = ATPD_STATE_ATIO; SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, atiop->ccb_h.path, "Take FREE ATIO\n"); atiop->ccb_h.target_id = ISP_MAX_TARGETS(isp); atiop->ccb_h.target_lun = lun; /* * We don't get 'suggested' sense data as we do with SCSI cards. */ atiop->sense_len = 0; /* * If we're not in the port database, add ourselves. */ if (IS_2100(isp)) atiop->init_id = nphdl; else { if (isp_find_pdb_by_handle(isp, 0, nphdl, &lp)) { atiop->init_id = FC_PORTDB_TGT(isp, 0, lp); } else { isp_prt(isp, ISP_LOGTINFO, "%s: port %x isn't in PDB", __func__, nphdl); isp_dump_portdb(isp, 0); isp_endcmd(isp, aep, NIL_HANDLE, 0, ECMD_TERMINATE, 0); return; } } atiop->cdb_len = ATIO2_CDBLEN; ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cdb, ATIO2_CDBLEN); atiop->ccb_h.status = CAM_CDB_RECVD; atiop->tag_id = atp->tag; switch (aep->at_taskflags & ATIO2_TC_ATTR_MASK) { case ATIO2_TC_ATTR_SIMPLEQ: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_SIMPLE_Q_TAG; break; case ATIO2_TC_ATTR_HEADOFQ: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_HEAD_OF_Q_TAG; break; case ATIO2_TC_ATTR_ORDERED: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_ORDERED_Q_TAG; break; case ATIO2_TC_ATTR_ACAQ: /* ?? */ case ATIO2_TC_ATTR_UNTAGGED: default: atiop->tag_action = 0; break; } atp->orig_datalen = aep->at_datalen; atp->bytes_xfered = 0; atp->lun = lun; atp->nphdl = nphdl; atp->sid = PORT_ANY; atp->oxid = aep->at_oxid; atp->cdb0 = aep->at_cdb[0]; atp->tattr = aep->at_taskflags & ATIO2_TC_ATTR_MASK; atp->state = ATPD_STATE_CAM; xpt_done((union ccb *)atiop); isp_prt(isp, ISP_LOGTDEBUG0, "ATIO2[0x%x] CDB=0x%x lun %jx datalen %u", aep->at_rxid, atp->cdb0, (uintmax_t)lun, atp->orig_datalen); return; noresrc: ntp = isp_get_ntpd(isp, 0); if (ntp == NULL) { isp_endcmd(isp, aep, nphdl, 0, SCSI_STATUS_BUSY, 0); return; } memcpy(ntp->data, aep, QENTRY_LEN); STAILQ_INSERT_TAIL(&tptr->restart_queue, ntp, next); } static void isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep) { int cdbxlen; lun_id_t lun; uint16_t chan, nphdl = NIL_HANDLE; uint32_t did, sid; fcportdb_t *lp; tstate_t *tptr; struct ccb_accept_tio *atiop; atio_private_data_t *atp = NULL; atio_private_data_t *oatp; inot_private_data_t *ntp; did = (aep->at_hdr.d_id[0] << 16) | (aep->at_hdr.d_id[1] << 8) | aep->at_hdr.d_id[2]; sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2]; lun = CAM_EXTLUN_BYTE_SWIZZLE(be64dec(aep->at_cmnd.fcp_cmnd_lun)); if (ISP_CAP_MULTI_ID(isp) && isp->isp_nchan > 1) { /* Channel has to be derived from D_ID */ isp_find_chan_by_did(isp, did, &chan); if (chan == ISP_NOCHAN) { isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel", __func__, aep->at_rxid, did); isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN, ECMD_TERMINATE, 0); return; } } else { chan = 0; } /* * Find the PDB entry for this initiator */ if (isp_find_pdb_by_portid(isp, chan, sid, &lp) == 0) { /* * If we're not in the port database terminate the exchange. */ isp_prt(isp, ISP_LOGTINFO, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x wasn't in PDB already", __func__, aep->at_rxid, did, chan, sid); isp_dump_portdb(isp, chan); isp_endcmd(isp, aep, NIL_HANDLE, chan, ECMD_TERMINATE, 0); return; } nphdl = lp->handle; /* * Get the tstate pointer */ tptr = get_lun_statep(isp, chan, lun); if (tptr == NULL) { tptr = get_lun_statep(isp, chan, CAM_LUN_WILDCARD); if (tptr == NULL) { isp_prt(isp, ISP_LOGWARN, "%s: [0x%x] no state pointer for lun %jx or wildcard", __func__, aep->at_rxid, (uintmax_t)lun); if (lun == 0) { isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0); } else { isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0); } return; } } /* * Start any commands pending resources first. */ if (isp_atio_restart(isp, chan, tptr)) goto noresrc; /* * If the f/w is out of resources, just send a BUSY status back. */ if (aep->at_rxid == AT7_NORESRC_RXID) { isp_endcmd(isp, aep, nphdl, chan, SCSI_BUSY, 0); return; } /* * If we're out of resources, just send a BUSY status back. */ atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios); if (atiop == NULL) { isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atios", aep->at_rxid); goto noresrc; } oatp = isp_find_atpd(isp, chan, aep->at_rxid); if (oatp) { isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] tag wraparound in isp_handle_platforms_atio7 (N-Port Handle 0x%04x S_ID 0x%04x OX_ID 0x%04x) oatp state %d", aep->at_rxid, nphdl, sid, aep->at_hdr.ox_id, oatp->state); /* * It's not a "no resource" condition- but we can treat it like one */ goto noresrc; } atp = isp_get_atpd(isp, chan, aep->at_rxid); if (atp == NULL) { isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid); goto noresrc; } atp->word3 = lp->prli_word3; atp->state = ATPD_STATE_ATIO; SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, atiop->ccb_h.path, "Take FREE ATIO\n"); atiop->init_id = FC_PORTDB_TGT(isp, chan, lp); atiop->ccb_h.target_id = ISP_MAX_TARGETS(isp); atiop->ccb_h.target_lun = lun; atiop->sense_len = 0; cdbxlen = aep->at_cmnd.fcp_cmnd_alen_datadir >> FCP_CMND_ADDTL_CDBLEN_SHIFT; if (cdbxlen) { isp_prt(isp, ISP_LOGWARN, "additional CDBLEN ignored"); } cdbxlen = sizeof (aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb); ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb, cdbxlen); atiop->cdb_len = cdbxlen; atiop->ccb_h.status = CAM_CDB_RECVD; atiop->tag_id = atp->tag; switch (aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK) { case FCP_CMND_TASK_ATTR_SIMPLE: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_SIMPLE_Q_TAG; break; case FCP_CMND_TASK_ATTR_HEAD: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_HEAD_OF_Q_TAG; break; case FCP_CMND_TASK_ATTR_ORDERED: atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID; atiop->tag_action = MSG_ORDERED_Q_TAG; break; default: /* FALLTHROUGH */ case FCP_CMND_TASK_ATTR_ACA: case FCP_CMND_TASK_ATTR_UNTAGGED: atiop->tag_action = 0; break; } atp->orig_datalen = aep->at_cmnd.cdb_dl.sf.fcp_cmnd_dl; atp->bytes_xfered = 0; atp->lun = lun; atp->nphdl = nphdl; atp->sid = sid; atp->did = did; atp->oxid = aep->at_hdr.ox_id; atp->rxid = aep->at_hdr.rx_id; atp->cdb0 = atiop->cdb_io.cdb_bytes[0]; atp->tattr = aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK; atp->state = ATPD_STATE_CAM; isp_prt(isp, ISP_LOGTDEBUG0, "ATIO7[0x%x] CDB=0x%x lun %jx datalen %u", aep->at_rxid, atp->cdb0, (uintmax_t)lun, atp->orig_datalen); xpt_done((union ccb *)atiop); return; noresrc: if (atp) isp_put_atpd(isp, chan, atp); ntp = isp_get_ntpd(isp, chan); if (ntp == NULL) { isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0); return; } memcpy(ntp->data, aep, QENTRY_LEN); STAILQ_INSERT_TAIL(&tptr->restart_queue, ntp, next); } /* * Handle starting an SRR (sequence retransmit request) * We get here when we've gotten the immediate notify * and the return of all outstanding CTIOs for this * transaction. */ static void isp_handle_srr_start(ispsoftc_t *isp, atio_private_data_t *atp) { in_fcentry_24xx_t *inot; uint32_t srr_off, ccb_off, ccb_len, ccb_end; union ccb *ccb; inot = (in_fcentry_24xx_t *)atp->srr; srr_off = inot->in_srr_reloff_lo | (inot->in_srr_reloff_hi << 16); ccb = atp->srr_ccb; atp->srr_ccb = NULL; atp->nsrr++; if (ccb == NULL) { isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] null ccb", atp->tag); goto fail; } ccb_off = ccb->ccb_h.spriv_field0; ccb_len = ccb->csio.dxfer_len; ccb_end = (ccb_off == ~0)? ~0 : ccb_off + ccb_len; switch (inot->in_srr_iu) { case R_CTL_INFO_SOLICITED_DATA: /* * We have to restart a FCP_DATA data out transaction */ atp->sendst = 0; atp->bytes_xfered = srr_off; if (ccb_len == 0) { isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x but current CCB doesn't transfer data", atp->tag, srr_off); goto mdp; } if (srr_off < ccb_off || ccb_off > srr_off + ccb_len) { isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x not covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end); goto mdp; } isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end); break; case R_CTL_INFO_COMMAND_STATUS: isp_prt(isp, ISP_LOGTINFO, "SRR[0x%x] Got an FCP RSP SRR- resending status", atp->tag); atp->sendst = 1; /* * We have to restart a FCP_RSP IU transaction */ break; case R_CTL_INFO_DATA_DESCRIPTOR: /* * We have to restart an FCP DATA in transaction */ isp_prt(isp, ISP_LOGWARN, "Got an FCP DATA IN SRR- dropping"); goto fail; default: isp_prt(isp, ISP_LOGWARN, "Got an unknown information (%x) SRR- dropping", inot->in_srr_iu); goto fail; } /* * We can't do anything until this is acked, so we might as well start it now. * We aren't going to do the usual asynchronous ack issue because we need * to make sure this gets on the wire first. */ if (isp_notify_ack(isp, inot)) { isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose"); goto fail; } isp_target_start_ctio(isp, ccb, FROM_SRR); return; fail: inot->in_reserved = 1; isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot); ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_REQ_CMP_ERR; isp_complete_ctio(ccb); return; mdp: if (isp_notify_ack(isp, inot)) { isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose"); goto fail; } ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status = CAM_MESSAGE_RECV; /* * This is not a strict interpretation of MDP, but it's close */ ccb->csio.msg_ptr = &ccb->csio.sense_data.sense_buf[SSD_FULL_SIZE - 16]; ccb->csio.msg_len = 7; ccb->csio.msg_ptr[0] = MSG_EXTENDED; ccb->csio.msg_ptr[1] = 5; ccb->csio.msg_ptr[2] = 0; /* modify data pointer */ ccb->csio.msg_ptr[3] = srr_off >> 24; ccb->csio.msg_ptr[4] = srr_off >> 16; ccb->csio.msg_ptr[5] = srr_off >> 8; ccb->csio.msg_ptr[6] = srr_off; isp_complete_ctio(ccb); } static void isp_handle_platform_srr(ispsoftc_t *isp, isp_notify_t *notify) { in_fcentry_24xx_t *inot = notify->nt_lreserved; atio_private_data_t *atp; uint32_t tag = notify->nt_tagval & 0xffffffff; atp = isp_find_atpd(isp, notify->nt_channel, tag); if (atp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x in SRR Notify", __func__, tag); isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot); return; } atp->srr_notify_rcvd = 1; memcpy(atp->srr, inot, sizeof (atp->srr)); isp_prt(isp, ISP_LOGTINFO, "SRR[0x%x] flags 0x%x srr_iu %x reloff 0x%x", inot->in_rxid, inot->in_flags, inot->in_srr_iu, ((uint32_t)inot->in_srr_reloff_hi << 16) | inot->in_srr_reloff_lo); if (atp->srr_ccb) isp_handle_srr_start(isp, atp); } static void isp_handle_platform_ctio(ispsoftc_t *isp, void *arg) { union ccb *ccb; int sentstatus = 0, ok = 0, notify_cam = 0, failure = 0; atio_private_data_t *atp = NULL; int bus; uint32_t handle, data_requested, resid; handle = ((ct2_entry_t *)arg)->ct_syshandle; ccb = isp_find_xs(isp, handle); if (ccb == NULL) { isp_print_bytes(isp, "null ccb in isp_handle_platform_ctio", QENTRY_LEN, arg); return; } isp_destroy_handle(isp, handle); resid = data_requested = PISP_PCMD(ccb)->datalen; isp_free_pcmd(isp, ccb); bus = XS_CHANNEL(ccb); if (IS_24XX(isp)) { atp = isp_find_atpd(isp, bus, ((ct7_entry_t *)arg)->ct_rxid); } else { atp = isp_find_atpd(isp, bus, ((ct2_entry_t *)arg)->ct_rxid); } if (atp == NULL) { /* * XXX: isp_clear_commands() generates fake CTIO with zero * ct_rxid value, filling only ct_syshandle. Workaround * that using tag_id from the CCB, pointed by ct_syshandle. */ atp = isp_find_atpd(isp, bus, ccb->csio.tag_id); } if (atp == NULL) { isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x after I/O", __func__, ccb->csio.tag_id); return; } KASSERT((atp->ctcnt > 0), ("ctio count not greater than zero")); atp->bytes_in_transit -= data_requested; atp->ctcnt -= 1; ccb->ccb_h.status &= ~CAM_STATUS_MASK; if (IS_24XX(isp)) { ct7_entry_t *ct = arg; if (ct->ct_nphdl == CT7_SRR) { atp->srr_ccb = ccb; if (atp->srr_notify_rcvd) isp_handle_srr_start(isp, atp); return; } if (ct->ct_nphdl == CT_HBA_RESET) { sentstatus = (ccb->ccb_h.flags & CAM_SEND_STATUS) && (atp->sendst == 0); failure = CAM_UNREC_HBA_ERROR; } else { sentstatus = ct->ct_flags & CT7_SENDSTATUS; ok = (ct->ct_nphdl == CT7_OK); notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0; if ((ct->ct_flags & CT7_DATAMASK) != CT7_NO_DATA) resid = ct->ct_resid; } isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO7[%x] seq %u nc %d sts 0x%x flg 0x%x sns %d resid %d %s", __func__, ct->ct_rxid, ATPD_GET_SEQNO(ct), notify_cam, ct->ct_nphdl, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID"); } else { ct2_entry_t *ct = arg; if (ct->ct_status == CT_SRR) { atp->srr_ccb = ccb; if (atp->srr_notify_rcvd) isp_handle_srr_start(isp, atp); isp_target_putback_atio(ccb); return; } if (ct->ct_status == CT_HBA_RESET) { sentstatus = (ccb->ccb_h.flags & CAM_SEND_STATUS) && (atp->sendst == 0); failure = CAM_UNREC_HBA_ERROR; } else { sentstatus = ct->ct_flags & CT2_SENDSTATUS; ok = (ct->ct_status & ~QLTM_SVALID) == CT_OK; notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0; if ((ct->ct_flags & CT2_DATAMASK) != CT2_NO_DATA) resid = ct->ct_resid; } isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO2[%x] seq %u nc %d sts 0x%x flg 0x%x sns %d resid %d %s", __func__, ct->ct_rxid, ATPD_GET_SEQNO(ct), notify_cam, ct->ct_status, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID"); } if (ok) { if (data_requested > 0) { atp->bytes_xfered += data_requested - resid; ccb->csio.resid = ccb->csio.dxfer_len - (data_requested - resid); } if (sentstatus && (ccb->ccb_h.flags & CAM_SEND_SENSE)) ccb->ccb_h.status |= CAM_SENT_SENSE; ccb->ccb_h.status |= CAM_REQ_CMP; } else { notify_cam = 1; if (failure == CAM_UNREC_HBA_ERROR) ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR; else ccb->ccb_h.status |= CAM_REQ_CMP_ERR; } atp->state = ATPD_STATE_PDON; /* * We never *not* notify CAM when there has been any error (ok == 0), * so we never need to do an ATIO putback if we're not notifying CAM. */ isp_prt(isp, ISP_LOGTDEBUG0, "%s CTIO[0x%x] done (ok=%d nc=%d nowsendstatus=%d ccb ss=%d)", (sentstatus)? " FINAL " : "MIDTERM ", atp->tag, ok, notify_cam, atp->sendst, (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0); if (notify_cam == 0) { if (atp->sendst) { isp_target_start_ctio(isp, ccb, FROM_CTIO_DONE); } return; } /* * We are done with this ATIO if we successfully sent status. * In all other cases expect either another CTIO or XPT_ABORT. */ if (ok && sentstatus) isp_put_atpd(isp, bus, atp); /* * We're telling CAM we're done with this CTIO transaction. * * 24XX cards never need an ATIO put back. * * Other cards need one put back only on error. * In the latter case, a timeout will re-fire * and try again in case we didn't have * queue resources to do so at first. In any case, * once the putback is done we do the completion * call. */ if (ok || IS_24XX(isp)) { isp_complete_ctio(ccb); } else { isp_target_putback_atio(ccb); } } static int isp_handle_platform_target_notify_ack(ispsoftc_t *isp, isp_notify_t *mp, uint32_t rsp) { if (isp->isp_state != ISP_RUNSTATE) { isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) acked- h/w not ready (dropping)", mp->nt_ncode, mp->nt_lreserved != NULL); return (0); } /* * This case is for a Task Management Function, which shows up as an ATIO7 entry. */ if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ATIO) { ct7_entry_t local, *cto = &local; at7_entry_t *aep = (at7_entry_t *)mp->nt_lreserved; fcportdb_t *lp; uint32_t sid; uint16_t nphdl; sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2]; if (isp_find_pdb_by_portid(isp, mp->nt_channel, sid, &lp)) { nphdl = lp->handle; } else { nphdl = NIL_HANDLE; } ISP_MEMZERO(&local, sizeof (local)); cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7; cto->ct_header.rqs_entry_count = 1; cto->ct_nphdl = nphdl; cto->ct_rxid = aep->at_rxid; cto->ct_vpidx = mp->nt_channel; cto->ct_iid_lo = sid; cto->ct_iid_hi = sid >> 16; cto->ct_oxid = aep->at_hdr.ox_id; cto->ct_flags = CT7_SENDSTATUS|CT7_NOACK|CT7_NO_DATA|CT7_FLAG_MODE1; cto->ct_flags |= (aep->at_ta_len >> 12) << CT7_TASK_ATTR_SHIFT; if (rsp != 0) { cto->ct_scsi_status |= (FCP_RSPLEN_VALID << 8); cto->rsp.m1.ct_resplen = 4; ISP_MEMZERO(cto->rsp.m1.ct_resp, sizeof (cto->rsp.m1.ct_resp)); cto->rsp.m1.ct_resp[0] = rsp & 0xff; cto->rsp.m1.ct_resp[1] = (rsp >> 8) & 0xff; cto->rsp.m1.ct_resp[2] = (rsp >> 16) & 0xff; cto->rsp.m1.ct_resp[3] = (rsp >> 24) & 0xff; } return (isp_target_put_entry(isp, &local)); } /* * This case is for a responding to an ABTS frame */ if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) { /* * Overload nt_need_ack here to mark whether we've terminated the associated command. */ if (mp->nt_need_ack) { uint8_t storage[QENTRY_LEN]; ct7_entry_t *cto = (ct7_entry_t *) storage; abts_t *abts = (abts_t *)mp->nt_lreserved; ISP_MEMZERO(cto, sizeof (ct7_entry_t)); isp_prt(isp, ISP_LOGTDEBUG0, "%s: [%x] terminating after ABTS received", __func__, abts->abts_rxid_task); cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7; cto->ct_header.rqs_entry_count = 1; cto->ct_nphdl = mp->nt_nphdl; cto->ct_rxid = abts->abts_rxid_task; cto->ct_iid_lo = mp->nt_sid; cto->ct_iid_hi = mp->nt_sid >> 16; cto->ct_oxid = abts->abts_ox_id; cto->ct_vpidx = mp->nt_channel; cto->ct_flags = CT7_NOACK|CT7_TERMINATE; if (isp_target_put_entry(isp, cto)) { return (ENOMEM); } mp->nt_need_ack = 0; } if (isp_acknak_abts(isp, mp->nt_lreserved, 0) == ENOMEM) { return (ENOMEM); } else { return (0); } } /* * Handle logout cases here */ if (mp->nt_ncode == NT_GLOBAL_LOGOUT) { isp_del_all_wwn_entries(isp, mp->nt_channel); } if (mp->nt_ncode == NT_LOGOUT) { if (!IS_2100(isp) && IS_FC(isp)) { isp_del_wwn_entries(isp, mp); } } /* * General purpose acknowledgement */ if (mp->nt_need_ack) { isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) being acked", mp->nt_ncode, mp->nt_lreserved != NULL); /* * Don't need to use the guaranteed send because the caller can retry */ return (isp_notify_ack(isp, mp->nt_lreserved)); } return (0); } /* * Handle task management functions. * * We show up here with a notify structure filled out. * * The nt_lreserved tag points to the original queue entry */ static void isp_handle_platform_target_tmf(ispsoftc_t *isp, isp_notify_t *notify) { tstate_t *tptr; fcportdb_t *lp; struct ccb_immediate_notify *inot; inot_private_data_t *ntp = NULL; atio_private_data_t *atp; lun_id_t lun; isp_prt(isp, ISP_LOGTDEBUG0, "%s: code 0x%x sid 0x%x tagval 0x%016llx chan %d lun %jx", __func__, notify->nt_ncode, notify->nt_sid, (unsigned long long) notify->nt_tagval, notify->nt_channel, notify->nt_lun); if (notify->nt_lun == LUN_ANY) { if (notify->nt_tagval == TAG_ANY) { lun = CAM_LUN_WILDCARD; } else { atp = isp_find_atpd(isp, notify->nt_channel, notify->nt_tagval & 0xffffffff); lun = atp ? atp->lun : CAM_LUN_WILDCARD; } } else { lun = notify->nt_lun; } tptr = get_lun_statep(isp, notify->nt_channel, lun); if (tptr == NULL) { tptr = get_lun_statep(isp, notify->nt_channel, CAM_LUN_WILDCARD); if (tptr == NULL) { isp_prt(isp, ISP_LOGWARN, "%s: no state pointer found for chan %d lun %#jx", __func__, notify->nt_channel, (uintmax_t)lun); goto bad; } } inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots); if (inot == NULL) { isp_prt(isp, ISP_LOGWARN, "%s: out of immediate notify structures for chan %d lun %#jx", __func__, notify->nt_channel, (uintmax_t)lun); goto bad; } inot->ccb_h.target_id = ISP_MAX_TARGETS(isp); inot->ccb_h.target_lun = lun; if (isp_find_pdb_by_portid(isp, notify->nt_channel, notify->nt_sid, &lp) == 0 && isp_find_pdb_by_handle(isp, notify->nt_channel, notify->nt_nphdl, &lp) == 0) { inot->initiator_id = CAM_TARGET_WILDCARD; } else { inot->initiator_id = FC_PORTDB_TGT(isp, notify->nt_channel, lp); } inot->seq_id = notify->nt_tagval; inot->tag_id = notify->nt_tagval >> 32; switch (notify->nt_ncode) { case NT_ABORT_TASK: isp_target_mark_aborted_early(isp, notify->nt_channel, tptr, inot->tag_id); inot->arg = MSG_ABORT_TASK; break; case NT_ABORT_TASK_SET: isp_target_mark_aborted_early(isp, notify->nt_channel, tptr, TAG_ANY); inot->arg = MSG_ABORT_TASK_SET; break; case NT_CLEAR_ACA: inot->arg = MSG_CLEAR_ACA; break; case NT_CLEAR_TASK_SET: inot->arg = MSG_CLEAR_TASK_SET; break; case NT_LUN_RESET: inot->arg = MSG_LOGICAL_UNIT_RESET; break; case NT_TARGET_RESET: inot->arg = MSG_TARGET_RESET; break; case NT_QUERY_TASK_SET: inot->arg = MSG_QUERY_TASK_SET; break; case NT_QUERY_ASYNC_EVENT: inot->arg = MSG_QUERY_ASYNC_EVENT; break; default: isp_prt(isp, ISP_LOGWARN, "%s: unknown TMF code 0x%x for chan %d lun %#jx", __func__, notify->nt_ncode, notify->nt_channel, (uintmax_t)lun); goto bad; } ntp = isp_get_ntpd(isp, notify->nt_channel); if (ntp == NULL) { isp_prt(isp, ISP_LOGWARN, "%s: out of inotify private structures", __func__); goto bad; } ISP_MEMCPY(&ntp->nt, notify, sizeof (isp_notify_t)); if (notify->nt_lreserved) { ISP_MEMCPY(&ntp->data, notify->nt_lreserved, QENTRY_LEN); ntp->nt.nt_lreserved = &ntp->data; } ntp->seq_id = notify->nt_tagval; ntp->tag_id = notify->nt_tagval >> 32; SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, inot->ccb_h.path, "Take FREE INOT\n"); inot->ccb_h.status = CAM_MESSAGE_RECV; xpt_done((union ccb *)inot); return; bad: if (notify->nt_need_ack) { if (((isphdr_t *)notify->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) { if (isp_acknak_abts(isp, notify->nt_lreserved, ENOMEM)) { isp_prt(isp, ISP_LOGWARN, "you lose- unable to send an ACKNAK"); } } else { isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, notify->nt_lreserved); } } } static void isp_target_mark_aborted_early(ispsoftc_t *isp, int chan, tstate_t *tptr, uint32_t tag_id) { atio_private_data_t *atp, *atpool; inot_private_data_t *ntp, *tmp; uint32_t this_tag_id; /* * First, clean any commands pending restart */ STAILQ_FOREACH_SAFE(ntp, &tptr->restart_queue, next, tmp) { if (IS_24XX(isp)) this_tag_id = ((at7_entry_t *)ntp->data)->at_rxid; else this_tag_id = ((at2_entry_t *)ntp->data)->at_rxid; if ((uint64_t)tag_id == TAG_ANY || tag_id == this_tag_id) { isp_endcmd(isp, ntp->data, NIL_HANDLE, chan, ECMD_TERMINATE, 0); isp_put_ntpd(isp, chan, ntp); STAILQ_REMOVE(&tptr->restart_queue, ntp, inot_private_data, next); } } /* * Now mark other ones dead as well. */ ISP_GET_PC(isp, chan, atpool, atpool); for (atp = atpool; atp < &atpool[ATPDPSIZE]; atp++) { if (atp->lun != tptr->ts_lun) continue; if ((uint64_t)tag_id == TAG_ANY || atp->tag == tag_id) atp->dead = 1; } } #endif static void isp_cam_async(void *cbarg, uint32_t code, struct cam_path *path, void *arg) { struct cam_sim *sim; int bus, tgt; ispsoftc_t *isp; sim = (struct cam_sim *)cbarg; isp = (ispsoftc_t *) cam_sim_softc(sim); bus = cam_sim_bus(sim); tgt = xpt_path_target_id(path); switch (code) { case AC_LOST_DEVICE: if (IS_SCSI(isp)) { uint16_t oflags, nflags; sdparam *sdp = SDPARAM(isp, bus); if (tgt >= 0) { nflags = sdp->isp_devparam[tgt].nvrm_flags; nflags &= DPARM_SAFE_DFLT; if (isp->isp_loaded_fw) { nflags |= DPARM_NARROW | DPARM_ASYNC; } oflags = sdp->isp_devparam[tgt].goal_flags; sdp->isp_devparam[tgt].goal_flags = nflags; sdp->isp_devparam[tgt].dev_update = 1; sdp->update = 1; (void) isp_control(isp, ISPCTL_UPDATE_PARAMS, bus); sdp->isp_devparam[tgt].goal_flags = oflags; } } break; default: isp_prt(isp, ISP_LOGWARN, "isp_cam_async: Code 0x%x", code); break; } } static void isp_poll(struct cam_sim *sim) { ispsoftc_t *isp = cam_sim_softc(sim); ISP_RUN_ISR(isp); } static void isp_watchdog(void *arg) { struct ccb_scsiio *xs = arg; ispsoftc_t *isp; uint32_t ohandle = ISP_HANDLE_FREE, handle; isp = XS_ISP(xs); handle = isp_find_handle(isp, xs); /* * Hand crank the interrupt code just to be sure the command isn't stuck somewhere. */ if (handle != ISP_HANDLE_FREE) { ISP_RUN_ISR(isp); ohandle = handle; handle = isp_find_handle(isp, xs); } if (handle != ISP_HANDLE_FREE) { /* * Try and make sure the command is really dead before * we release the handle (and DMA resources) for reuse. * * If we are successful in aborting the command then * we're done here because we'll get the command returned * back separately. */ if (isp_control(isp, ISPCTL_ABORT_CMD, xs) == 0) { return; } /* * Note that after calling the above, the command may in * fact have been completed. */ xs = isp_find_xs(isp, handle); /* * If the command no longer exists, then we won't * be able to find the xs again with this handle. */ if (xs == NULL) { return; } /* * After this point, the command is really dead. */ if (XS_XFRLEN(xs)) { ISP_DMAFREE(isp, xs, handle); } isp_destroy_handle(isp, handle); isp_prt(isp, ISP_LOGERR, "%s: timeout for handle 0x%x", __func__, handle); XS_SETERR(xs, CAM_CMD_TIMEOUT); isp_done(xs); } else { if (ohandle != ISP_HANDLE_FREE) { isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle 0x%x, recovered during interrupt", __func__, ohandle); } else { isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle already free", __func__); } } } static void isp_make_here(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt) { union ccb *ccb; struct isp_fc *fc = ISP_FC_PC(isp, chan); /* * Allocate a CCB, create a wildcard path for this target and schedule a rescan. */ ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { isp_prt(isp, ISP_LOGWARN, "Chan %d unable to alloc CCB for rescan", chan); return; } if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(fc->sim), tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { isp_prt(isp, ISP_LOGWARN, "unable to create path for rescan"); xpt_free_ccb(ccb); return; } xpt_rescan(ccb); } static void isp_make_gone(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt) { struct cam_path *tp; struct isp_fc *fc = ISP_FC_PC(isp, chan); if (xpt_create_path(&tp, NULL, cam_sim_path(fc->sim), tgt, CAM_LUN_WILDCARD) == CAM_REQ_CMP) { xpt_async(AC_LOST_DEVICE, tp, NULL); xpt_free_path(tp); } } /* * Gone Device Timer Function- when we have decided that a device has gone * away, we wait a specific period of time prior to telling the OS it has * gone away. * * This timer function fires once a second and then scans the port database * for devices that are marked dead but still have a virtual target assigned. * We decrement a counter for that port database entry, and when it hits zero, * we tell the OS the device has gone away. */ static void isp_gdt(void *arg) { struct isp_fc *fc = arg; taskqueue_enqueue(taskqueue_thread, &fc->gtask); } static void isp_gdt_task(void *arg, int pending) { struct isp_fc *fc = arg; ispsoftc_t *isp = fc->isp; int chan = fc - isp->isp_osinfo.pc.fc; fcportdb_t *lp; struct ac_contract ac; struct ac_device_changed *adc; int dbidx, more_to_do = 0; ISP_LOCK(isp); isp_prt(isp, ISP_LOGDEBUG0, "Chan %d GDT timer expired", chan); for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) { lp = &FCPARAM(isp, chan)->portdb[dbidx]; if (lp->state != FC_PORTDB_STATE_ZOMBIE) { continue; } if (lp->gone_timer != 0) { lp->gone_timer -= 1; more_to_do++; continue; } isp_prt(isp, ISP_LOGCONFIG, prom3, chan, dbidx, lp->portid, "Gone Device Timeout"); if (lp->is_target) { lp->is_target = 0; isp_make_gone(isp, lp, chan, dbidx); } if (lp->is_initiator) { lp->is_initiator = 0; ac.contract_number = AC_CONTRACT_DEV_CHG; adc = (struct ac_device_changed *) ac.contract_data; adc->wwpn = lp->port_wwn; adc->port = lp->portid; adc->target = dbidx; adc->arrived = 0; xpt_async(AC_CONTRACT, fc->path, &ac); } lp->state = FC_PORTDB_STATE_NIL; } if (fc->ready) { if (more_to_do) { callout_reset(&fc->gdt, hz, isp_gdt, fc); } else { callout_deactivate(&fc->gdt); isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Stopping Gone Device Timer @ %lu", chan, (unsigned long) time_uptime); } } ISP_UNLOCK(isp); } /* * When loop goes down we remember the time and freeze CAM command queue. * During some time period we are trying to reprobe the loop. But if we * fail, we tell the OS that devices have gone away and drop the freeze. * * We don't clear the devices out of our port database because, when loop * come back up, we have to do some actual cleanup with the chip at that * point (implicit PLOGO, e.g., to get the chip's port database state right). */ static void isp_loop_changed(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); struct isp_fc *fc = ISP_FC_PC(isp, chan); if (fc->loop_down_time) return; isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop changed", chan); if (fcp->role & ISP_ROLE_INITIATOR) isp_freeze_loopdown(isp, chan); fc->loop_down_time = time_uptime; wakeup(fc); } static void isp_loop_up(ispsoftc_t *isp, int chan) { struct isp_fc *fc = ISP_FC_PC(isp, chan); isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop is up", chan); fc->loop_seen_once = 1; fc->loop_down_time = 0; isp_unfreeze_loopdown(isp, chan); } static void isp_loop_dead(ispsoftc_t *isp, int chan) { fcparam *fcp = FCPARAM(isp, chan); struct isp_fc *fc = ISP_FC_PC(isp, chan); fcportdb_t *lp; struct ac_contract ac; struct ac_device_changed *adc; int dbidx, i; isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop is dead", chan); /* * Notify to the OS all targets who we now consider have departed. */ for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) { lp = &fcp->portdb[dbidx]; if (lp->state == FC_PORTDB_STATE_NIL) continue; for (i = 0; i < isp->isp_maxcmds; i++) { struct ccb_scsiio *xs; if (ISP_H2HT(isp->isp_xflist[i].handle) != ISP_HANDLE_INITIATOR) { continue; } if ((xs = isp->isp_xflist[i].cmd) == NULL) { continue; } if (dbidx != XS_TGT(xs)) { continue; } isp_prt(isp, ISP_LOGWARN, "command handle 0x%x for %d.%d.%jx orphaned by loop down timeout", isp->isp_xflist[i].handle, chan, XS_TGT(xs), (uintmax_t)XS_LUN(xs)); /* * Just like in isp_watchdog, abort the outstanding * command or immediately free its resources if it is * not active */ if (isp_control(isp, ISPCTL_ABORT_CMD, xs) == 0) { continue; } if (XS_XFRLEN(xs)) { ISP_DMAFREE(isp, xs, isp->isp_xflist[i].handle); } isp_destroy_handle(isp, isp->isp_xflist[i].handle); isp_prt(isp, ISP_LOGWARN, "command handle 0x%x for %d.%d.%jx could not be aborted and was destroyed", isp->isp_xflist[i].handle, chan, XS_TGT(xs), (uintmax_t)XS_LUN(xs)); XS_SETERR(xs, HBA_BUSRESET); isp_done(xs); } isp_prt(isp, ISP_LOGCONFIG, prom3, chan, dbidx, lp->portid, "Loop Down Timeout"); if (lp->is_target) { lp->is_target = 0; isp_make_gone(isp, lp, chan, dbidx); } if (lp->is_initiator) { lp->is_initiator = 0; ac.contract_number = AC_CONTRACT_DEV_CHG; adc = (struct ac_device_changed *) ac.contract_data; adc->wwpn = lp->port_wwn; adc->port = lp->portid; adc->target = dbidx; adc->arrived = 0; xpt_async(AC_CONTRACT, fc->path, &ac); } } isp_unfreeze_loopdown(isp, chan); fc->loop_down_time = 0; } static void isp_kthread(void *arg) { struct isp_fc *fc = arg; ispsoftc_t *isp = fc->isp; int chan = fc - isp->isp_osinfo.pc.fc; int slp = 0, d; int lb, lim; ISP_LOCK(isp); while (isp->isp_osinfo.is_exiting == 0) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Checking FC state", chan); lb = isp_fc_runstate(isp, chan, 250000); isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d FC got to %s state", chan, isp_fc_loop_statename(lb)); /* * Our action is different based upon whether we're supporting * Initiator mode or not. If we are, we might freeze the simq * when loop is down and set all sorts of different delays to * check again. * * If not, we simply just wait for loop to come up. */ if (lb == LOOP_READY || lb < 0) { slp = 0; } else { /* * If we've never seen loop up and we've waited longer * than quickboot time, or we've seen loop up but we've * waited longer than loop_down_limit, give up and go * to sleep until loop comes up. */ if (fc->loop_seen_once == 0) lim = isp_quickboot_time; else lim = fc->loop_down_limit; d = time_uptime - fc->loop_down_time; if (d >= lim) slp = 0; else if (d < 10) slp = 1; else if (d < 30) slp = 5; else if (d < 60) slp = 10; else if (d < 120) slp = 20; else slp = 30; } if (slp == 0) { if (lb == LOOP_READY) isp_loop_up(isp, chan); else isp_loop_dead(isp, chan); } isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d sleep for %d seconds", chan, slp); msleep(fc, &isp->isp_lock, PRIBIO, "ispf", slp * hz); } fc->num_threads -= 1; ISP_UNLOCK(isp); kthread_exit(); } #ifdef ISP_TARGET_MODE static void isp_abort_atio(ispsoftc_t *isp, union ccb *ccb) { atio_private_data_t *atp; union ccb *accb = ccb->cab.abort_ccb; struct ccb_hdr *sccb; tstate_t *tptr; tptr = get_lun_statep(isp, XS_CHANNEL(accb), XS_LUN(accb)); if (tptr != NULL) { /* Search for the ATIO among queueued. */ SLIST_FOREACH(sccb, &tptr->atios, sim_links.sle) { if (sccb != &accb->ccb_h) continue; SLIST_REMOVE(&tptr->atios, sccb, ccb_hdr, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, sccb->path, "Abort FREE ATIO\n"); accb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(accb); ccb->ccb_h.status = CAM_REQ_CMP; return; } } /* Search for the ATIO among running. */ atp = isp_find_atpd(isp, XS_CHANNEL(accb), accb->atio.tag_id); if (atp != NULL) { /* Send TERMINATE to firmware. */ if (!atp->dead && IS_24XX(isp)) { uint8_t storage[QENTRY_LEN]; ct7_entry_t *cto = (ct7_entry_t *) storage; ISP_MEMZERO(cto, sizeof (ct7_entry_t)); cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7; cto->ct_header.rqs_entry_count = 1; cto->ct_nphdl = atp->nphdl; cto->ct_rxid = atp->tag; cto->ct_iid_lo = atp->sid; cto->ct_iid_hi = atp->sid >> 16; cto->ct_oxid = atp->oxid; cto->ct_vpidx = XS_CHANNEL(accb); cto->ct_flags = CT7_NOACK|CT7_TERMINATE; isp_target_put_entry(isp, cto); } isp_put_atpd(isp, XS_CHANNEL(accb), atp); ccb->ccb_h.status = CAM_REQ_CMP; } else { ccb->ccb_h.status = CAM_UA_ABORT; } } static void isp_abort_inot(ispsoftc_t *isp, union ccb *ccb) { inot_private_data_t *ntp; union ccb *accb = ccb->cab.abort_ccb; struct ccb_hdr *sccb; tstate_t *tptr; tptr = get_lun_statep(isp, XS_CHANNEL(accb), XS_LUN(accb)); if (tptr != NULL) { /* Search for the INOT among queueued. */ SLIST_FOREACH(sccb, &tptr->inots, sim_links.sle) { if (sccb != &accb->ccb_h) continue; SLIST_REMOVE(&tptr->inots, sccb, ccb_hdr, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, sccb->path, "Abort FREE INOT\n"); accb->ccb_h.status = CAM_REQ_ABORTED; xpt_done(accb); ccb->ccb_h.status = CAM_REQ_CMP; return; } } /* Search for the INOT among running. */ ntp = isp_find_ntpd(isp, XS_CHANNEL(accb), accb->cin1.tag_id, accb->cin1.seq_id); if (ntp != NULL) { if (ntp->nt.nt_need_ack) { isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, ntp->nt.nt_lreserved); } isp_put_ntpd(isp, XS_CHANNEL(accb), ntp); ccb->ccb_h.status = CAM_REQ_CMP; } else { ccb->ccb_h.status = CAM_UA_ABORT; return; } } #endif static void isp_action(struct cam_sim *sim, union ccb *ccb) { int bus, tgt, error; ispsoftc_t *isp; struct ccb_trans_settings *cts; sbintime_t ts; CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("isp_action\n")); isp = (ispsoftc_t *)cam_sim_softc(sim); ISP_ASSERT_LOCKED(isp); bus = cam_sim_bus(sim); isp_prt(isp, ISP_LOGDEBUG2, "isp_action code %x", ccb->ccb_h.func_code); ISP_PCMD(ccb) = NULL; switch (ccb->ccb_h.func_code) { case XPT_SCSI_IO: /* Execute the requested I/O operation */ /* * Do a couple of preliminary checks... */ if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0) { if ((ccb->ccb_h.flags & CAM_CDB_PHYS) != 0) { ccb->ccb_h.status = CAM_REQ_INVALID; isp_done((struct ccb_scsiio *) ccb); break; } } ccb->csio.req_map = NULL; #ifdef DIAGNOSTIC if (ccb->ccb_h.target_id >= ISP_MAX_TARGETS(isp)) { xpt_print(ccb->ccb_h.path, "invalid target\n"); ccb->ccb_h.status = CAM_PATH_INVALID; } else if (ISP_MAX_LUNS(isp) > 0 && ccb->ccb_h.target_lun >= ISP_MAX_LUNS(isp)) { xpt_print(ccb->ccb_h.path, "invalid lun\n"); ccb->ccb_h.status = CAM_PATH_INVALID; } if (ccb->ccb_h.status == CAM_PATH_INVALID) { xpt_done(ccb); break; } #endif ccb->csio.scsi_status = SCSI_STATUS_OK; if (isp_get_pcmd(isp, ccb)) { isp_prt(isp, ISP_LOGWARN, "out of PCMDs"); cam_freeze_devq(ccb->ccb_h.path); cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0); ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); break; } error = isp_start((XS_T *) ccb); switch (error) { case CMD_QUEUED: ccb->ccb_h.status |= CAM_SIM_QUEUED; if (ccb->ccb_h.timeout == CAM_TIME_INFINITY) break; /* Give firmware extra 10s to handle timeout. */ ts = SBT_1MS * ccb->ccb_h.timeout + 10 * SBT_1S; callout_reset_sbt(&PISP_PCMD(ccb)->wdog, ts, 0, isp_watchdog, ccb, 0); break; case CMD_RQLATER: isp_prt(isp, ISP_LOGDEBUG0, "%d.%jx retry later", XS_TGT(ccb), (uintmax_t)XS_LUN(ccb)); cam_freeze_devq(ccb->ccb_h.path); cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0); ccb->ccb_h.status = CAM_REQUEUE_REQ; isp_free_pcmd(isp, ccb); xpt_done(ccb); break; case CMD_EAGAIN: isp_free_pcmd(isp, ccb); cam_freeze_devq(ccb->ccb_h.path); cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 100, 0); ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); break; case CMD_COMPLETE: isp_done((struct ccb_scsiio *) ccb); break; default: isp_prt(isp, ISP_LOGERR, "What's this? 0x%x at %d in file %s", error, __LINE__, __FILE__); ccb->ccb_h.status = CAM_REQUEUE_REQ; isp_free_pcmd(isp, ccb); xpt_done(ccb); } break; #ifdef ISP_TARGET_MODE case XPT_EN_LUN: /* Enable/Disable LUN as a target */ if (ccb->cel.enable) { isp_enable_lun(isp, ccb); } else { isp_disable_lun(isp, ccb); } break; case XPT_IMMEDIATE_NOTIFY: /* Add Immediate Notify Resource */ case XPT_ACCEPT_TARGET_IO: /* Add Accept Target IO Resource */ { tstate_t *tptr = get_lun_statep(isp, XS_CHANNEL(ccb), ccb->ccb_h.target_lun); if (tptr == NULL) { const char *str; if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY) str = "XPT_IMMEDIATE_NOTIFY"; else str = "XPT_ACCEPT_TARGET_IO"; ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "%s: no state pointer found for %s\n", __func__, str); ccb->ccb_h.status = CAM_DEV_NOT_THERE; xpt_done(ccb); break; } ccb->ccb_h.spriv_field0 = 0; ccb->ccb_h.spriv_ptr1 = isp; if (ccb->ccb_h.func_code == XPT_ACCEPT_TARGET_IO) { ccb->atio.tag_id = 0; SLIST_INSERT_HEAD(&tptr->atios, &ccb->ccb_h, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path, "Put FREE ATIO\n"); } else if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY) { ccb->cin1.seq_id = ccb->cin1.tag_id = 0; SLIST_INSERT_HEAD(&tptr->inots, &ccb->ccb_h, sim_links.sle); ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path, "Put FREE INOT\n"); } ccb->ccb_h.status = CAM_REQ_INPROG; break; } case XPT_NOTIFY_ACKNOWLEDGE: /* notify ack */ { inot_private_data_t *ntp; /* * XXX: Because we cannot guarantee that the path information in the notify acknowledge ccb * XXX: matches that for the immediate notify, we have to *search* for the notify structure */ /* * All the relevant path information is in the associated immediate notify */ ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] NOTIFY ACKNOWLEDGE for 0x%x seen\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id); ntp = isp_find_ntpd(isp, XS_CHANNEL(ccb), ccb->cna2.tag_id, ccb->cna2.seq_id); if (ntp == NULL) { ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "%s: [0x%x] XPT_NOTIFY_ACKNOWLEDGE of 0x%x cannot find ntp private data\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id); ccb->ccb_h.status = CAM_DEV_NOT_THERE; xpt_done(ccb); break; } if (isp_handle_platform_target_notify_ack(isp, &ntp->nt, (ccb->ccb_h.flags & CAM_SEND_STATUS) ? ccb->cna2.arg : 0)) { cam_freeze_devq(ccb->ccb_h.path); cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0); ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_REQUEUE_REQ; break; } isp_put_ntpd(isp, XS_CHANNEL(ccb), ntp); ccb->ccb_h.status = CAM_REQ_CMP; ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] calling xpt_done for tag 0x%x\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id); xpt_done(ccb); break; } case XPT_CONT_TARGET_IO: isp_target_start_ctio(isp, ccb, FROM_CAM); break; #endif case XPT_RESET_DEV: /* BDR the specified SCSI device */ tgt = ccb->ccb_h.target_id; tgt |= (bus << 16); error = isp_control(isp, ISPCTL_RESET_DEV, bus, tgt); if (error) { ccb->ccb_h.status = CAM_REQ_CMP_ERR; } else { /* * If we have a FC device, reset the Command * Reference Number, because the target will expect * that we re-start the CRN at 1 after a reset. */ if (IS_FC(isp)) isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1); ccb->ccb_h.status = CAM_REQ_CMP; } xpt_done(ccb); break; case XPT_ABORT: /* Abort the specified CCB */ { union ccb *accb = ccb->cab.abort_ccb; switch (accb->ccb_h.func_code) { #ifdef ISP_TARGET_MODE case XPT_ACCEPT_TARGET_IO: isp_abort_atio(isp, ccb); break; case XPT_IMMEDIATE_NOTIFY: isp_abort_inot(isp, ccb); break; #endif case XPT_SCSI_IO: error = isp_control(isp, ISPCTL_ABORT_CMD, accb); if (error) { ccb->ccb_h.status = CAM_UA_ABORT; } else { ccb->ccb_h.status = CAM_REQ_CMP; } break; default: ccb->ccb_h.status = CAM_REQ_INVALID; break; } /* * This is not a queued CCB, so the caller expects it to be * complete when control is returned. */ break; } #define IS_CURRENT_SETTINGS(c) (c->type == CTS_TYPE_CURRENT_SETTINGS) case XPT_SET_TRAN_SETTINGS: /* Nexus Settings */ cts = &ccb->cts; if (!IS_CURRENT_SETTINGS(cts)) { ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); break; } tgt = cts->ccb_h.target_id; if (IS_SCSI(isp)) { struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi; struct ccb_trans_settings_spi *spi = &cts->xport_specific.spi; sdparam *sdp = SDPARAM(isp, bus); uint16_t *dptr; if (spi->valid == 0 && scsi->valid == 0) { ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); break; } /* * We always update (internally) from goal_flags * so any request to change settings just gets * vectored to that location. */ dptr = &sdp->isp_devparam[tgt].goal_flags; if ((spi->valid & CTS_SPI_VALID_DISC) != 0) { if ((spi->flags & CTS_SPI_FLAGS_DISC_ENB) != 0) *dptr |= DPARM_DISC; else *dptr &= ~DPARM_DISC; } if ((scsi->valid & CTS_SCSI_VALID_TQ) != 0) { if ((scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0) *dptr |= DPARM_TQING; else *dptr &= ~DPARM_TQING; } if ((spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0) { if (spi->bus_width == MSG_EXT_WDTR_BUS_16_BIT) *dptr |= DPARM_WIDE; else *dptr &= ~DPARM_WIDE; } /* * XXX: FIX ME */ if ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET) && (spi->valid & CTS_SPI_VALID_SYNC_RATE) && (spi->sync_period && spi->sync_offset)) { *dptr |= DPARM_SYNC; /* * XXX: CHECK FOR LEGALITY */ sdp->isp_devparam[tgt].goal_period = spi->sync_period; sdp->isp_devparam[tgt].goal_offset = spi->sync_offset; } else { *dptr &= ~DPARM_SYNC; } isp_prt(isp, ISP_LOGDEBUG0, "SET (%d.%d.%jx) to flags %x off %x per %x", bus, tgt, (uintmax_t)cts->ccb_h.target_lun, sdp->isp_devparam[tgt].goal_flags, sdp->isp_devparam[tgt].goal_offset, sdp->isp_devparam[tgt].goal_period); sdp->isp_devparam[tgt].dev_update = 1; sdp->update = 1; } ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); break; case XPT_GET_TRAN_SETTINGS: cts = &ccb->cts; tgt = cts->ccb_h.target_id; if (IS_FC(isp)) { fcparam *fcp = FCPARAM(isp, bus); struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi; struct ccb_trans_settings_fc *fc = &cts->xport_specific.fc; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_2; cts->transport = XPORT_FC; cts->transport_version = 0; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; fc->valid = CTS_FC_VALID_SPEED; fc->bitrate = 100000; fc->bitrate *= fcp->isp_gbspeed; if (tgt < MAX_FC_TARG) { fcportdb_t *lp = &fcp->portdb[tgt]; fc->wwnn = lp->node_wwn; fc->wwpn = lp->port_wwn; fc->port = lp->portid; fc->valid |= CTS_FC_VALID_WWNN | CTS_FC_VALID_WWPN | CTS_FC_VALID_PORT; } } else { struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi; struct ccb_trans_settings_spi *spi = &cts->xport_specific.spi; sdparam *sdp = SDPARAM(isp, bus); uint16_t dval, pval, oval; if (IS_CURRENT_SETTINGS(cts)) { sdp->isp_devparam[tgt].dev_refresh = 1; sdp->update = 1; (void) isp_control(isp, ISPCTL_UPDATE_PARAMS, bus); dval = sdp->isp_devparam[tgt].actv_flags; oval = sdp->isp_devparam[tgt].actv_offset; pval = sdp->isp_devparam[tgt].actv_period; } else { dval = sdp->isp_devparam[tgt].nvrm_flags; oval = sdp->isp_devparam[tgt].nvrm_offset; pval = sdp->isp_devparam[tgt].nvrm_period; } cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_2; cts->transport = XPORT_SPI; cts->transport_version = 2; spi->valid = 0; scsi->valid = 0; spi->flags = 0; scsi->flags = 0; if (dval & DPARM_DISC) { spi->flags |= CTS_SPI_FLAGS_DISC_ENB; } if ((dval & DPARM_SYNC) && oval && pval) { spi->sync_offset = oval; spi->sync_period = pval; } else { spi->sync_offset = 0; spi->sync_period = 0; } spi->valid |= CTS_SPI_VALID_SYNC_OFFSET; spi->valid |= CTS_SPI_VALID_SYNC_RATE; spi->valid |= CTS_SPI_VALID_BUS_WIDTH; if (dval & DPARM_WIDE) { spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT; } else { spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT; } if (cts->ccb_h.target_lun != CAM_LUN_WILDCARD) { scsi->valid = CTS_SCSI_VALID_TQ; if (dval & DPARM_TQING) { scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB; } spi->valid |= CTS_SPI_VALID_DISC; } isp_prt(isp, ISP_LOGDEBUG0, "GET %s (%d.%d.%jx) to flags %x off %x per %x", IS_CURRENT_SETTINGS(cts)? "ACTIVE" : "NVRAM", bus, tgt, (uintmax_t)cts->ccb_h.target_lun, dval, oval, pval); } ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); break; case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, 1); xpt_done(ccb); break; case XPT_RESET_BUS: /* Reset the specified bus */ error = isp_control(isp, ISPCTL_RESET_BUS, bus); if (error) { ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); break; } if (bootverbose) { xpt_print(ccb->ccb_h.path, "reset bus on channel %d\n", bus); } if (IS_FC(isp)) { xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, 0); } else { xpt_async(AC_BUS_RESET, ISP_SPI_PC(isp, bus)->path, 0); } ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); break; case XPT_TERM_IO: /* Terminate the I/O process */ ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); break; case XPT_SET_SIM_KNOB: /* Set SIM knobs */ { struct ccb_sim_knob *kp = &ccb->knob; fcparam *fcp; if (!IS_FC(isp)) { ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); break; } fcp = FCPARAM(isp, bus); if (kp->xport_specific.fc.valid & KNOB_VALID_ADDRESS) { fcp->isp_wwnn = ISP_FC_PC(isp, bus)->def_wwnn = kp->xport_specific.fc.wwnn; fcp->isp_wwpn = ISP_FC_PC(isp, bus)->def_wwpn = kp->xport_specific.fc.wwpn; isp_prt(isp, ISP_LOGALL, "Setting Channel %d wwns to 0x%jx 0x%jx", bus, fcp->isp_wwnn, fcp->isp_wwpn); } ccb->ccb_h.status = CAM_REQ_CMP; if (kp->xport_specific.fc.valid & KNOB_VALID_ROLE) { int rchange = 0; int newrole = 0; switch (kp->xport_specific.fc.role) { case KNOB_ROLE_NONE: if (fcp->role != ISP_ROLE_NONE) { rchange = 1; newrole = ISP_ROLE_NONE; } break; case KNOB_ROLE_TARGET: if (fcp->role != ISP_ROLE_TARGET) { rchange = 1; newrole = ISP_ROLE_TARGET; } break; case KNOB_ROLE_INITIATOR: if (fcp->role != ISP_ROLE_INITIATOR) { rchange = 1; newrole = ISP_ROLE_INITIATOR; } break; case KNOB_ROLE_BOTH: if (fcp->role != ISP_ROLE_BOTH) { rchange = 1; newrole = ISP_ROLE_BOTH; } break; } if (rchange) { ISP_PATH_PRT(isp, ISP_LOGCONFIG, ccb->ccb_h.path, "changing role on from %d to %d\n", fcp->role, newrole); if (isp_control(isp, ISPCTL_CHANGE_ROLE, bus, newrole) != 0) { ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); break; } } } xpt_done(ccb); break; } case XPT_GET_SIM_KNOB_OLD: /* Get SIM knobs -- compat value */ case XPT_GET_SIM_KNOB: /* Get SIM knobs */ { struct ccb_sim_knob *kp = &ccb->knob; if (IS_FC(isp)) { fcparam *fcp; fcp = FCPARAM(isp, bus); kp->xport_specific.fc.wwnn = fcp->isp_wwnn; kp->xport_specific.fc.wwpn = fcp->isp_wwpn; switch (fcp->role) { case ISP_ROLE_NONE: kp->xport_specific.fc.role = KNOB_ROLE_NONE; break; case ISP_ROLE_TARGET: kp->xport_specific.fc.role = KNOB_ROLE_TARGET; break; case ISP_ROLE_INITIATOR: kp->xport_specific.fc.role = KNOB_ROLE_INITIATOR; break; case ISP_ROLE_BOTH: kp->xport_specific.fc.role = KNOB_ROLE_BOTH; break; } kp->xport_specific.fc.valid = KNOB_VALID_ADDRESS | KNOB_VALID_ROLE; ccb->ccb_h.status = CAM_REQ_CMP; } else { ccb->ccb_h.status = CAM_REQ_INVALID; } xpt_done(ccb); break; } case XPT_PATH_INQ: /* Path routing inquiry */ { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; #ifdef ISP_TARGET_MODE if (IS_FC(isp) && ISP_CAP_TMODE(isp) && ISP_CAP_SCCFW(isp)) cpi->target_sprt = PIT_PROCESSOR | PIT_DISCONNECT | PIT_TERM_IO; else #endif cpi->target_sprt = 0; cpi->hba_eng_cnt = 0; cpi->max_target = ISP_MAX_TARGETS(isp) - 1; cpi->max_lun = ISP_MAX_LUNS(isp) == 0 ? 255 : ISP_MAX_LUNS(isp) - 1; cpi->bus_id = cam_sim_bus(sim); if (sizeof (bus_size_t) > 4) cpi->maxio = (ISP_NSEG64_MAX - 1) * PAGE_SIZE; else cpi->maxio = (ISP_NSEG_MAX - 1) * PAGE_SIZE; if (IS_FC(isp)) { fcparam *fcp = FCPARAM(isp, bus); cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED; cpi->hba_misc |= PIM_EXTLUNS | PIM_NOSCAN; /* * Because our loop ID can shift from time to time, * make our initiator ID out of range of our bus. */ cpi->initiator_id = cpi->max_target + 1; /* * Set base transfer capabilities for Fibre Channel, for this HBA. */ if (IS_25XX(isp)) { cpi->base_transfer_speed = 8000000; } else if (IS_24XX(isp)) { cpi->base_transfer_speed = 4000000; } else if (IS_23XX(isp)) { cpi->base_transfer_speed = 2000000; } else { cpi->base_transfer_speed = 1000000; } cpi->hba_inquiry = PI_TAG_ABLE; cpi->transport = XPORT_FC; cpi->transport_version = 0; cpi->xport_specific.fc.wwnn = fcp->isp_wwnn; cpi->xport_specific.fc.wwpn = fcp->isp_wwpn; cpi->xport_specific.fc.port = fcp->isp_portid; cpi->xport_specific.fc.bitrate = fcp->isp_gbspeed * 1000; } else { sdparam *sdp = SDPARAM(isp, bus); cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16; cpi->hba_misc = PIM_UNMAPPED; cpi->initiator_id = sdp->isp_initiator_id; cpi->base_transfer_speed = 3300; cpi->transport = XPORT_SPI; cpi->transport_version = 2; } cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_2; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "Qlogic", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); break; } default: ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); break; } } void isp_done(XS_T *sccb) { ispsoftc_t *isp = XS_ISP(sccb); uint32_t status; if (XS_NOERR(sccb)) XS_SETERR(sccb, CAM_REQ_CMP); if ((sccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && (sccb->scsi_status != SCSI_STATUS_OK)) { sccb->ccb_h.status &= ~CAM_STATUS_MASK; if ((sccb->scsi_status == SCSI_STATUS_CHECK_COND) && (sccb->ccb_h.status & CAM_AUTOSNS_VALID) == 0) { sccb->ccb_h.status |= CAM_AUTOSENSE_FAIL; } else { sccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR; } } sccb->ccb_h.status &= ~CAM_SIM_QUEUED; status = sccb->ccb_h.status & CAM_STATUS_MASK; if (status != CAM_REQ_CMP && (sccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { sccb->ccb_h.status |= CAM_DEV_QFRZN; xpt_freeze_devq(sccb->ccb_h.path, 1); } if (ISP_PCMD(sccb)) { if (callout_active(&PISP_PCMD(sccb)->wdog)) callout_stop(&PISP_PCMD(sccb)->wdog); isp_free_pcmd(isp, (union ccb *) sccb); } xpt_done((union ccb *) sccb); } void isp_async(ispsoftc_t *isp, ispasync_t cmd, ...) { int bus; static const char prom[] = "Chan %d [%d] WWPN 0x%16jx PortID 0x%06x handle 0x%x %s %s"; char buf[64]; char *msg = NULL; target_id_t tgt = 0; fcportdb_t *lp; struct isp_fc *fc; struct cam_path *tmppath; struct ac_contract ac; struct ac_device_changed *adc; va_list ap; switch (cmd) { case ISPASYNC_NEW_TGT_PARAMS: { struct ccb_trans_settings_scsi *scsi; struct ccb_trans_settings_spi *spi; int flags, tgt; sdparam *sdp; struct ccb_trans_settings cts; memset(&cts, 0, sizeof (struct ccb_trans_settings)); va_start(ap, cmd); bus = va_arg(ap, int); tgt = va_arg(ap, int); va_end(ap); sdp = SDPARAM(isp, bus); if (xpt_create_path(&tmppath, NULL, cam_sim_path(ISP_SPI_PC(isp, bus)->sim), tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { isp_prt(isp, ISP_LOGWARN, "isp_async cannot make temp path for %d.%d", tgt, bus); break; } flags = sdp->isp_devparam[tgt].actv_flags; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.protocol = PROTO_SCSI; cts.transport = XPORT_SPI; scsi = &cts.proto_specific.scsi; spi = &cts.xport_specific.spi; if (flags & DPARM_TQING) { scsi->valid |= CTS_SCSI_VALID_TQ; scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB; } if (flags & DPARM_DISC) { spi->valid |= CTS_SPI_VALID_DISC; spi->flags |= CTS_SPI_FLAGS_DISC_ENB; } spi->flags |= CTS_SPI_VALID_BUS_WIDTH; if (flags & DPARM_WIDE) { spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT; } else { spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT; } if (flags & DPARM_SYNC) { spi->valid |= CTS_SPI_VALID_SYNC_RATE; spi->valid |= CTS_SPI_VALID_SYNC_OFFSET; spi->sync_period = sdp->isp_devparam[tgt].actv_period; spi->sync_offset = sdp->isp_devparam[tgt].actv_offset; } isp_prt(isp, ISP_LOGDEBUG2, "NEW_TGT_PARAMS bus %d tgt %d period %x offset %x flags %x", bus, tgt, sdp->isp_devparam[tgt].actv_period, sdp->isp_devparam[tgt].actv_offset, flags); xpt_setup_ccb(&cts.ccb_h, tmppath, 1); xpt_async(AC_TRANSFER_NEG, tmppath, &cts); xpt_free_path(tmppath); break; } case ISPASYNC_BUS_RESET: { va_start(ap, cmd); bus = va_arg(ap, int); va_end(ap); isp_prt(isp, ISP_LOGINFO, "SCSI bus reset on bus %d detected", bus); if (IS_FC(isp)) { xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, NULL); } else { xpt_async(AC_BUS_RESET, ISP_SPI_PC(isp, bus)->path, NULL); } break; } case ISPASYNC_LOOP_RESET: { uint16_t lipp; fcparam *fcp; va_start(ap, cmd); bus = va_arg(ap, int); va_end(ap); lipp = ISP_READ(isp, OUTMAILBOX1); fcp = FCPARAM(isp, bus); isp_prt(isp, ISP_LOGINFO, "Chan %d LOOP Reset, LIP primitive %x", bus, lipp); /* * Per FCP-4, a Reset LIP should result in a CRN reset. Other * LIPs and loop up/down events should never reset the CRN. For * an as of yet unknown reason, 24xx series cards (and * potentially others) can interrupt with a LIP Reset status * when no LIP reset came down the wire. Additionally, the LIP * primitive accompanying this status would not be a valid LIP * Reset primitive, but some variation of an invalid AL_PA * LIP. As a result, we have to verify the AL_PD in the LIP * addresses our port before blindly resetting. */ if (FCP_IS_DEST_ALPD(fcp, (lipp & 0x00FF))) isp_fcp_reset_crn(isp, bus, /*tgt*/0, /*tgt_set*/ 0); isp_loop_changed(isp, bus); break; } case ISPASYNC_LIP: if (msg == NULL) msg = "LIP Received"; /* FALLTHROUGH */ case ISPASYNC_LOOP_DOWN: if (msg == NULL) msg = "LOOP Down"; /* FALLTHROUGH */ case ISPASYNC_LOOP_UP: if (msg == NULL) msg = "LOOP Up"; va_start(ap, cmd); bus = va_arg(ap, int); va_end(ap); isp_loop_changed(isp, bus); isp_prt(isp, ISP_LOGINFO, "Chan %d %s", bus, msg); break; case ISPASYNC_DEV_ARRIVED: va_start(ap, cmd); bus = va_arg(ap, int); lp = va_arg(ap, fcportdb_t *); va_end(ap); fc = ISP_FC_PC(isp, bus); tgt = FC_PORTDB_TGT(isp, bus, lp); isp_gen_role_str(buf, sizeof (buf), lp->prli_word3); isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "arrived"); if ((FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) && (lp->prli_word3 & PRLI_WD3_TARGET_FUNCTION)) { lp->is_target = 1; isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1); isp_make_here(isp, lp, bus, tgt); } if ((FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) && (lp->prli_word3 & PRLI_WD3_INITIATOR_FUNCTION)) { lp->is_initiator = 1; ac.contract_number = AC_CONTRACT_DEV_CHG; adc = (struct ac_device_changed *) ac.contract_data; adc->wwpn = lp->port_wwn; adc->port = lp->portid; adc->target = tgt; adc->arrived = 1; xpt_async(AC_CONTRACT, fc->path, &ac); } break; case ISPASYNC_DEV_CHANGED: case ISPASYNC_DEV_STAYED: + { + int crn_reset_done; + + crn_reset_done = 0; va_start(ap, cmd); bus = va_arg(ap, int); lp = va_arg(ap, fcportdb_t *); va_end(ap); fc = ISP_FC_PC(isp, bus); tgt = FC_PORTDB_TGT(isp, bus, lp); isp_gen_role_str(buf, sizeof (buf), lp->new_prli_word3); if (cmd == ISPASYNC_DEV_CHANGED) isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->new_portid, lp->handle, buf, "changed"); else isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "stayed"); if (lp->is_target != ((FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) && (lp->new_prli_word3 & PRLI_WD3_TARGET_FUNCTION))) { lp->is_target = !lp->is_target; if (lp->is_target) { - if (cmd == ISPASYNC_DEV_CHANGED) + if (cmd == ISPASYNC_DEV_CHANGED) { isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1); + crn_reset_done = 1; + } isp_make_here(isp, lp, bus, tgt); } else { isp_make_gone(isp, lp, bus, tgt); - if (cmd == ISPASYNC_DEV_CHANGED) + if (cmd == ISPASYNC_DEV_CHANGED) { isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1); + crn_reset_done = 1; + } } } if (lp->is_initiator != ((FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) && (lp->new_prli_word3 & PRLI_WD3_INITIATOR_FUNCTION))) { lp->is_initiator = !lp->is_initiator; ac.contract_number = AC_CONTRACT_DEV_CHG; adc = (struct ac_device_changed *) ac.contract_data; adc->wwpn = lp->port_wwn; adc->port = lp->portid; adc->target = tgt; adc->arrived = lp->is_initiator; xpt_async(AC_CONTRACT, fc->path, &ac); } + + if ((lp->new_prli_word0 & PRLI_WD0_EST_IMAGE_PAIR) && + (crn_reset_done == 0)) + isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1); + break; + } case ISPASYNC_DEV_GONE: va_start(ap, cmd); bus = va_arg(ap, int); lp = va_arg(ap, fcportdb_t *); va_end(ap); fc = ISP_FC_PC(isp, bus); tgt = FC_PORTDB_TGT(isp, bus, lp); /* * If this has a virtual target or initiator set the isp_gdt * timer running on it to delay its departure. */ isp_gen_role_str(buf, sizeof (buf), lp->prli_word3); if (lp->is_target || lp->is_initiator) { lp->state = FC_PORTDB_STATE_ZOMBIE; lp->gone_timer = fc->gone_device_time; isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "gone zombie"); if (fc->ready && !callout_active(&fc->gdt)) { isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Starting Gone Device Timer with %u seconds time now %lu", bus, lp->gone_timer, (unsigned long)time_uptime); callout_reset(&fc->gdt, hz, isp_gdt, fc); } break; } isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "gone"); break; case ISPASYNC_CHANGE_NOTIFY: { char *msg; int evt, nphdl, nlstate, portid, reason; va_start(ap, cmd); bus = va_arg(ap, int); evt = va_arg(ap, int); if (evt == ISPASYNC_CHANGE_PDB) { nphdl = va_arg(ap, int); nlstate = va_arg(ap, int); reason = va_arg(ap, int); } else if (evt == ISPASYNC_CHANGE_SNS) { portid = va_arg(ap, int); } else { nphdl = NIL_HANDLE; nlstate = reason = 0; } va_end(ap); if (evt == ISPASYNC_CHANGE_PDB) { int tgt_set = 0; msg = "Port Database Changed"; isp_prt(isp, ISP_LOGINFO, "Chan %d %s (nphdl 0x%x state 0x%x reason 0x%x)", bus, msg, nphdl, nlstate, reason); /* * Port database syncs are not sufficient for * determining that logins or logouts are done on the * loop, but this information is directly available from * the reason code from the incoming mbox. We must reset * the fcp crn on these events according to FCP-4 */ switch (reason) { case PDB24XX_AE_IMPL_LOGO_1: case PDB24XX_AE_IMPL_LOGO_2: case PDB24XX_AE_IMPL_LOGO_3: case PDB24XX_AE_PLOGI_RCVD: case PDB24XX_AE_PRLI_RCVD: case PDB24XX_AE_PRLO_RCVD: case PDB24XX_AE_LOGO_RCVD: case PDB24XX_AE_PLOGI_DONE: case PDB24XX_AE_PRLI_DONE: /* * If the event is not global, twiddle tgt and * tgt_set to nominate only the target * associated with the nphdl. */ if (nphdl != PDB24XX_AE_GLOBAL) { /* Break if we don't yet have the pdb */ if (!isp_find_pdb_by_handle(isp, bus, nphdl, &lp)) break; tgt = FC_PORTDB_TGT(isp, bus, lp); tgt_set = 1; } isp_fcp_reset_crn(isp, bus, tgt, tgt_set); break; default: break; /* NOP */ } } else if (evt == ISPASYNC_CHANGE_SNS) { msg = "Name Server Database Changed"; isp_prt(isp, ISP_LOGINFO, "Chan %d %s (PortID 0x%06x)", bus, msg, portid); } else { msg = "Other Change Notify"; isp_prt(isp, ISP_LOGINFO, "Chan %d %s", bus, msg); } isp_loop_changed(isp, bus); break; } #ifdef ISP_TARGET_MODE case ISPASYNC_TARGET_NOTIFY: { isp_notify_t *notify; va_start(ap, cmd); notify = va_arg(ap, isp_notify_t *); va_end(ap); switch (notify->nt_ncode) { case NT_ABORT_TASK: case NT_ABORT_TASK_SET: case NT_CLEAR_ACA: case NT_CLEAR_TASK_SET: case NT_LUN_RESET: case NT_TARGET_RESET: case NT_QUERY_TASK_SET: case NT_QUERY_ASYNC_EVENT: /* * These are task management functions. */ isp_handle_platform_target_tmf(isp, notify); break; case NT_BUS_RESET: case NT_LIP_RESET: case NT_LINK_UP: case NT_LINK_DOWN: case NT_HBA_RESET: /* * No action need be taken here. */ break; case NT_GLOBAL_LOGOUT: case NT_LOGOUT: /* * This is device arrival/departure notification */ isp_handle_platform_target_notify_ack(isp, notify, 0); break; case NT_SRR: isp_handle_platform_srr(isp, notify); break; default: isp_prt(isp, ISP_LOGALL, "target notify code 0x%x", notify->nt_ncode); isp_handle_platform_target_notify_ack(isp, notify, 0); break; } break; } case ISPASYNC_TARGET_NOTIFY_ACK: { void *inot; va_start(ap, cmd); inot = va_arg(ap, void *); va_end(ap); if (isp_notify_ack(isp, inot)) { isp_tna_t *tp = malloc(sizeof (*tp), M_DEVBUF, M_NOWAIT); if (tp) { tp->isp = isp; memcpy(tp->data, inot, sizeof (tp->data)); tp->not = tp->data; callout_init_mtx(&tp->timer, &isp->isp_lock, 0); callout_reset(&tp->timer, 5, isp_refire_notify_ack, tp); } else { isp_prt(isp, ISP_LOGERR, "you lose- cannot allocate a notify refire"); } } break; } case ISPASYNC_TARGET_ACTION: { isphdr_t *hp; va_start(ap, cmd); hp = va_arg(ap, isphdr_t *); va_end(ap); switch (hp->rqs_entry_type) { case RQSTYPE_ATIO: isp_handle_platform_atio7(isp, (at7_entry_t *) hp); break; case RQSTYPE_ATIO2: isp_handle_platform_atio2(isp, (at2_entry_t *) hp); break; case RQSTYPE_CTIO7: case RQSTYPE_CTIO3: case RQSTYPE_CTIO2: case RQSTYPE_CTIO: isp_handle_platform_ctio(isp, hp); break; default: isp_prt(isp, ISP_LOGWARN, "%s: unhandled target action 0x%x", __func__, hp->rqs_entry_type); break; } break; } #endif case ISPASYNC_FW_CRASH: { uint16_t mbox1, mbox6; mbox1 = ISP_READ(isp, OUTMAILBOX1); if (IS_DUALBUS(isp)) { mbox6 = ISP_READ(isp, OUTMAILBOX6); } else { mbox6 = 0; } isp_prt(isp, ISP_LOGERR, "Internal Firmware Error on bus %d @ RISC Address 0x%x", mbox6, mbox1); #if 0 mbox1 = isp->isp_osinfo.mbox_sleep_ok; isp->isp_osinfo.mbox_sleep_ok = 0; isp_reinit(isp, 1); isp->isp_osinfo.mbox_sleep_ok = mbox1; isp_async(isp, ISPASYNC_FW_RESTARTED, NULL); #endif break; } default: isp_prt(isp, ISP_LOGERR, "unknown isp_async event %d", cmd); break; } } uint64_t isp_default_wwn(ispsoftc_t * isp, int chan, int isactive, int iswwnn) { uint64_t seed; struct isp_fc *fc = ISP_FC_PC(isp, chan); /* First try to use explicitly configured WWNs. */ seed = iswwnn ? fc->def_wwnn : fc->def_wwpn; if (seed) return (seed); /* Otherwise try to use WWNs from NVRAM. */ if (isactive) { seed = iswwnn ? FCPARAM(isp, chan)->isp_wwnn_nvram : FCPARAM(isp, chan)->isp_wwpn_nvram; if (seed) return (seed); } /* If still no WWNs, try to steal them from the first channel. */ if (chan > 0) { seed = iswwnn ? ISP_FC_PC(isp, 0)->def_wwnn : ISP_FC_PC(isp, 0)->def_wwpn; if (seed == 0) { seed = iswwnn ? FCPARAM(isp, 0)->isp_wwnn_nvram : FCPARAM(isp, 0)->isp_wwpn_nvram; } } /* If still nothing -- improvise. */ if (seed == 0) { seed = 0x400000007F000000ull + device_get_unit(isp->isp_dev); if (!iswwnn) seed ^= 0x0100000000000000ULL; } /* For additional channels we have to improvise even more. */ if (!iswwnn && chan > 0) { /* * We'll stick our channel number plus one first into bits * 57..59 and thence into bits 52..55 which allows for 8 bits * of channel which is enough for our maximum of 255 channels. */ seed ^= 0x0100000000000000ULL; seed ^= ((uint64_t) (chan + 1) & 0xf) << 56; seed ^= ((uint64_t) ((chan + 1) >> 4) & 0xf) << 52; } return (seed); } void isp_prt(ispsoftc_t *isp, int level, const char *fmt, ...) { int loc; char lbuf[200]; va_list ap; if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) { return; } snprintf(lbuf, sizeof (lbuf), "%s: ", device_get_nameunit(isp->isp_dev)); loc = strlen(lbuf); va_start(ap, fmt); vsnprintf(&lbuf[loc], sizeof (lbuf) - loc - 1, fmt, ap); va_end(ap); printf("%s\n", lbuf); } void isp_xs_prt(ispsoftc_t *isp, XS_T *xs, int level, const char *fmt, ...) { va_list ap; if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) { return; } xpt_print_path(xs->ccb_h.path); va_start(ap, fmt); vprintf(fmt, ap); va_end(ap); printf("\n"); } uint64_t isp_nanotime_sub(struct timespec *b, struct timespec *a) { uint64_t elapsed; struct timespec x; timespecsub(b, a, &x); elapsed = GET_NANOSEC(&x); if (elapsed == 0) elapsed++; return (elapsed); } int isp_mbox_acquire(ispsoftc_t *isp) { if (isp->isp_osinfo.mboxbsy) { return (1); } else { isp->isp_osinfo.mboxcmd_done = 0; isp->isp_osinfo.mboxbsy = 1; return (0); } } void isp_mbox_wait_complete(ispsoftc_t *isp, mbreg_t *mbp) { u_int t, to; to = (mbp->timeout == 0) ? MBCMD_DEFAULT_TIMEOUT : mbp->timeout; if (isp->isp_osinfo.mbox_sleep_ok) { isp->isp_osinfo.mbox_sleep_ok = 0; isp->isp_osinfo.mbox_sleeping = 1; msleep_sbt(&isp->isp_osinfo.mboxcmd_done, &isp->isp_lock, PRIBIO, "ispmbx_sleep", to * SBT_1US, 0, 0); isp->isp_osinfo.mbox_sleep_ok = 1; isp->isp_osinfo.mbox_sleeping = 0; } else { for (t = 0; t < to; t += 100) { if (isp->isp_osinfo.mboxcmd_done) break; ISP_RUN_ISR(isp); if (isp->isp_osinfo.mboxcmd_done) break; ISP_DELAY(100); } } if (isp->isp_osinfo.mboxcmd_done == 0) { isp_prt(isp, ISP_LOGWARN, "%s Mailbox Command (0x%x) Timeout (%uus) (%s:%d)", isp->isp_osinfo.mbox_sleep_ok? "Interrupting" : "Polled", isp->isp_lastmbxcmd, to, mbp->func, mbp->lineno); mbp->param[0] = MBOX_TIMEOUT; isp->isp_osinfo.mboxcmd_done = 1; } } void isp_mbox_notify_done(ispsoftc_t *isp) { isp->isp_osinfo.mboxcmd_done = 1; if (isp->isp_osinfo.mbox_sleeping) wakeup(&isp->isp_osinfo.mboxcmd_done); } void isp_mbox_release(ispsoftc_t *isp) { isp->isp_osinfo.mboxbsy = 0; } int isp_fc_scratch_acquire(ispsoftc_t *isp, int chan) { int ret = 0; if (isp->isp_osinfo.pc.fc[chan].fcbsy) { ret = -1; } else { isp->isp_osinfo.pc.fc[chan].fcbsy = 1; } return (ret); } void isp_platform_intr(void *arg) { ispsoftc_t *isp = arg; ISP_LOCK(isp); ISP_RUN_ISR(isp); ISP_UNLOCK(isp); } void isp_platform_intr_resp(void *arg) { ispsoftc_t *isp = arg; ISP_LOCK(isp); isp_intr_respq(isp); ISP_UNLOCK(isp); /* We have handshake enabled, so explicitly complete interrupt */ ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT); } void isp_platform_intr_atio(void *arg) { ispsoftc_t *isp = arg; ISP_LOCK(isp); #ifdef ISP_TARGET_MODE isp_intr_atioq(isp); #endif ISP_UNLOCK(isp); /* We have handshake enabled, so explicitly complete interrupt */ ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT); } void isp_common_dmateardown(ispsoftc_t *isp, struct ccb_scsiio *csio, uint32_t hdl) { if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, BUS_DMASYNC_POSTREAD); } else { bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, BUS_DMASYNC_POSTWRITE); } bus_dmamap_unload(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap); } /* * Reset the command reference number for all LUNs on a specific target * (needed when a target arrives again) or for all targets on a port * (needed for events like a LIP). */ void isp_fcp_reset_crn(ispsoftc_t *isp, int chan, uint32_t tgt, int tgt_set) { struct isp_fc *fc = ISP_FC_PC(isp, chan); struct isp_nexus *nxp; int i; if (tgt_set == 0) isp_prt(isp, ISP_LOGDEBUG0, "Chan %d resetting CRN on all targets", chan); else isp_prt(isp, ISP_LOGDEBUG0, "Chan %d resetting CRN on target %u", chan, tgt); for (i = 0; i < NEXUS_HASH_WIDTH; i++) { for (nxp = fc->nexus_hash[i]; nxp != NULL; nxp = nxp->next) { if (tgt_set == 0 || tgt == nxp->tgt) nxp->crnseed = 0; } } } int isp_fcp_next_crn(ispsoftc_t *isp, uint8_t *crnp, XS_T *cmd) { lun_id_t lun; uint32_t chan, tgt; struct isp_fc *fc; struct isp_nexus *nxp; int idx; if (IS_2100(isp)) return (0); chan = XS_CHANNEL(cmd); tgt = XS_TGT(cmd); lun = XS_LUN(cmd); fc = &isp->isp_osinfo.pc.fc[chan]; idx = NEXUS_HASH(tgt, lun); nxp = fc->nexus_hash[idx]; while (nxp) { if (nxp->tgt == tgt && nxp->lun == lun) break; nxp = nxp->next; } if (nxp == NULL) { nxp = fc->nexus_free_list; if (nxp == NULL) { nxp = malloc(sizeof (struct isp_nexus), M_DEVBUF, M_ZERO|M_NOWAIT); if (nxp == NULL) { return (-1); } } else { fc->nexus_free_list = nxp->next; } nxp->tgt = tgt; nxp->lun = lun; nxp->next = fc->nexus_hash[idx]; fc->nexus_hash[idx] = nxp; } if (nxp->crnseed == 0) nxp->crnseed = 1; *crnp = nxp->crnseed++; return (0); } /* * We enter with the lock held */ void isp_timer(void *arg) { ispsoftc_t *isp = arg; #ifdef ISP_TARGET_MODE isp_tmcmd_restart(isp); #endif callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp); } isp_ecmd_t * isp_get_ecmd(ispsoftc_t *isp) { isp_ecmd_t *ecmd = isp->isp_osinfo.ecmd_free; if (ecmd) { isp->isp_osinfo.ecmd_free = ecmd->next; } return (ecmd); } void isp_put_ecmd(ispsoftc_t *isp, isp_ecmd_t *ecmd) { ecmd->next = isp->isp_osinfo.ecmd_free; isp->isp_osinfo.ecmd_free = ecmd; } Index: projects/import-googletest-1.8.1/sys/dev/isp/isp_stds.h =================================================================== --- projects/import-googletest-1.8.1/sys/dev/isp/isp_stds.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/isp/isp_stds.h (revision 345026) @@ -1,332 +1,340 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1997-2009 by Matthew Jacob * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Structures that derive directly from public standards. */ #ifndef _ISP_STDS_H #define _ISP_STDS_H /* * FC Frame Header * * Source: dpANS-X3.xxx-199x, section 18 (AKA FC-PH-2) * */ typedef struct { uint8_t r_ctl; uint8_t d_id[3]; uint8_t cs_ctl; uint8_t s_id[3]; uint8_t type; uint8_t f_ctl[3]; uint8_t seq_id; uint8_t df_ctl; uint16_t seq_cnt; uint16_t ox_id; uint16_t rx_id; uint32_t parameter; } fc_hdr_t; /* * FCP_CMND_IU Payload * * Source: NICTS T10, Project 1144D, Revision 07a, Section 9 (AKA fcp2-r07a) * * Notes: * When additional cdb length is defined in fcp_cmnd_alen_datadir, * bits 2..7, the actual cdb length is 16 + ((fcp_cmnd_alen_datadir>>2)*4), * with the datalength following in MSB format just after. */ typedef struct { uint8_t fcp_cmnd_lun[8]; uint8_t fcp_cmnd_crn; uint8_t fcp_cmnd_task_attribute; uint8_t fcp_cmnd_task_management; uint8_t fcp_cmnd_alen_datadir; union { struct { uint8_t fcp_cmnd_cdb[16]; uint32_t fcp_cmnd_dl; } sf; struct { uint8_t fcp_cmnd_cdb[1]; } lf; } cdb_dl; } fcp_cmnd_iu_t; #define FCP_CMND_TASK_ATTR_SIMPLE 0x00 #define FCP_CMND_TASK_ATTR_HEAD 0x01 #define FCP_CMND_TASK_ATTR_ORDERED 0x02 #define FCP_CMND_TASK_ATTR_ACA 0x04 #define FCP_CMND_TASK_ATTR_UNTAGGED 0x05 #define FCP_CMND_TASK_ATTR_MASK 0x07 #define FCP_CMND_ADDTL_CDBLEN_SHIFT 2 #define FCP_CMND_DATA_WRITE 0x01 #define FCP_CMND_DATA_READ 0x02 #define FCP_CMND_DATA_DIR_MASK 0x03 #define FCP_CMND_TMF_CLEAR_ACA 0x40 #define FCP_CMND_TMF_TGT_RESET 0x20 #define FCP_CMND_TMF_LUN_RESET 0x10 #define FCP_CMND_TMF_QUERY_ASYNC_EVENT 0x08 #define FCP_CMND_TMF_CLEAR_TASK_SET 0x04 #define FCP_CMND_TMF_ABORT_TASK_SET 0x02 #define FCP_CMND_TMF_QUERY_TASK_SET 0x01 /* * Basic CT IU Header * * Source: X3.288-199x Generic Services 2 Rev 5.3 (FC-GS-2) Section 4.3.1 */ typedef struct { uint8_t ct_revision; uint8_t ct_in_id[3]; uint8_t ct_fcs_type; uint8_t ct_fcs_subtype; uint8_t ct_options; uint8_t ct_reserved0; uint16_t ct_cmd_resp; uint16_t ct_bcnt_resid; uint8_t ct_reserved1; uint8_t ct_reason; uint8_t ct_explanation; uint8_t ct_vunique; } ct_hdr_t; #define CT_REVISION 1 #define CT_FC_TYPE_FC 0xFC #define CT_FC_SUBTYPE_NS 0x02 /* * RFT_ID Requet CT_IU * * Source: NCITS xxx-200x Generic Services- 5 Rev 8.5 Section 5.2.5.30 */ typedef struct { ct_hdr_t rftid_hdr; uint8_t rftid_reserved; uint8_t rftid_portid[3]; uint32_t rftid_fc4types[8]; } rft_id_t; /* * RSPN_ID Requet CT_IU * * Source: INCITS 463-2010 Generic Services 6 Section 5.2.5.32 */ typedef struct { ct_hdr_t rspnid_hdr; uint8_t rspnid_reserved; uint8_t rspnid_portid[3]; uint8_t rspnid_length; uint8_t rspnid_name[0]; } rspn_id_t; /* * RFF_ID Requet CT_IU * * Source: INCITS 463-2010 Generic Services 6 Section 5.2.5.34 */ typedef struct { ct_hdr_t rffid_hdr; uint8_t rffid_reserved; uint8_t rffid_portid[3]; uint16_t rffid_reserved2; uint8_t rffid_fc4features; uint8_t rffid_fc4type; } rff_id_t; /* * RSNN_NN Requet CT_IU * * Source: INCITS 463-2010 Generic Services 6 Section 5.2.5.35 */ typedef struct { ct_hdr_t rsnnnn_hdr; uint8_t rsnnnn_nodename[8]; uint8_t rsnnnn_length; uint8_t rsnnnn_name[0]; } rsnn_nn_t; /* * FCP Response IU and bits of interest * Source: NCITS T10, Project 1828D, Revision 02b (aka FCP4r02b) */ typedef struct { uint8_t fcp_rsp_reserved[8]; uint16_t fcp_rsp_status_qualifier; /* SAM-5 Status Qualifier */ uint8_t fcp_rsp_bits; uint8_t fcp_rsp_scsi_status; /* SAM-5 SCSI Status Byte */ uint32_t fcp_rsp_resid; uint32_t fcp_rsp_snslen; uint32_t fcp_rsp_rsplen; /* * In the bytes that follow, it's going to be * FCP RESPONSE INFO (max 8 bytes, possibly 0) * FCP SENSE INFO (if any) * FCP BIDIRECTIONAL READ RESID (if any) */ uint8_t fcp_rsp_extra[0]; } fcp_rsp_iu_t; #define MIN_FCP_RESPONSE_SIZE 24 #define FCP_BIDIR_RSP 0x80 /* Bi-Directional response */ #define FCP_BIDIR_RESID_UNDERFLOW 0x40 #define FCP_BIDIR_RESID_OVERFLOW 0x20 #define FCP_CONF_REQ 0x10 #define FCP_RESID_UNDERFLOW 0x08 #define FCP_RESID_OVERFLOW 0x04 #define FCP_SNSLEN_VALID 0x02 #define FCP_RSPLEN_VALID 0x01 #define FCP_MAX_RSPLEN 0x08 /* * FCP Response Code Definitions * Source: NCITS T10, Project 1144D, Revision 08 (aka FCP2r08) */ #define FCP_RSPNS_CODE_OFFSET 3 #define FCP_RSPNS_TMF_DONE 0 #define FCP_RSPNS_DLBRSTX 1 #define FCP_RSPNS_BADCMND 2 #define FCP_RSPNS_EROFS 3 #define FCP_RSPNS_TMF_REJECT 4 #define FCP_RSPNS_TMF_FAILED 5 #define FCP_RSPNS_TMF_SUCCEEDED 8 #define FCP_RSPNS_TMF_INCORRECT_LUN 9 /* * R_CTL field definitions * * Bits 31-28 are ROUTING * Bits 27-24 are INFORMATION * * These are nibble values, not bits */ #define R_CTL_ROUTE_DATA 0x00 #define R_CTL_ROUTE_ELS 0x02 #define R_CTL_ROUTE_FC4_LINK 0x03 #define R_CTL_ROUTE_VDATA 0x04 #define R_CTL_ROUTE_EXENDED 0x05 #define R_CTL_ROUTE_BASIC 0x08 #define R_CTL_ROUTE_LINK 0x0c #define R_CTL_ROUTE_EXT_ROUTING 0x0f #define R_CTL_INFO_UNCATEGORIZED 0x00 #define R_CTL_INFO_SOLICITED_DATA 0x01 #define R_CTL_INFO_UNSOLICITED_CONTROL 0x02 #define R_CTL_INFO_SOLICITED_CONTROL 0x03 #define R_CTL_INFO_UNSOLICITED_DATA 0x04 #define R_CTL_INFO_DATA_DESCRIPTOR 0x05 #define R_CTL_INFO_UNSOLICITED_COMMAND 0x06 #define R_CTL_INFO_COMMAND_STATUS 0x07 #define MAKE_RCTL(a, b) (((a) << 4) | (b)) /* unconverted miscellany */ /* * Basic FC Link Service defines */ /* #define ABTS MAKE_RCTL(R_CTL_ROUTE_BASIC, R_CTL_INFO_SOLICITED_DATA) */ #define BA_ACC MAKE_RCTL(R_CTL_ROUTE_BASIC, R_CTL_INFO_UNSOLICITED_DATA) /* of ABORT */ #define BA_RJT MAKE_RCTL(R_CTL_ROUTE_BASIC, R_CTL_INFO_DATA_DESCRIPTOR) /* of ABORT */ /* * Link Service Accept/Reject */ #define LS_ACC 0x8002 #define LS_RJT 0x8001 /* * FC ELS Codes- bits 31-24 of the first payload word of an ELS frame. */ #define PLOGI 0x03 #define FLOGI 0x04 #define LOGO 0x05 #define ABTX 0x06 #define PRLI 0x20 #define PRLO 0x21 #define SCN 0x22 #define TPRLO 0x24 #define PDISC 0x50 #define ADISC 0x52 #define RNC 0x53 /* + * PRLI Word 0 definitions + * FPC4-r02b January, 2011 + */ +#define PRLI_WD0_TYPE_MASK 0xff000000 +#define PRLI_WD0_TC_EXT_MASK 0x00ff0000 +#define PRLI_WD0_EST_IMAGE_PAIR (1 << 13) + +/* * PRLI Word 3 definitions * FPC4-r02b January, 2011 */ #define PRLI_WD3_ENHANCED_DISCOVERY (1 << 11) #define PRLI_WD3_REC_SUPPORT (1 << 10) #define PRLI_WD3_TASK_RETRY_IDENTIFICATION_REQUESTED (1 << 9) #define PRLI_WD3_RETRY (1 << 8) #define PRLI_WD3_CONFIRMED_COMPLETION_ALLOWED (1 << 7) #define PRLI_WD3_DATA_OVERLAY_ALLOWED (1 << 6) #define PRLI_WD3_INITIATOR_FUNCTION (1 << 5) #define PRLI_WD3_TARGET_FUNCTION (1 << 4) #define PRLI_WD3_READ_FCP_XFER_RDY_DISABLED (1 << 1) /* definitely supposed to be set */ #define PRLI_WD3_WRITE_FCP_XFER_RDY_DISABLED (1 << 0) /* * FC4 defines */ #define FC4_IP 5 /* ISO/EEC 8802-2 LLC/SNAP */ #define FC4_SCSI 8 /* SCSI-3 via Fibre Channel Protocol (FCP) */ #define FC4_FC_SVC 0x20 /* Fibre Channel Services */ #ifndef MSG_ABORT #define MSG_ABORT 0x06 #endif #ifndef MSG_BUS_DEV_RESET #define MSG_BUS_DEV_RESET 0x0c #endif #ifndef MSG_ABORT_TAG #define MSG_ABORT_TAG 0x0d #endif #ifndef MSG_CLEAR_QUEUE #define MSG_CLEAR_QUEUE 0x0e #endif #ifndef MSG_REL_RECOVERY #define MSG_REL_RECOVERY 0x10 #endif #ifndef MSG_TERM_IO_PROC #define MSG_TERM_IO_PROC 0x11 #endif #ifndef MSG_LUN_RESET #define MSG_LUN_RESET 0x17 #endif #endif /* _ISP_STDS_H */ Index: projects/import-googletest-1.8.1/sys/dev/isp/ispmbox.h =================================================================== --- projects/import-googletest-1.8.1/sys/dev/isp/ispmbox.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/isp/ispmbox.h (revision 345026) @@ -1,2717 +1,2718 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2018 Alexander Motin * Copyright (c) 1997-2009 by Matthew Jacob * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Mailbox and Queue Entry Definitions for for Qlogic ISP SCSI adapters. */ #ifndef _ISPMBOX_H #define _ISPMBOX_H /* * Mailbox Command Opcodes */ #define MBOX_NO_OP 0x0000 #define MBOX_LOAD_RAM 0x0001 #define MBOX_EXEC_FIRMWARE 0x0002 #define MBOX_DUMP_RAM 0x0003 #define MBOX_WRITE_RAM_WORD 0x0004 #define MBOX_READ_RAM_WORD 0x0005 #define MBOX_MAILBOX_REG_TEST 0x0006 #define MBOX_VERIFY_CHECKSUM 0x0007 #define MBOX_ABOUT_FIRMWARE 0x0008 #define MBOX_LOAD_RISC_RAM_2100 0x0009 /* a */ #define MBOX_LOAD_RISC_RAM 0x000b #define MBOX_DUMP_RISC_RAM 0x000c #define MBOX_WRITE_RAM_WORD_EXTENDED 0x000d #define MBOX_CHECK_FIRMWARE 0x000e #define MBOX_READ_RAM_WORD_EXTENDED 0x000f #define MBOX_INIT_REQ_QUEUE 0x0010 #define MBOX_INIT_RES_QUEUE 0x0011 #define MBOX_EXECUTE_IOCB 0x0012 #define MBOX_WAKE_UP 0x0013 #define MBOX_STOP_FIRMWARE 0x0014 #define MBOX_ABORT 0x0015 #define MBOX_ABORT_DEVICE 0x0016 #define MBOX_ABORT_TARGET 0x0017 #define MBOX_BUS_RESET 0x0018 #define MBOX_STOP_QUEUE 0x0019 #define MBOX_START_QUEUE 0x001a #define MBOX_SINGLE_STEP_QUEUE 0x001b #define MBOX_ABORT_QUEUE 0x001c #define MBOX_GET_DEV_QUEUE_STATUS 0x001d /* 1e */ #define MBOX_GET_FIRMWARE_STATUS 0x001f #define MBOX_GET_INIT_SCSI_ID 0x0020 #define MBOX_GET_SELECT_TIMEOUT 0x0021 #define MBOX_GET_RETRY_COUNT 0x0022 #define MBOX_GET_TAG_AGE_LIMIT 0x0023 #define MBOX_GET_CLOCK_RATE 0x0024 #define MBOX_GET_ACT_NEG_STATE 0x0025 #define MBOX_GET_ASYNC_DATA_SETUP_TIME 0x0026 #define MBOX_GET_SBUS_PARAMS 0x0027 #define MBOX_GET_PCI_PARAMS MBOX_GET_SBUS_PARAMS #define MBOX_GET_TARGET_PARAMS 0x0028 #define MBOX_GET_DEV_QUEUE_PARAMS 0x0029 #define MBOX_GET_RESET_DELAY_PARAMS 0x002a /* 2b */ /* 2c */ /* 2d */ /* 2e */ /* 2f */ #define MBOX_SET_INIT_SCSI_ID 0x0030 #define MBOX_SET_SELECT_TIMEOUT 0x0031 #define MBOX_SET_RETRY_COUNT 0x0032 #define MBOX_SET_TAG_AGE_LIMIT 0x0033 #define MBOX_SET_CLOCK_RATE 0x0034 #define MBOX_SET_ACT_NEG_STATE 0x0035 #define MBOX_SET_ASYNC_DATA_SETUP_TIME 0x0036 #define MBOX_SET_SBUS_CONTROL_PARAMS 0x0037 #define MBOX_SET_PCI_PARAMETERS 0x0037 #define MBOX_SET_TARGET_PARAMS 0x0038 #define MBOX_SET_DEV_QUEUE_PARAMS 0x0039 #define MBOX_SET_RESET_DELAY_PARAMS 0x003a /* 3b */ /* 3c */ /* 3d */ /* 3e */ /* 3f */ #define MBOX_RETURN_BIOS_BLOCK_ADDR 0x0040 #define MBOX_WRITE_FOUR_RAM_WORDS 0x0041 #define MBOX_EXEC_BIOS_IOCB 0x0042 #define MBOX_SET_FW_FEATURES 0x004a #define MBOX_GET_FW_FEATURES 0x004b #define FW_FEATURE_FAST_POST 0x1 #define FW_FEATURE_LVD_NOTIFY 0x2 #define FW_FEATURE_RIO_32BIT 0x4 #define FW_FEATURE_RIO_16BIT 0x8 #define MBOX_INIT_REQ_QUEUE_A64 0x0052 #define MBOX_INIT_RES_QUEUE_A64 0x0053 #define MBOX_ENABLE_TARGET_MODE 0x0055 #define ENABLE_TARGET_FLAG 0x8000 #define ENABLE_TQING_FLAG 0x0004 #define ENABLE_MANDATORY_DISC 0x0002 #define MBOX_GET_TARGET_STATUS 0x0056 /* These are for the ISP2X00 FC cards */ #define MBOX_LOAD_FLASH_FIRMWARE 0x0003 #define MBOX_WRITE_FC_SERDES_REG 0x0003 /* FC only */ #define MBOX_READ_FC_SERDES_REG 0x0004 /* FC only */ #define MBOX_GET_IO_STATUS 0x0012 #define MBOX_SET_TRANSMIT_PARAMS 0x0019 #define MBOX_SET_PORT_PARAMS 0x001a #define MBOX_LOAD_OP_FW_PARAMS 0x001b #define MBOX_INIT_MULTIPLE_QUEUE 0x001f #define MBOX_GET_LOOP_ID 0x0020 /* for 24XX cards, outgoing mailbox 7 has these values for F or FL topologies */ #define ISP24XX_INORDER 0x0100 #define ISP24XX_NPIV_SAN 0x0400 #define ISP24XX_VSAN_SAN 0x1000 #define ISP24XX_FC_SP_SAN 0x2000 #define MBOX_GET_TIMEOUT_PARAMS 0x0022 #define MBOX_GET_FIRMWARE_OPTIONS 0x0028 #define MBOX_GENERATE_SYSTEM_ERROR 0x002a #define MBOX_WRITE_SFP 0x0030 #define MBOX_READ_SFP 0x0031 #define MBOX_SET_TIMEOUT_PARAMS 0x0032 #define MBOX_SET_FIRMWARE_OPTIONS 0x0038 #define MBOX_GET_SET_FC_LED_CONF 0x003b #define MBOX_RESTART_NIC_FIRMWARE 0x003d /* FCoE only */ #define MBOX_ACCESS_CONTROL 0x003e #define MBOX_LOOP_PORT_BYPASS 0x0040 /* FC only */ #define MBOX_LOOP_PORT_ENABLE 0x0041 /* FC only */ #define MBOX_GET_RESOURCE_COUNT 0x0042 #define MBOX_REQUEST_OFFLINE_MODE 0x0043 #define MBOX_DIAGNOSTIC_ECHO_TEST 0x0044 #define MBOX_DIAGNOSTIC_LOOPBACK 0x0045 #define MBOX_ENHANCED_GET_PDB 0x0047 #define MBOX_INIT_FIRMWARE_MULTI_ID 0x0048 /* 2400 only */ #define MBOX_GET_VP_DATABASE 0x0049 /* 2400 only */ #define MBOX_GET_VP_DATABASE_ENTRY 0x004a /* 2400 only */ #define MBOX_GET_FCF_LIST 0x0050 /* FCoE only */ #define MBOX_GET_DCBX_PARAMETERS 0x0051 /* FCoE only */ #define MBOX_HOST_MEMORY_COPY 0x0053 #define MBOX_EXEC_COMMAND_IOCB_A64 0x0054 #define MBOX_SEND_RNID 0x0057 #define MBOX_SET_PARAMETERS 0x0059 #define MBOX_GET_PARAMETERS 0x005a #define MBOX_DRIVER_HEARTBEAT 0x005B /* FC only */ #define MBOX_FW_HEARTBEAT 0x005C #define MBOX_GET_SET_DATA_RATE 0x005D /* >=23XX only */ #define MBGSD_GET_RATE 0 #define MBGSD_SET_RATE 1 #define MBGSD_SET_RATE_NOW 2 /* 24XX only */ #define MBGSD_1GB 0x00 #define MBGSD_2GB 0x01 #define MBGSD_AUTO 0x02 #define MBGSD_4GB 0x03 /* 24XX only */ #define MBGSD_8GB 0x04 /* 25XX only */ #define MBGSD_16GB 0x05 /* 26XX only */ #define MBGSD_32GB 0x06 /* 27XX only */ #define MBGSD_10GB 0x13 /* 26XX only */ #define MBOX_SEND_RNFT 0x005e #define MBOX_INIT_FIRMWARE 0x0060 #define MBOX_GET_INIT_CONTROL_BLOCK 0x0061 #define MBOX_INIT_LIP 0x0062 #define MBOX_GET_FC_AL_POSITION_MAP 0x0063 #define MBOX_GET_PORT_DB 0x0064 #define MBOX_CLEAR_ACA 0x0065 #define MBOX_TARGET_RESET 0x0066 #define MBOX_CLEAR_TASK_SET 0x0067 #define MBOX_ABORT_TASK_SET 0x0068 #define MBOX_GET_FW_STATE 0x0069 #define MBOX_GET_PORT_NAME 0x006A #define MBOX_GET_LINK_STATUS 0x006B #define MBOX_INIT_LIP_RESET 0x006C #define MBOX_GET_LINK_STAT_PR_DATA_CNT 0x006D #define MBOX_SEND_SNS 0x006E #define MBOX_FABRIC_LOGIN 0x006F #define MBOX_SEND_CHANGE_REQUEST 0x0070 #define MBOX_FABRIC_LOGOUT 0x0071 #define MBOX_INIT_LIP_LOGIN 0x0072 #define MBOX_GET_PORT_NODE_NAME_LIST 0x0075 #define MBOX_SET_VENDOR_ID 0x0076 #define MBOX_GET_XGMAC_STATS 0x007a #define MBOX_GET_ID_LIST 0x007C #define MBOX_SEND_LFA 0x007d #define MBOX_LUN_RESET 0x007E #define ISP2100_SET_PCI_PARAM 0x00ff #define MBOX_BUSY 0x04 /* * Mailbox Command Complete Status Codes */ #define MBOX_COMMAND_COMPLETE 0x4000 #define MBOX_INVALID_COMMAND 0x4001 #define MBOX_HOST_INTERFACE_ERROR 0x4002 #define MBOX_TEST_FAILED 0x4003 #define MBOX_COMMAND_ERROR 0x4005 #define MBOX_COMMAND_PARAM_ERROR 0x4006 #define MBOX_PORT_ID_USED 0x4007 #define MBOX_LOOP_ID_USED 0x4008 #define MBOX_ALL_IDS_USED 0x4009 #define MBOX_NOT_LOGGED_IN 0x400A #define MBOX_LINK_DOWN_ERROR 0x400B #define MBOX_LOOPBACK_ERROR 0x400C #define MBOX_CHECKSUM_ERROR 0x4010 #define MBOX_INVALID_PRODUCT_KEY 0x4020 /* pseudo mailbox completion codes */ #define MBOX_REGS_BUSY 0x6000 /* registers in use */ #define MBOX_TIMEOUT 0x6001 /* command timed out */ #define MBLOGALL 0xffffffff #define MBLOGNONE 0x00000000 #define MBLOGMASK(x) (1 << (((x) - 1) & 0x1f)) /* * Asynchronous event status codes */ #define ASYNC_BUS_RESET 0x8001 #define ASYNC_SYSTEM_ERROR 0x8002 #define ASYNC_RQS_XFER_ERR 0x8003 #define ASYNC_RSP_XFER_ERR 0x8004 #define ASYNC_QWAKEUP 0x8005 #define ASYNC_TIMEOUT_RESET 0x8006 #define ASYNC_DEVICE_RESET 0x8007 #define ASYNC_EXTMSG_UNDERRUN 0x800A #define ASYNC_SCAM_INT 0x800B #define ASYNC_HUNG_SCSI 0x800C #define ASYNC_KILLED_BUS 0x800D #define ASYNC_BUS_TRANSIT 0x800E /* LVD -> HVD, eg. */ #define ASYNC_LIP_OCCURRED 0x8010 /* FC only */ #define ASYNC_LOOP_UP 0x8011 #define ASYNC_LOOP_DOWN 0x8012 #define ASYNC_LOOP_RESET 0x8013 /* FC only */ #define ASYNC_PDB_CHANGED 0x8014 #define ASYNC_CHANGE_NOTIFY 0x8015 #define ASYNC_LIP_NOS_OLS_RECV 0x8016 /* FC only */ #define ASYNC_LIP_ERROR 0x8017 /* FC only */ #define ASYNC_AUTO_PLOGI_RJT 0x8018 #define ASYNC_SECURITY_UPDATE 0x801B #define ASYNC_CMD_CMPLT 0x8020 #define ASYNC_CTIO_DONE 0x8021 #define ASYNC_RIO32_1 0x8021 #define ASYNC_RIO32_2 0x8022 #define ASYNC_IP_XMIT_DONE 0x8022 #define ASYNC_IP_RECV_DONE 0x8023 #define ASYNC_IP_BROADCAST 0x8024 #define ASYNC_IP_RCVQ_LOW 0x8025 #define ASYNC_IP_RCVQ_EMPTY 0x8026 #define ASYNC_IP_RECV_DONE_ALIGNED 0x8027 #define ASYNC_ERR_LOGGING_DISABLED 0x8029 #define ASYNC_PTPMODE 0x8030 /* FC only */ #define ASYNC_RIO16_1 0x8031 #define ASYNC_RIO16_2 0x8032 #define ASYNC_RIO16_3 0x8033 #define ASYNC_RIO16_4 0x8034 #define ASYNC_RIO16_5 0x8035 #define ASYNC_CONNMODE 0x8036 #define ISP_CONN_LOOP 1 #define ISP_CONN_PTP 2 #define ISP_CONN_BADLIP 3 #define ISP_CONN_FATAL 4 #define ISP_CONN_LOOPBACK 5 #define ASYNC_P2P_INIT_ERR 0x8037 #define ASYNC_RIOZIO_STALL 0x8040 /* there's a RIO/ZIO entry that hasn't been serviced */ #define ASYNC_RIO32_2_2200 0x8042 /* same as ASYNC_RIO32_2, but for 2100/2200 */ #define ASYNC_RCV_ERR 0x8048 /* * 2.01.31 2200 Only. Need Bit 13 in Mailbox 1 for Set Firmware Options * mailbox command to enable this. */ #define ASYNC_QFULL_SENT 0x8049 #define ASYNC_RJT_SENT 0x8049 /* 24XX only */ #define ASYNC_SEL_CLASS2_P_RJT_SENT 0x804f #define ASYNC_FW_RESTART_COMPLETE 0x8060 #define ASYNC_TEMPERATURE_ALERT 0x8070 #define ASYNC_INTER_DRIVER_COMP 0x8100 /* FCoE only */ #define ASYNC_INTER_DRIVER_NOTIFY 0x8101 /* FCoE only */ #define ASYNC_INTER_DRIVER_TIME_EXT 0x8102 /* FCoE only */ #define ASYNC_TRANSCEIVER_INSERTION 0x8130 #define ASYNC_TRANSCEIVER_REMOVAL 0x8131 #define ASYNC_NIC_FW_STATE_CHANGE 0x8200 /* FCoE only */ #define ASYNC_AUTOLOAD_FW_COMPLETE 0x8400 #define ASYNC_AUTOLOAD_FW_FAILURE 0x8401 /* * Firmware Options. There are a lot of them. * * IFCOPTN - ISP Fibre Channel Option Word N */ #define IFCOPT1_EQFQASYNC (1 << 13) /* enable QFULL notification */ #define IFCOPT1_EAABSRCVD (1 << 12) #define IFCOPT1_RJTASYNC (1 << 11) /* enable 8018 notification */ #define IFCOPT1_ENAPURE (1 << 10) #define IFCOPT1_ENA8017 (1 << 7) #define IFCOPT1_DISGPIO67 (1 << 6) #define IFCOPT1_LIPLOSSIMM (1 << 5) #define IFCOPT1_DISF7SWTCH (1 << 4) #define IFCOPT1_CTIO_RETRY (1 << 3) #define IFCOPT1_LIPASYNC (1 << 1) #define IFCOPT1_LIPF8 (1 << 0) #define IFCOPT2_LOOPBACK (1 << 1) #define IFCOPT2_ATIO3_ONLY (1 << 0) #define IFCOPT3_NOPRLI (1 << 4) /* disable automatic sending of PRLI on local loops */ #define IFCOPT3_RNDASYNC (1 << 1) /* * All IOCB Queue entries are this size */ #define QENTRY_LEN 64 /* * Command Structure Definitions */ typedef struct { uint32_t ds_base; uint32_t ds_count; } ispds_t; typedef struct { uint32_t ds_base; uint32_t ds_basehi; uint32_t ds_count; } ispds64_t; #define DSTYPE_32BIT 0 #define DSTYPE_64BIT 1 typedef struct { uint16_t ds_type; /* 0-> ispds_t, 1-> ispds64_t */ uint32_t ds_segment; /* unused */ uint32_t ds_base; /* 32 bit address of DSD list */ } ispdslist_t; typedef struct { uint8_t rqs_entry_type; uint8_t rqs_entry_count; uint8_t rqs_seqno; uint8_t rqs_flags; } isphdr_t; /* RQS Flag definitions */ #define RQSFLAG_CONTINUATION 0x01 #define RQSFLAG_FULL 0x02 #define RQSFLAG_BADHEADER 0x04 #define RQSFLAG_BADPACKET 0x08 #define RQSFLAG_BADCOUNT 0x10 #define RQSFLAG_BADORDER 0x20 #define RQSFLAG_MASK 0x3f /* RQS entry_type definitions */ #define RQSTYPE_REQUEST 0x01 #define RQSTYPE_DATASEG 0x02 #define RQSTYPE_RESPONSE 0x03 #define RQSTYPE_MARKER 0x04 #define RQSTYPE_CMDONLY 0x05 #define RQSTYPE_ATIO 0x06 /* Target Mode */ #define RQSTYPE_CTIO 0x07 /* Target Mode */ #define RQSTYPE_SCAM 0x08 #define RQSTYPE_A64 0x09 #define RQSTYPE_A64_CONT 0x0a #define RQSTYPE_ENABLE_LUN 0x0b /* Target Mode */ #define RQSTYPE_MODIFY_LUN 0x0c /* Target Mode */ #define RQSTYPE_NOTIFY 0x0d /* Target Mode */ #define RQSTYPE_NOTIFY_ACK 0x0e /* Target Mode */ #define RQSTYPE_CTIO1 0x0f /* Target Mode */ #define RQSTYPE_STATUS_CONT 0x10 #define RQSTYPE_T2RQS 0x11 #define RQSTYPE_CTIO7 0x12 #define RQSTYPE_IP_XMIT 0x13 #define RQSTYPE_TSK_MGMT 0x14 #define RQSTYPE_T4RQS 0x15 #define RQSTYPE_ATIO2 0x16 /* Target Mode */ #define RQSTYPE_CTIO2 0x17 /* Target Mode */ #define RQSTYPE_T7RQS 0x18 #define RQSTYPE_T3RQS 0x19 #define RQSTYPE_IP_XMIT_64 0x1b #define RQSTYPE_CTIO4 0x1e /* Target Mode */ #define RQSTYPE_CTIO3 0x1f /* Target Mode */ #define RQSTYPE_RIO1 0x21 #define RQSTYPE_RIO2 0x22 #define RQSTYPE_IP_RECV 0x23 #define RQSTYPE_IP_RECV_CONT 0x24 #define RQSTYPE_CT_PASSTHRU 0x29 #define RQSTYPE_MS_PASSTHRU 0x29 #define RQSTYPE_VP_CTRL 0x30 /* 24XX only */ #define RQSTYPE_VP_MODIFY 0x31 /* 24XX only */ #define RQSTYPE_RPT_ID_ACQ 0x32 /* 24XX only */ #define RQSTYPE_ABORT_IO 0x33 #define RQSTYPE_T6RQS 0x48 #define RQSTYPE_LOGIN 0x52 #define RQSTYPE_ABTS_RCVD 0x54 /* 24XX only */ #define RQSTYPE_ABTS_RSP 0x55 /* 24XX only */ #define ISP_RQDSEG 4 typedef struct { isphdr_t req_header; uint32_t req_handle; uint8_t req_lun_trn; uint8_t req_target; uint16_t req_cdblen; uint16_t req_flags; uint16_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[12]; ispds_t req_dataseg[ISP_RQDSEG]; } ispreq_t; #define ISP_RQDSEG_A64 2 typedef struct { isphdr_t mrk_header; uint32_t mrk_handle; uint8_t mrk_reserved0; uint8_t mrk_target; uint16_t mrk_modifier; uint16_t mrk_flags; uint16_t mrk_lun; uint8_t mrk_reserved1[48]; } isp_marker_t; typedef struct { isphdr_t mrk_header; uint32_t mrk_handle; uint16_t mrk_nphdl; uint8_t mrk_modifier; uint8_t mrk_reserved0; uint8_t mrk_reserved1; uint8_t mrk_vphdl; uint16_t mrk_reserved2; uint8_t mrk_lun[8]; uint8_t mrk_reserved3[40]; } isp_marker_24xx_t; #define SYNC_DEVICE 0 #define SYNC_TARGET 1 #define SYNC_ALL 2 #define SYNC_LIP 3 #define ISP_RQDSEG_T2 3 typedef struct { isphdr_t req_header; uint32_t req_handle; uint8_t req_lun_trn; uint8_t req_target; uint16_t req_scclun; uint16_t req_flags; uint8_t req_crn; uint8_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[16]; uint32_t req_totalcnt; ispds_t req_dataseg[ISP_RQDSEG_T2]; } ispreqt2_t; typedef struct { isphdr_t req_header; uint32_t req_handle; uint16_t req_target; uint16_t req_scclun; uint16_t req_flags; uint8_t req_crn; uint8_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[16]; uint32_t req_totalcnt; ispds_t req_dataseg[ISP_RQDSEG_T2]; } ispreqt2e_t; #define ISP_RQDSEG_T3 2 typedef struct { isphdr_t req_header; uint32_t req_handle; uint8_t req_lun_trn; uint8_t req_target; uint16_t req_scclun; uint16_t req_flags; uint8_t req_crn; uint8_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[16]; uint32_t req_totalcnt; ispds64_t req_dataseg[ISP_RQDSEG_T3]; } ispreqt3_t; #define ispreq64_t ispreqt3_t /* same as.... */ typedef struct { isphdr_t req_header; uint32_t req_handle; uint16_t req_target; uint16_t req_scclun; uint16_t req_flags; uint8_t req_crn; uint8_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[16]; uint32_t req_totalcnt; ispds64_t req_dataseg[ISP_RQDSEG_T3]; } ispreqt3e_t; /* req_flag values */ #define REQFLAG_NODISCON 0x0001 #define REQFLAG_HTAG 0x0002 #define REQFLAG_OTAG 0x0004 #define REQFLAG_STAG 0x0008 #define REQFLAG_TARGET_RTN 0x0010 #define REQFLAG_NODATA 0x0000 #define REQFLAG_DATA_IN 0x0020 #define REQFLAG_DATA_OUT 0x0040 #define REQFLAG_DATA_UNKNOWN 0x0060 #define REQFLAG_DISARQ 0x0100 #define REQFLAG_FRC_ASYNC 0x0200 #define REQFLAG_FRC_SYNC 0x0400 #define REQFLAG_FRC_WIDE 0x0800 #define REQFLAG_NOPARITY 0x1000 #define REQFLAG_STOPQ 0x2000 #define REQFLAG_XTRASNS 0x4000 #define REQFLAG_PRIORITY 0x8000 typedef struct { isphdr_t req_header; uint32_t req_handle; uint8_t req_lun_trn; uint8_t req_target; uint16_t req_cdblen; uint16_t req_flags; uint16_t req_reserved; uint16_t req_time; uint16_t req_seg_count; uint8_t req_cdb[44]; } ispextreq_t; /* * ISP24XX structures */ typedef struct { isphdr_t req_header; uint32_t req_handle; uint16_t req_nphdl; uint16_t req_time; uint16_t req_seg_count; uint16_t req_reserved; uint8_t req_lun[8]; uint8_t req_alen_datadir; uint8_t req_task_management; uint8_t req_task_attribute; uint8_t req_crn; uint8_t req_cdb[16]; uint32_t req_dl; uint16_t req_tidlo; uint8_t req_tidhi; uint8_t req_vpidx; ispds64_t req_dataseg; } ispreqt7_t; /* Task Management Request Function */ typedef struct { isphdr_t tmf_header; uint32_t tmf_handle; uint16_t tmf_nphdl; uint8_t tmf_reserved0[2]; uint16_t tmf_delay; uint16_t tmf_timeout; uint8_t tmf_lun[8]; uint32_t tmf_flags; uint8_t tmf_reserved1[20]; uint16_t tmf_tidlo; uint8_t tmf_tidhi; uint8_t tmf_vpidx; uint8_t tmf_reserved2[12]; } isp24xx_tmf_t; #define ISP24XX_TMF_NOSEND 0x80000000 #define ISP24XX_TMF_LUN_RESET 0x00000010 #define ISP24XX_TMF_ABORT_TASK_SET 0x00000008 #define ISP24XX_TMF_CLEAR_TASK_SET 0x00000004 #define ISP24XX_TMF_TARGET_RESET 0x00000002 #define ISP24XX_TMF_CLEAR_ACA 0x00000001 /* I/O Abort Structure */ typedef struct { isphdr_t abrt_header; uint32_t abrt_handle; uint16_t abrt_nphdl; uint16_t abrt_options; uint32_t abrt_cmd_handle; uint16_t abrt_queue_number; uint8_t abrt_reserved[30]; uint16_t abrt_tidlo; uint8_t abrt_tidhi; uint8_t abrt_vpidx; uint8_t abrt_reserved1[12]; } isp24xx_abrt_t; #define ISP24XX_ABRT_NOSEND 0x01 /* don't actually send ABTS */ #define ISP24XX_ABRT_OKAY 0x00 /* in nphdl on return */ #define ISP24XX_ABRT_ENXIO 0x31 /* in nphdl on return */ #define ISP_CDSEG 7 typedef struct { isphdr_t req_header; uint32_t req_reserved; ispds_t req_dataseg[ISP_CDSEG]; } ispcontreq_t; #define ISP_CDSEG64 5 typedef struct { isphdr_t req_header; ispds64_t req_dataseg[ISP_CDSEG64]; } ispcontreq64_t; typedef struct { isphdr_t req_header; uint32_t req_handle; uint16_t req_scsi_status; uint16_t req_completion_status; uint16_t req_state_flags; uint16_t req_status_flags; uint16_t req_time; #define req_response_len req_time /* FC only */ uint16_t req_sense_len; uint32_t req_resid; uint8_t req_response[8]; /* FC only */ uint8_t req_sense_data[32]; } ispstatusreq_t; /* * Status Continuation */ typedef struct { isphdr_t req_header; uint8_t req_sense_data[60]; } ispstatus_cont_t; /* * 24XX Type 0 status */ typedef struct { isphdr_t req_header; uint32_t req_handle; uint16_t req_completion_status; uint16_t req_oxid; uint32_t req_resid; uint16_t req_reserved0; uint16_t req_state_flags; uint16_t req_retry_delay; /* aka Status Qualifier */ uint16_t req_scsi_status; uint32_t req_fcp_residual; uint32_t req_sense_len; uint32_t req_response_len; uint8_t req_rsp_sense[28]; } isp24xx_statusreq_t; /* * For Qlogic 2X00, the high order byte of SCSI status has * additional meaning. */ #define RQCS_CR 0x1000 /* Confirmation Request */ #define RQCS_RU 0x0800 /* Residual Under */ #define RQCS_RO 0x0400 /* Residual Over */ #define RQCS_RESID (RQCS_RU|RQCS_RO) #define RQCS_SV 0x0200 /* Sense Length Valid */ #define RQCS_RV 0x0100 /* FCP Response Length Valid */ /* * CT Passthru IOCB */ typedef struct { isphdr_t ctp_header; uint32_t ctp_handle; uint16_t ctp_status; uint16_t ctp_nphdl; /* n-port handle */ uint16_t ctp_cmd_cnt; /* Command DSD count */ uint8_t ctp_vpidx; uint8_t ctp_reserved0; uint16_t ctp_time; uint16_t ctp_reserved1; uint16_t ctp_rsp_cnt; /* Response DSD count */ uint16_t ctp_reserved2[5]; uint32_t ctp_rsp_bcnt; /* Response byte count */ uint32_t ctp_cmd_bcnt; /* Command byte count */ ispds64_t ctp_dataseg[2]; } isp_ct_pt_t; /* * MS Passthru IOCB */ typedef struct { isphdr_t ms_header; uint32_t ms_handle; uint16_t ms_nphdl; /* handle in high byte for !2k f/w */ uint16_t ms_status; uint16_t ms_flags; uint16_t ms_reserved1; /* low 8 bits */ uint16_t ms_time; uint16_t ms_cmd_cnt; /* Command DSD count */ uint16_t ms_tot_cnt; /* Total DSD Count */ uint8_t ms_type; /* MS type */ uint8_t ms_r_ctl; /* R_CTL */ uint16_t ms_rxid; /* RX_ID */ uint16_t ms_reserved2; uint32_t ms_handle2; uint32_t ms_rsp_bcnt; /* Response byte count */ uint32_t ms_cmd_bcnt; /* Command byte count */ ispds64_t ms_dataseg[2]; } isp_ms_t; /* * Completion Status Codes. */ #define RQCS_COMPLETE 0x0000 #define RQCS_DMA_ERROR 0x0002 #define RQCS_RESET_OCCURRED 0x0004 #define RQCS_ABORTED 0x0005 #define RQCS_TIMEOUT 0x0006 #define RQCS_DATA_OVERRUN 0x0007 #define RQCS_DATA_UNDERRUN 0x0015 #define RQCS_QUEUE_FULL 0x001C /* 1X00 Only Completion Codes */ #define RQCS_INCOMPLETE 0x0001 #define RQCS_TRANSPORT_ERROR 0x0003 #define RQCS_COMMAND_OVERRUN 0x0008 #define RQCS_STATUS_OVERRUN 0x0009 #define RQCS_BAD_MESSAGE 0x000a #define RQCS_NO_MESSAGE_OUT 0x000b #define RQCS_EXT_ID_FAILED 0x000c #define RQCS_IDE_MSG_FAILED 0x000d #define RQCS_ABORT_MSG_FAILED 0x000e #define RQCS_REJECT_MSG_FAILED 0x000f #define RQCS_NOP_MSG_FAILED 0x0010 #define RQCS_PARITY_ERROR_MSG_FAILED 0x0011 #define RQCS_DEVICE_RESET_MSG_FAILED 0x0012 #define RQCS_ID_MSG_FAILED 0x0013 #define RQCS_UNEXP_BUS_FREE 0x0014 #define RQCS_XACT_ERR1 0x0018 #define RQCS_XACT_ERR2 0x0019 #define RQCS_XACT_ERR3 0x001A #define RQCS_BAD_ENTRY 0x001B #define RQCS_PHASE_SKIPPED 0x001D #define RQCS_ARQS_FAILED 0x001E #define RQCS_WIDE_FAILED 0x001F #define RQCS_SYNCXFER_FAILED 0x0020 #define RQCS_LVD_BUSERR 0x0021 /* 2X00 Only Completion Codes */ #define RQCS_PORT_UNAVAILABLE 0x0028 #define RQCS_PORT_LOGGED_OUT 0x0029 #define RQCS_PORT_CHANGED 0x002A #define RQCS_PORT_BUSY 0x002B /* 24XX Only Completion Codes */ #define RQCS_24XX_DRE 0x0011 /* data reassembly error */ #define RQCS_24XX_TABORT 0x0013 /* aborted by target */ #define RQCS_24XX_ENOMEM 0x002C /* f/w resource unavailable */ #define RQCS_24XX_TMO 0x0030 /* task management overrun */ /* * 1X00 specific State Flags */ #define RQSF_GOT_BUS 0x0100 #define RQSF_GOT_TARGET 0x0200 #define RQSF_SENT_CDB 0x0400 #define RQSF_XFRD_DATA 0x0800 #define RQSF_GOT_STATUS 0x1000 #define RQSF_GOT_SENSE 0x2000 #define RQSF_XFER_COMPLETE 0x4000 /* * 2X00 specific State Flags * (same as 1X00 except RQSF_GOT_BUS/RQSF_GOT_TARGET are not available) */ #define RQSF_DATA_IN 0x0020 #define RQSF_DATA_OUT 0x0040 #define RQSF_STAG 0x0008 #define RQSF_OTAG 0x0004 #define RQSF_HTAG 0x0002 /* * 1X00 Status Flags */ #define RQSTF_DISCONNECT 0x0001 #define RQSTF_SYNCHRONOUS 0x0002 #define RQSTF_PARITY_ERROR 0x0004 #define RQSTF_BUS_RESET 0x0008 #define RQSTF_DEVICE_RESET 0x0010 #define RQSTF_ABORTED 0x0020 #define RQSTF_TIMEOUT 0x0040 #define RQSTF_NEGOTIATION 0x0080 /* * 2X00 specific state flags */ /* RQSF_SENT_CDB */ /* RQSF_XFRD_DATA */ /* RQSF_GOT_STATUS */ /* RQSF_XFER_COMPLETE */ /* * 2X00 specific status flags */ /* RQSTF_ABORTED */ /* RQSTF_TIMEOUT */ #define RQSTF_DMA_ERROR 0x0080 #define RQSTF_LOGOUT 0x2000 /* * Miscellaneous */ #ifndef ISP_EXEC_THROTTLE #define ISP_EXEC_THROTTLE 16 #endif /* * About Firmware returns an 'attribute' word in mailbox 6. * These attributes are for 2200 and 2300. */ #define ISP_FW_ATTR_TMODE 0x0001 #define ISP_FW_ATTR_SCCLUN 0x0002 #define ISP_FW_ATTR_FABRIC 0x0004 #define ISP_FW_ATTR_CLASS2 0x0008 #define ISP_FW_ATTR_FCTAPE 0x0010 #define ISP_FW_ATTR_IP 0x0020 #define ISP_FW_ATTR_VI 0x0040 #define ISP_FW_ATTR_VI_SOLARIS 0x0080 #define ISP_FW_ATTR_2KLOGINS 0x0100 /* just a guess... */ /* and these are for the 2400 */ #define ISP2400_FW_ATTR_CLASS2 0x0001 #define ISP2400_FW_ATTR_IP 0x0002 #define ISP2400_FW_ATTR_MULTIID 0x0004 #define ISP2400_FW_ATTR_SB2 0x0008 #define ISP2400_FW_ATTR_T10CRC 0x0010 #define ISP2400_FW_ATTR_VI 0x0020 #define ISP2400_FW_ATTR_MQ 0x0040 #define ISP2400_FW_ATTR_MSIX 0x0080 #define ISP2400_FW_ATTR_FCOE 0x0800 #define ISP2400_FW_ATTR_VP0 0x1000 #define ISP2400_FW_ATTR_EXPFW 0x2000 #define ISP2400_FW_ATTR_HOTFW 0x4000 #define ISP2400_FW_ATTR_EXTNDED 0x8000 #define ISP2400_FW_ATTR_EXTVP 0x00010000 #define ISP2400_FW_ATTR_VN2VN 0x00040000 #define ISP2400_FW_ATTR_EXMOFF 0x00080000 #define ISP2400_FW_ATTR_NPMOFF 0x00100000 #define ISP2400_FW_ATTR_DIFCHOP 0x00400000 #define ISP2400_FW_ATTR_SRIOV 0x02000000 #define ISP2400_FW_ATTR_ASICTMP 0x0200000000 #define ISP2400_FW_ATTR_ATIOMQ 0x0400000000 /* * These are either manifestly true or are dependent on f/w attributes */ #define ISP_CAP_TMODE(isp) \ (IS_24XX(isp)? 1 : (isp->isp_fwattr & ISP_FW_ATTR_TMODE)) #define ISP_CAP_SCCFW(isp) \ (IS_24XX(isp)? 1 : (isp->isp_fwattr & ISP_FW_ATTR_SCCLUN)) #define ISP_CAP_2KLOGIN(isp) \ (IS_24XX(isp)? 1 : (isp->isp_fwattr & ISP_FW_ATTR_2KLOGINS)) /* * This is only true for 24XX cards with this f/w attribute */ #define ISP_CAP_MULTI_ID(isp) \ (IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_MULTIID) : 0) #define ISP_GET_VPIDX(isp, tag) \ (ISP_CAP_MULTI_ID(isp) ? tag : 0) #define ISP_CAP_MSIX(isp) \ (IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_MSIX) : 0) #define ISP_CAP_VP0(isp) \ (IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_VP0) : 0) /* * This is true manifestly or is dependent on a f/w attribute * but may or may not actually be *enabled*. In any case, it * is enabled on a per-channel basis. */ #define ISP_CAP_FCTAPE(isp) \ (IS_24XX(isp)? 1 : (isp->isp_fwattr & ISP_FW_ATTR_FCTAPE)) #define ISP_FCTAPE_ENABLED(isp, chan) \ (IS_24XX(isp)? (FCPARAM(isp, chan)->isp_xfwoptions & ICB2400_OPT2_FCTAPE) != 0 : (FCPARAM(isp, chan)->isp_xfwoptions & ICBXOPT_FCTAPE) != 0) /* * Reduced Interrupt Operation Response Queue Entries */ typedef struct { isphdr_t req_header; uint32_t req_handles[15]; } isp_rio1_t; typedef struct { isphdr_t req_header; uint16_t req_handles[30]; } isp_rio2_t; /* * FC (ISP2100/ISP2200/ISP2300/ISP2400) specific data structures */ /* * Initialization Control Block * * Version One (prime) format. */ typedef struct { uint8_t icb_version; uint8_t icb_reserved0; uint16_t icb_fwoptions; uint16_t icb_maxfrmlen; uint16_t icb_maxalloc; uint16_t icb_execthrottle; uint8_t icb_retry_count; uint8_t icb_retry_delay; uint8_t icb_portname[8]; uint16_t icb_hardaddr; uint8_t icb_iqdevtype; uint8_t icb_logintime; uint8_t icb_nodename[8]; uint16_t icb_rqstout; uint16_t icb_rspnsin; uint16_t icb_rqstqlen; uint16_t icb_rsltqlen; uint16_t icb_rqstaddr[4]; uint16_t icb_respaddr[4]; uint16_t icb_lunenables; uint8_t icb_ccnt; uint8_t icb_icnt; uint16_t icb_lunetimeout; uint16_t icb_reserved1; uint16_t icb_xfwoptions; uint8_t icb_racctimer; uint8_t icb_idelaytimer; uint16_t icb_zfwoptions; uint16_t icb_reserved2[13]; } isp_icb_t; #define ICB_VERSION1 1 #define ICBOPT_EXTENDED 0x8000 #define ICBOPT_BOTH_WWNS 0x4000 #define ICBOPT_FULL_LOGIN 0x2000 #define ICBOPT_STOP_ON_QFULL 0x1000 /* 2200/2100 only */ #define ICBOPT_PREV_ADDRESS 0x0800 #define ICBOPT_SRCHDOWN 0x0400 #define ICBOPT_NOLIP 0x0200 #define ICBOPT_PDBCHANGE_AE 0x0100 #define ICBOPT_TGT_TYPE 0x0080 #define ICBOPT_INI_ADISC 0x0040 #define ICBOPT_INI_DISABLE 0x0020 #define ICBOPT_TGT_ENABLE 0x0010 #define ICBOPT_FAST_POST 0x0008 #define ICBOPT_FULL_DUPLEX 0x0004 #define ICBOPT_FAIRNESS 0x0002 #define ICBOPT_HARD_ADDRESS 0x0001 #define ICBXOPT_NO_LOGOUT 0x8000 /* no logout on link failure */ #define ICBXOPT_FCTAPE_CCQ 0x4000 /* FC-Tape Command Queueing */ #define ICBXOPT_FCTAPE_CONFIRM 0x2000 #define ICBXOPT_FCTAPE 0x1000 #define ICBXOPT_CLASS2_ACK0 0x0200 #define ICBXOPT_CLASS2 0x0100 #define ICBXOPT_NO_PLAY 0x0080 /* don't play if can't get hard addr */ #define ICBXOPT_TOPO_MASK 0x0070 #define ICBXOPT_LOOP_ONLY 0x0000 #define ICBXOPT_PTP_ONLY 0x0010 #define ICBXOPT_LOOP_2_PTP 0x0020 #define ICBXOPT_PTP_2_LOOP 0x0030 /* * The lower 4 bits of the xfwoptions field are the OPERATION MODE bits. * RIO is not defined for the 23XX cards (just 2200) */ #define ICBXOPT_RIO_OFF 0 #define ICBXOPT_RIO_16BIT 1 #define ICBXOPT_RIO_32BIT 2 #define ICBXOPT_RIO_16BIT_IOCB 3 #define ICBXOPT_RIO_32BIT_IOCB 4 #define ICBXOPT_ZIO 5 #define ICBXOPT_TIMER_MASK 0x7 #define ICBZOPT_RATE_MASK 0xC000 #define ICBZOPT_RATE_1GB 0x0000 #define ICBZOPT_RATE_AUTO 0x8000 #define ICBZOPT_RATE_2GB 0x4000 #define ICBZOPT_50_OHM 0x2000 #define ICBZOPT_NO_LOCAL_PLOGI 0x0080 #define ICBZOPT_ENA_OOF 0x0040 /* out of order frame handling */ #define ICBZOPT_RSPSZ_MASK 0x0030 #define ICBZOPT_RSPSZ_24 0x0000 #define ICBZOPT_RSPSZ_12 0x0010 #define ICBZOPT_RSPSZ_24A 0x0020 #define ICBZOPT_RSPSZ_32 0x0030 #define ICBZOPT_SOFTID 0x0002 #define ICBZOPT_ENA_RDXFR_RDY 0x0001 /* 2400 F/W options */ #define ICB2400_OPT1_BOTH_WWNS 0x00004000 #define ICB2400_OPT1_FULL_LOGIN 0x00002000 #define ICB2400_OPT1_PREV_ADDRESS 0x00000800 #define ICB2400_OPT1_SRCHDOWN 0x00000400 #define ICB2400_OPT1_NOLIP 0x00000200 #define ICB2400_OPT1_INI_DISABLE 0x00000020 #define ICB2400_OPT1_TGT_ENABLE 0x00000010 #define ICB2400_OPT1_FULL_DUPLEX 0x00000004 #define ICB2400_OPT1_FAIRNESS 0x00000002 #define ICB2400_OPT1_HARD_ADDRESS 0x00000001 #define ICB2400_OPT2_ENA_ATIOMQ 0x08000000 #define ICB2400_OPT2_ENA_IHA 0x04000000 #define ICB2400_OPT2_QOS 0x02000000 #define ICB2400_OPT2_IOCBS 0x01000000 #define ICB2400_OPT2_ENA_IHR 0x00400000 #define ICB2400_OPT2_ENA_VMS 0x00200000 #define ICB2400_OPT2_ENA_TA 0x00100000 #define ICB2400_OPT2_TPRLIC 0x00004000 #define ICB2400_OPT2_FCTAPE 0x00001000 #define ICB2400_OPT2_FCSP 0x00000800 #define ICB2400_OPT2_CLASS2_ACK0 0x00000200 #define ICB2400_OPT2_CLASS2 0x00000100 #define ICB2400_OPT2_NO_PLAY 0x00000080 #define ICB2400_OPT2_TOPO_MASK 0x00000070 #define ICB2400_OPT2_LOOP_ONLY 0x00000000 #define ICB2400_OPT2_PTP_ONLY 0x00000010 #define ICB2400_OPT2_LOOP_2_PTP 0x00000020 #define ICB2400_OPT2_TIMER_MASK 0x0000000f #define ICB2400_OPT2_ZIO 0x00000005 #define ICB2400_OPT2_ZIO1 0x00000006 #define ICB2400_OPT3_NO_CTXDIS 0x40000000 #define ICB2400_OPT3_ENA_ETH_RESP 0x08000000 #define ICB2400_OPT3_ENA_ETH_ATIO 0x04000000 #define ICB2400_OPT3_ENA_MFCF 0x00020000 #define ICB2400_OPT3_SKIP_4GB 0x00010000 #define ICB2400_OPT3_RATE_MASK 0x0000E000 #define ICB2400_OPT3_RATE_1GB 0x00000000 #define ICB2400_OPT3_RATE_2GB 0x00002000 #define ICB2400_OPT3_RATE_AUTO 0x00004000 #define ICB2400_OPT3_RATE_4GB 0x00006000 #define ICB2400_OPT3_RATE_8GB 0x00008000 #define ICB2400_OPT3_RATE_16GB 0x0000A000 #define ICB2400_OPT3_RATE_32GB 0x0000C000 #define ICB2400_OPT3_ENA_OOF_XFRDY 0x00000200 #define ICB2400_OPT3_NO_N2N_LOGI 0x00000100 #define ICB2400_OPT3_NO_LOCAL_PLOGI 0x00000080 #define ICB2400_OPT3_ENA_OOF 0x00000040 /* note that a response size flag of zero is reserved! */ #define ICB2400_OPT3_RSPSZ_MASK 0x00000030 #define ICB2400_OPT3_RSPSZ_12 0x00000010 #define ICB2400_OPT3_RSPSZ_24 0x00000020 #define ICB2400_OPT3_RSPSZ_32 0x00000030 #define ICB2400_OPT3_SOFTID 0x00000002 #define ICB_MIN_FRMLEN 256 #define ICB_MAX_FRMLEN 2112 #define ICB_DFLT_FRMLEN 1024 #define ICB_DFLT_ALLOC 256 #define ICB_DFLT_THROTTLE 16 #define ICB_DFLT_RDELAY 5 #define ICB_DFLT_RCOUNT 3 #define ICB_LOGIN_TOV 10 #define ICB_LUN_ENABLE_TOV 15 /* * And somebody at QLogic had a great idea that you could just change * the structure *and* keep the version number the same as the other cards. */ typedef struct { uint16_t icb_version; uint16_t icb_reserved0; uint16_t icb_maxfrmlen; uint16_t icb_execthrottle; uint16_t icb_xchgcnt; uint16_t icb_hardaddr; uint8_t icb_portname[8]; uint8_t icb_nodename[8]; uint16_t icb_rspnsin; uint16_t icb_rqstout; uint16_t icb_retry_count; uint16_t icb_priout; uint16_t icb_rsltqlen; uint16_t icb_rqstqlen; uint16_t icb_ldn_nols; uint16_t icb_prqstqlen; uint16_t icb_rqstaddr[4]; uint16_t icb_respaddr[4]; uint16_t icb_priaddr[4]; uint16_t icb_msixresp; uint16_t icb_msixatio; uint16_t icb_reserved1[2]; uint16_t icb_atio_in; uint16_t icb_atioqlen; uint16_t icb_atioqaddr[4]; uint16_t icb_idelaytimer; uint16_t icb_logintime; uint32_t icb_fwoptions1; uint32_t icb_fwoptions2; uint32_t icb_fwoptions3; uint16_t icb_qos; uint16_t icb_reserved2[3]; uint16_t icb_enodemac[3]; uint16_t icb_disctime; uint16_t icb_reserved3[4]; } isp_icb_2400_t; #define RQRSP_ADDR0015 0 #define RQRSP_ADDR1631 1 #define RQRSP_ADDR3247 2 #define RQRSP_ADDR4863 3 #define ICB_NNM0 7 #define ICB_NNM1 6 #define ICB_NNM2 5 #define ICB_NNM3 4 #define ICB_NNM4 3 #define ICB_NNM5 2 #define ICB_NNM6 1 #define ICB_NNM7 0 #define MAKE_NODE_NAME_FROM_WWN(array, wwn) \ array[ICB_NNM0] = (uint8_t) ((wwn >> 0) & 0xff), \ array[ICB_NNM1] = (uint8_t) ((wwn >> 8) & 0xff), \ array[ICB_NNM2] = (uint8_t) ((wwn >> 16) & 0xff), \ array[ICB_NNM3] = (uint8_t) ((wwn >> 24) & 0xff), \ array[ICB_NNM4] = (uint8_t) ((wwn >> 32) & 0xff), \ array[ICB_NNM5] = (uint8_t) ((wwn >> 40) & 0xff), \ array[ICB_NNM6] = (uint8_t) ((wwn >> 48) & 0xff), \ array[ICB_NNM7] = (uint8_t) ((wwn >> 56) & 0xff) #define MAKE_WWN_FROM_NODE_NAME(wwn, array) \ wwn = ((uint64_t) array[ICB_NNM0]) | \ ((uint64_t) array[ICB_NNM1] << 8) | \ ((uint64_t) array[ICB_NNM2] << 16) | \ ((uint64_t) array[ICB_NNM3] << 24) | \ ((uint64_t) array[ICB_NNM4] << 32) | \ ((uint64_t) array[ICB_NNM5] << 40) | \ ((uint64_t) array[ICB_NNM6] << 48) | \ ((uint64_t) array[ICB_NNM7] << 56) /* * For MULTI_ID firmware, this describes a * virtual port entity for getting status. */ typedef struct { uint16_t vp_port_status; uint8_t vp_port_options; uint8_t vp_port_loopid; uint8_t vp_port_portname[8]; uint8_t vp_port_nodename[8]; uint16_t vp_port_portid_lo; /* not present when trailing icb */ uint16_t vp_port_portid_hi; /* not present when trailing icb */ } vp_port_info_t; #define ICB2400_VPOPT_ENA_SNSLOGIN 0x00000040 /* Enable SNS Login and SCR for Virtual Ports */ #define ICB2400_VPOPT_TGT_DISABLE 0x00000020 /* Target Mode Disabled */ #define ICB2400_VPOPT_INI_ENABLE 0x00000010 /* Initiator Mode Enabled */ #define ICB2400_VPOPT_ENABLED 0x00000008 /* VP Enabled */ #define ICB2400_VPOPT_NOPLAY 0x00000004 /* ID Not Acquired */ #define ICB2400_VPOPT_PREV_ADDRESS 0x00000002 /* Previously Assigned ID */ #define ICB2400_VPOPT_HARD_ADDRESS 0x00000001 /* Hard Assigned ID */ #define ICB2400_VPOPT_WRITE_SIZE 20 /* * For MULTI_ID firmware, we append this structure * to the isp_icb_2400_t above, followed by a list * structures that are *most* of the vp_port_info_t. */ typedef struct { uint16_t vp_count; uint16_t vp_global_options; } isp_icb_2400_vpinfo_t; #define ICB2400_VPINFO_OFF 0x80 /* offset from start of ICB */ #define ICB2400_VPINFO_PORT_OFF(chan) \ (ICB2400_VPINFO_OFF + \ sizeof (isp_icb_2400_vpinfo_t) + ((chan) * ICB2400_VPOPT_WRITE_SIZE)) #define ICB2400_VPGOPT_FCA 0x01 /* Assume Clean Address bit in FLOGI ACC set (works only in static configurations) */ #define ICB2400_VPGOPT_MID_DISABLE 0x02 /* when set, connection mode2 will work with NPIV-capable switched */ #define ICB2400_VPGOPT_VP0_DECOUPLE 0x04 /* Allow VP0 decoupling if firmware supports it */ #define ICB2400_VPGOPT_SUSP_FDISK 0x10 /* Suspend FDISC for Enabled VPs */ #define ICB2400_VPGOPT_GEN_RIDA 0x20 /* Generate RIDA if FLOGI Fails */ typedef struct { isphdr_t vp_ctrl_hdr; uint32_t vp_ctrl_handle; uint16_t vp_ctrl_index_fail; uint16_t vp_ctrl_status; uint16_t vp_ctrl_command; uint16_t vp_ctrl_vp_count; uint16_t vp_ctrl_idmap[16]; uint16_t vp_ctrl_reserved[7]; uint16_t vp_ctrl_fcf_index; } vp_ctrl_info_t; #define VP_CTRL_CMD_ENABLE_VP 0x00 #define VP_CTRL_CMD_DISABLE_VP 0x08 #define VP_CTRL_CMD_DISABLE_VP_REINIT_LINK 0x09 #define VP_CTRL_CMD_DISABLE_VP_LOGO 0x0A #define VP_CTRL_CMD_DISABLE_VP_LOGO_ALL 0x0B /* * We can use this structure for modifying either one or two VP ports after initialization */ typedef struct { isphdr_t vp_mod_hdr; uint32_t vp_mod_hdl; uint16_t vp_mod_reserved0; uint16_t vp_mod_status; uint8_t vp_mod_cmd; uint8_t vp_mod_cnt; uint8_t vp_mod_idx0; uint8_t vp_mod_idx1; struct { uint8_t options; uint8_t loopid; uint16_t reserved1; uint8_t wwpn[8]; uint8_t wwnn[8]; } vp_mod_ports[2]; uint8_t vp_mod_reserved2[8]; } vp_modify_t; #define VP_STS_OK 0x00 #define VP_STS_ERR 0x01 #define VP_CNT_ERR 0x02 #define VP_GEN_ERR 0x03 #define VP_IDX_ERR 0x04 #define VP_STS_BSY 0x05 #define VP_MODIFY 0x00 #define VP_MODIFY_ENA 0x01 #define VP_MODIFY_OPT 0x02 #define VP_RESUME 0x03 /* * Port Data Base Element */ typedef struct { uint16_t pdb_options; uint8_t pdb_mstate; uint8_t pdb_sstate; uint8_t pdb_hardaddr_bits[4]; uint8_t pdb_portid_bits[4]; uint8_t pdb_nodename[8]; uint8_t pdb_portname[8]; uint16_t pdb_execthrottle; uint16_t pdb_exec_count; uint8_t pdb_retry_count; uint8_t pdb_retry_delay; uint16_t pdb_resalloc; uint16_t pdb_curalloc; uint16_t pdb_qhead; uint16_t pdb_qtail; uint16_t pdb_tl_next; uint16_t pdb_tl_last; uint16_t pdb_features; /* PLOGI, Common Service */ uint16_t pdb_pconcurrnt; /* PLOGI, Common Service */ uint16_t pdb_roi; /* PLOGI, Common Service */ uint8_t pdb_target; uint8_t pdb_initiator; /* PLOGI, Class 3 Control Flags */ uint16_t pdb_rdsiz; /* PLOGI, Class 3 */ uint16_t pdb_ncseq; /* PLOGI, Class 3 */ uint16_t pdb_noseq; /* PLOGI, Class 3 */ uint16_t pdb_labrtflg; uint16_t pdb_lstopflg; uint16_t pdb_sqhead; uint16_t pdb_sqtail; uint16_t pdb_ptimer; uint16_t pdb_nxt_seqid; uint16_t pdb_fcount; uint16_t pdb_prli_len; uint16_t pdb_prli_svc0; uint16_t pdb_prli_svc3; uint16_t pdb_loopid; uint16_t pdb_il_ptr; uint16_t pdb_sl_ptr; } isp_pdb_21xx_t; #define PDB_OPTIONS_XMITTING (1<<11) #define PDB_OPTIONS_LNKXMIT (1<<10) #define PDB_OPTIONS_ABORTED (1<<9) #define PDB_OPTIONS_ADISC (1<<1) #define PDB_STATE_DISCOVERY 0 #define PDB_STATE_WDISC_ACK 1 #define PDB_STATE_PLOGI 2 #define PDB_STATE_PLOGI_ACK 3 #define PDB_STATE_PRLI 4 #define PDB_STATE_PRLI_ACK 5 #define PDB_STATE_LOGGED_IN 6 #define PDB_STATE_PORT_UNAVAIL 7 #define PDB_STATE_PRLO 8 #define PDB_STATE_PRLO_ACK 9 #define PDB_STATE_PLOGO 10 #define PDB_STATE_PLOG_ACK 11 #define SVC3_ROLE_MASK 0x30 #define SVC3_ROLE_SHIFT 4 #define BITS2WORD(x) ((x)[0] << 16 | (x)[3] << 8 | (x)[2]) #define BITS2WORD_24XX(x) ((x)[0] << 16 | (x)[1] << 8 | (x)[2]) /* * Port Data Base Element- 24XX cards */ typedef struct { uint16_t pdb_flags; uint8_t pdb_curstate; uint8_t pdb_laststate; uint8_t pdb_hardaddr_bits[4]; uint8_t pdb_portid_bits[4]; #define pdb_nxt_seqid_2400 pdb_portid_bits[3] uint16_t pdb_retry_timer; uint16_t pdb_handle; uint16_t pdb_rcv_dsize; uint16_t pdb_reserved0; uint16_t pdb_prli_svc0; uint16_t pdb_prli_svc3; uint8_t pdb_portname[8]; uint8_t pdb_nodename[8]; uint8_t pdb_reserved1[24]; } isp_pdb_24xx_t; #define PDB2400_TID_SUPPORTED 0x4000 #define PDB2400_FC_TAPE 0x0080 #define PDB2400_CLASS2_ACK0 0x0040 #define PDB2400_FCP_CONF 0x0020 #define PDB2400_CLASS2 0x0010 #define PDB2400_ADDR_VALID 0x0002 #define PDB2400_STATE_PLOGI_PEND 0x03 #define PDB2400_STATE_PLOGI_DONE 0x04 #define PDB2400_STATE_PRLI_PEND 0x05 #define PDB2400_STATE_LOGGED_IN 0x06 #define PDB2400_STATE_PORT_UNAVAIL 0x07 #define PDB2400_STATE_PRLO_PEND 0x09 #define PDB2400_STATE_LOGO_PEND 0x0B /* * Common elements from the above two structures that are actually useful to us. */ typedef struct { uint16_t handle; + uint16_t prli_word0; uint16_t prli_word3; uint32_t : 8, portid : 24; uint8_t portname[8]; uint8_t nodename[8]; } isp_pdb_t; /* * Port/Node Name List Element */ typedef struct { uint8_t pnnle_name[8]; uint16_t pnnle_handle; uint16_t pnnle_reserved; } isp_pnnle_t; #define PNNL_OPTIONS_NODE_NAMES (1<<0) #define PNNL_OPTIONS_PORT_DATA (1<<2) #define PNNL_OPTIONS_INITIATORS (1<<3) /* * Port and N-Port Handle List Element */ typedef struct { uint16_t pnhle_port_id_lo; uint16_t pnhle_port_id_hi_handle; } isp_pnhle_21xx_t; typedef struct { uint16_t pnhle_port_id_lo; uint16_t pnhle_port_id_hi; uint16_t pnhle_handle; } isp_pnhle_23xx_t; typedef struct { uint16_t pnhle_port_id_lo; uint16_t pnhle_port_id_hi; uint16_t pnhle_handle; uint16_t pnhle_reserved; } isp_pnhle_24xx_t; /* * Port Database Changed Async Event information for 24XX cards */ /* N-Port Handle */ #define PDB24XX_AE_GLOBAL 0xFFFF /* Reason Codes */ #define PDB24XX_AE_OK 0x00 #define PDB24XX_AE_IMPL_LOGO_1 0x01 #define PDB24XX_AE_IMPL_LOGO_2 0x02 #define PDB24XX_AE_IMPL_LOGO_3 0x03 #define PDB24XX_AE_PLOGI_RCVD 0x04 #define PDB24XX_AE_PLOGI_RJT 0x05 #define PDB24XX_AE_PRLI_RCVD 0x06 #define PDB24XX_AE_PRLI_RJT 0x07 #define PDB24XX_AE_TPRLO 0x08 #define PDB24XX_AE_TPRLO_RJT 0x09 #define PDB24XX_AE_PRLO_RCVD 0x0a #define PDB24XX_AE_LOGO_RCVD 0x0b #define PDB24XX_AE_TOPO_CHG 0x0c #define PDB24XX_AE_NPORT_CHG 0x0d #define PDB24XX_AE_FLOGI_RJT 0x0e #define PDB24XX_AE_BAD_FANN 0x0f #define PDB24XX_AE_FLOGI_TIMO 0x10 #define PDB24XX_AE_ABX_LOGO 0x11 #define PDB24XX_AE_PLOGI_DONE 0x12 #define PDB24XX_AE_PRLI_DONE 0x13 #define PDB24XX_AE_OPN_1 0x14 #define PDB24XX_AE_OPN_2 0x15 #define PDB24XX_AE_TXERR 0x16 #define PDB24XX_AE_FORCED_LOGO 0x17 #define PDB24XX_AE_DISC_TIMO 0x18 /* * Genericized Port Login/Logout software structure */ typedef struct { uint16_t handle; uint16_t channel; uint32_t flags : 8, portid : 24; } isp_plcmd_t; /* the flags to use are those for PLOGX_FLG_* below */ /* * ISP24XX- Login/Logout Port IOCB */ typedef struct { isphdr_t plogx_header; uint32_t plogx_handle; uint16_t plogx_status; uint16_t plogx_nphdl; uint16_t plogx_flags; uint16_t plogx_vphdl; /* low 8 bits */ uint16_t plogx_portlo; /* low 16 bits */ uint16_t plogx_rspsz_porthi; struct { uint16_t lo16; uint16_t hi16; } plogx_ioparm[11]; } isp_plogx_t; #define PLOGX_STATUS_OK 0x00 #define PLOGX_STATUS_UNAVAIL 0x28 #define PLOGX_STATUS_LOGOUT 0x29 #define PLOGX_STATUS_IOCBERR 0x31 #define PLOGX_IOCBERR_NOLINK 0x01 #define PLOGX_IOCBERR_NOIOCB 0x02 #define PLOGX_IOCBERR_NOXGHG 0x03 #define PLOGX_IOCBERR_FAILED 0x04 /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_NOFABRIC 0x05 #define PLOGX_IOCBERR_NOTREADY 0x07 #define PLOGX_IOCBERR_NOLOGIN 0x09 /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_NOPCB 0x0a #define PLOGX_IOCBERR_REJECT 0x18 /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_EINVAL 0x19 /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_PORTUSED 0x1a /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_HNDLUSED 0x1b /* further info in IOPARM 1 */ #define PLOGX_IOCBERR_NOHANDLE 0x1c #define PLOGX_IOCBERR_NOFLOGI 0x1f /* further info in IOPARM 1 */ #define PLOGX_FLG_CMD_MASK 0xf #define PLOGX_FLG_CMD_PLOGI 0 #define PLOGX_FLG_CMD_PRLI 1 #define PLOGX_FLG_CMD_PDISC 2 #define PLOGX_FLG_CMD_LOGO 8 #define PLOGX_FLG_CMD_PRLO 9 #define PLOGX_FLG_CMD_TPRLO 10 #define PLOGX_FLG_COND_PLOGI 0x10 /* if with PLOGI */ #define PLOGX_FLG_IMPLICIT 0x10 /* if with LOGO, PRLO, TPRLO */ #define PLOGX_FLG_SKIP_PRLI 0x20 /* if with PLOGI */ #define PLOGX_FLG_IMPLICIT_LOGO_ALL 0x20 /* if with LOGO */ #define PLOGX_FLG_EXPLICIT_LOGO 0x40 /* if with LOGO */ #define PLOGX_FLG_COMMON_FEATURES 0x80 /* if with PLOGI */ #define PLOGX_FLG_FREE_NPHDL 0x80 /* if with with LOGO */ #define PLOGX_FLG_CLASS2 0x100 /* if with PLOGI */ #define PLOGX_FLG_FCP2_OVERRIDE 0x200 /* if with PRLOG, PRLI */ /* * Report ID Acquisistion (24XX multi-id firmware) */ typedef struct { isphdr_t ridacq_hdr; uint32_t ridacq_handle; uint8_t ridacq_vp_acquired; uint8_t ridacq_vp_setup; uint8_t ridacq_vp_index; uint8_t ridacq_vp_status; uint16_t ridacq_vp_port_lo; uint8_t ridacq_vp_port_hi; uint8_t ridacq_format; /* 0 or 1 */ uint16_t ridacq_map[8]; uint8_t ridacq_reserved1[32]; } isp_ridacq_t; #define RIDACQ_STS_COMPLETE 0 #define RIDACQ_STS_UNACQUIRED 1 #define RIDACQ_STS_CHANGED 2 #define RIDACQ_STS_SNS_TIMEOUT 3 #define RIDACQ_STS_SNS_REJECTED 4 #define RIDACQ_STS_SCR_TIMEOUT 5 #define RIDACQ_STS_SCR_REJECTED 6 /* * Simple Name Server Data Structures */ #define SNS_GA_NXT 0x100 #define SNS_GPN_ID 0x112 #define SNS_GNN_ID 0x113 #define SNS_GFT_ID 0x117 #define SNS_GFF_ID 0x11F #define SNS_GID_FT 0x171 #define SNS_GID_PT 0x1A1 #define SNS_RFT_ID 0x217 #define SNS_RSPN_ID 0x218 #define SNS_RFF_ID 0x21F #define SNS_RSNN_NN 0x239 typedef struct { uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_data[]; /* variable data */ } sns_screq_t; /* Subcommand Request Structure */ typedef struct { uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_cmd; uint16_t snscb_reserved2; uint32_t snscb_reserved3; uint32_t snscb_port; } sns_ga_nxt_req_t; #define SNS_GA_NXT_REQ_SIZE (sizeof (sns_ga_nxt_req_t)) typedef struct { /* Used for GFT_ID, GFF_ID, etc. */ uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_cmd; uint16_t snscb_mword_div_2; uint32_t snscb_reserved3; uint32_t snscb_portid; } sns_gxx_id_req_t; #define SNS_GXX_ID_REQ_SIZE (sizeof (sns_gxx_id_req_t)) typedef struct { uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_cmd; uint16_t snscb_mword_div_2; uint32_t snscb_reserved3; uint32_t snscb_fc4_type; } sns_gid_ft_req_t; #define SNS_GID_FT_REQ_SIZE (sizeof (sns_gid_ft_req_t)) typedef struct { uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_cmd; uint16_t snscb_mword_div_2; uint32_t snscb_reserved3; uint8_t snscb_port_type; uint8_t snscb_domain; uint8_t snscb_area; uint8_t snscb_flags; } sns_gid_pt_req_t; #define SNS_GID_PT_REQ_SIZE (sizeof (sns_gid_pt_req_t)) typedef struct { uint16_t snscb_rblen; /* response buffer length (words) */ uint16_t snscb_reserved0; uint16_t snscb_addr[4]; /* response buffer address */ uint16_t snscb_sblen; /* subcommand buffer length (words) */ uint16_t snscb_reserved1; uint16_t snscb_cmd; uint16_t snscb_reserved2; uint32_t snscb_reserved3; uint32_t snscb_port; uint32_t snscb_fc4_types[8]; } sns_rft_id_req_t; #define SNS_RFT_ID_REQ_SIZE (sizeof (sns_rft_id_req_t)) typedef struct { ct_hdr_t snscb_cthdr; uint8_t snscb_port_type; uint8_t snscb_port_id[3]; uint8_t snscb_portname[8]; uint16_t snscb_data[]; /* variable data */ } sns_scrsp_t; /* Subcommand Response Structure */ typedef struct { ct_hdr_t snscb_cthdr; uint8_t snscb_port_type; uint8_t snscb_port_id[3]; uint8_t snscb_portname[8]; uint8_t snscb_pnlen; /* symbolic port name length */ uint8_t snscb_pname[255]; /* symbolic port name */ uint8_t snscb_nodename[8]; uint8_t snscb_nnlen; /* symbolic node name length */ uint8_t snscb_nname[255]; /* symbolic node name */ uint8_t snscb_ipassoc[8]; uint8_t snscb_ipaddr[16]; uint8_t snscb_svc_class[4]; uint8_t snscb_fc4_types[32]; uint8_t snscb_fpname[8]; uint8_t snscb_reserved; uint8_t snscb_hardaddr[3]; } sns_ga_nxt_rsp_t; /* Subcommand Response Structure */ #define SNS_GA_NXT_RESP_SIZE (sizeof (sns_ga_nxt_rsp_t)) typedef struct { ct_hdr_t snscb_cthdr; uint8_t snscb_wwn[8]; } sns_gxn_id_rsp_t; #define SNS_GXN_ID_RESP_SIZE (sizeof (sns_gxn_id_rsp_t)) typedef struct { ct_hdr_t snscb_cthdr; uint32_t snscb_fc4_types[8]; } sns_gft_id_rsp_t; #define SNS_GFT_ID_RESP_SIZE (sizeof (sns_gft_id_rsp_t)) typedef struct { ct_hdr_t snscb_cthdr; uint32_t snscb_fc4_features[32]; } sns_gff_id_rsp_t; #define SNS_GFF_ID_RESP_SIZE (sizeof (sns_gff_id_rsp_t)) typedef struct { /* Used for GID_FT, GID_PT, etc. */ ct_hdr_t snscb_cthdr; struct { uint8_t control; uint8_t portid[3]; } snscb_ports[1]; } sns_gid_xx_rsp_t; #define SNS_GID_XX_RESP_SIZE(x) ((sizeof (sns_gid_xx_rsp_t)) + ((x - 1) << 2)) /* * Other Misc Structures */ /* ELS Pass Through */ typedef struct { isphdr_t els_hdr; uint32_t els_handle; uint16_t els_status; uint16_t els_nphdl; uint16_t els_xmit_dsd_count; /* outgoing only */ uint8_t els_vphdl; uint8_t els_sof; uint32_t els_rxid; uint16_t els_recv_dsd_count; /* outgoing only */ uint8_t els_opcode; uint8_t els_reserved1; uint8_t els_did_lo; uint8_t els_did_mid; uint8_t els_did_hi; uint8_t els_reserved2; uint16_t els_reserved3; uint16_t els_ctl_flags; union { struct { uint32_t _els_bytecnt; uint32_t _els_subcode1; uint32_t _els_subcode2; uint8_t _els_reserved4[20]; } in; struct { uint32_t _els_recv_bytecnt; uint32_t _els_xmit_bytecnt; uint32_t _els_xmit_dsd_length; uint16_t _els_xmit_dsd_a1500; uint16_t _els_xmit_dsd_a3116; uint16_t _els_xmit_dsd_a4732; uint16_t _els_xmit_dsd_a6348; uint32_t _els_recv_dsd_length; uint16_t _els_recv_dsd_a1500; uint16_t _els_recv_dsd_a3116; uint16_t _els_recv_dsd_a4732; uint16_t _els_recv_dsd_a6348; } out; } inout; #define els_bytecnt inout.in._els_bytecnt #define els_subcode1 inout.in._els_subcode1 #define els_subcode2 inout.in._els_subcode2 #define els_reserved4 inout.in._els_reserved4 #define els_recv_bytecnt inout.out._els_recv_bytecnt #define els_xmit_bytecnt inout.out._els_xmit_bytecnt #define els_xmit_dsd_length inout.out._els_xmit_dsd_length #define els_xmit_dsd_a1500 inout.out._els_xmit_dsd_a1500 #define els_xmit_dsd_a3116 inout.out._els_xmit_dsd_a3116 #define els_xmit_dsd_a4732 inout.out._els_xmit_dsd_a4732 #define els_xmit_dsd_a6348 inout.out._els_xmit_dsd_a6348 #define els_recv_dsd_length inout.out._els_recv_dsd_length #define els_recv_dsd_a1500 inout.out._els_recv_dsd_a1500 #define els_recv_dsd_a3116 inout.out._els_recv_dsd_a3116 #define els_recv_dsd_a4732 inout.out._els_recv_dsd_a4732 #define els_recv_dsd_a6348 inout.out._els_recv_dsd_a6348 } els_t; /* * A handy package structure for running FC-SCSI commands internally */ typedef struct { uint16_t handle; uint16_t lun; uint32_t channel : 8, portid : 24; uint32_t timeout; union { struct { uint32_t data_length; uint32_t no_wait : 1, do_read : 1; uint8_t cdb[16]; void *data_ptr; } beg; struct { uint32_t data_residual; uint8_t status; uint8_t pad; uint16_t sense_length; uint8_t sense_data[32]; } end; } fcd; } isp_xcmd_t; /* * Target Mode related definitions */ #define QLTM_SENSELEN 18 /* non-FC cards only */ #define QLTM_SVALID 0x80 /* * Structure for Enable Lun and Modify Lun queue entries */ typedef struct { isphdr_t le_header; uint32_t le_reserved; uint8_t le_lun; uint8_t le_rsvd; uint8_t le_ops; /* Modify LUN only */ uint8_t le_tgt; /* Not for FC */ uint32_t le_flags; /* Not for FC */ uint8_t le_status; uint8_t le_reserved2; uint8_t le_cmd_count; uint8_t le_in_count; uint8_t le_cdb6len; /* Not for FC */ uint8_t le_cdb7len; /* Not for FC */ uint16_t le_timeout; uint16_t le_reserved3[20]; } lun_entry_t; /* * le_flags values */ #define LUN_TQAE 0x00000002 /* bit1 Tagged Queue Action Enable */ #define LUN_DSSM 0x01000000 /* bit24 Disable Sending SDP Message */ #define LUN_DISAD 0x02000000 /* bit25 Disable autodisconnect */ #define LUN_DM 0x40000000 /* bit30 Disconnects Mandatory */ /* * le_ops values */ #define LUN_CCINCR 0x01 /* increment command count */ #define LUN_CCDECR 0x02 /* decrement command count */ #define LUN_ININCR 0x40 /* increment immed. notify count */ #define LUN_INDECR 0x80 /* decrement immed. notify count */ /* * le_status values */ #define LUN_OK 0x01 /* we be rockin' */ #define LUN_ERR 0x04 /* request completed with error */ #define LUN_INVAL 0x06 /* invalid request */ #define LUN_NOCAP 0x16 /* can't provide requested capability */ #define LUN_ENABLED 0x3E /* LUN already enabled */ /* * Immediate Notify Entry structure */ #define IN_MSGLEN 8 /* 8 bytes */ #define IN_RSVDLEN 8 /* 8 words */ typedef struct { isphdr_t in_header; uint32_t in_reserved; uint8_t in_lun; /* lun */ uint8_t in_iid; /* initiator */ uint8_t in_reserved2; uint8_t in_tgt; /* target */ uint32_t in_flags; uint8_t in_status; uint8_t in_rsvd2; uint8_t in_tag_val; /* tag value */ uint8_t in_tag_type; /* tag type */ uint16_t in_seqid; /* sequence id */ uint8_t in_msg[IN_MSGLEN]; /* SCSI message bytes */ uint16_t in_reserved3[IN_RSVDLEN]; uint8_t in_sense[QLTM_SENSELEN];/* suggested sense data */ } in_entry_t; typedef struct { isphdr_t in_header; uint32_t in_reserved; uint8_t in_lun; /* lun */ uint8_t in_iid; /* initiator */ uint16_t in_scclun; uint32_t in_reserved2; uint16_t in_status; uint16_t in_task_flags; uint16_t in_seqid; /* sequence id */ } in_fcentry_t; typedef struct { isphdr_t in_header; uint32_t in_reserved; uint16_t in_iid; /* initiator */ uint16_t in_scclun; uint32_t in_reserved2; uint16_t in_status; uint16_t in_task_flags; uint16_t in_seqid; /* sequence id */ } in_fcentry_e_t; /* * Values for the in_status field */ #define IN_REJECT 0x0D /* Message Reject message received */ #define IN_RESET 0x0E /* Bus Reset occurred */ #define IN_NO_RCAP 0x16 /* requested capability not available */ #define IN_IDE_RECEIVED 0x33 /* Initiator Detected Error msg received */ #define IN_RSRC_UNAVAIL 0x34 /* resource unavailable */ #define IN_MSG_RECEIVED 0x36 /* SCSI message received */ #define IN_ABORT_TASK 0x20 /* task named in RX_ID is being aborted (FC) */ #define IN_PORT_LOGOUT 0x29 /* port has logged out (FC) */ #define IN_PORT_CHANGED 0x2A /* port changed */ #define IN_GLOBAL_LOGO 0x2E /* all ports logged out */ #define IN_NO_NEXUS 0x3B /* Nexus not established */ #define IN_SRR_RCVD 0x45 /* SRR received */ /* * Values for the in_task_flags field- should only get one at a time! */ #define TASK_FLAGS_RESERVED_MASK (0xe700) #define TASK_FLAGS_CLEAR_ACA (1<<14) #define TASK_FLAGS_TARGET_RESET (1<<13) #define TASK_FLAGS_LUN_RESET (1<<12) #define TASK_FLAGS_CLEAR_TASK_SET (1<<10) #define TASK_FLAGS_ABORT_TASK_SET (1<<9) /* * ISP24XX Immediate Notify */ typedef struct { isphdr_t in_header; uint32_t in_reserved; uint16_t in_nphdl; uint16_t in_reserved1; uint16_t in_flags; uint16_t in_srr_rxid; uint16_t in_status; uint8_t in_status_subcode; uint8_t in_fwhandle; uint32_t in_rxid; uint16_t in_srr_reloff_lo; uint16_t in_srr_reloff_hi; uint16_t in_srr_iu; uint16_t in_srr_oxid; /* * If bit 2 is set in in_flags, the N-Port and * handle tags are valid. If the received ELS is * a LOGO, then these tags contain the N Port ID * from the LOGO payload. If the received ELS * request is TPRLO, these tags contain the * Third Party Originator N Port ID. */ uint16_t in_nport_id_hi; #define in_prli_options in_nport_id_hi uint8_t in_nport_id_lo; uint8_t in_reserved3; uint16_t in_np_handle; uint8_t in_reserved4[12]; uint8_t in_reserved5; uint8_t in_vpidx; uint32_t in_reserved6; uint16_t in_portid_lo; uint8_t in_portid_hi; uint8_t in_reserved7; uint16_t in_reserved8; uint16_t in_oxid; } in_fcentry_24xx_t; #define IN24XX_FLAG_PUREX_IOCB 0x1 #define IN24XX_FLAG_GLOBAL_LOGOUT 0x2 #define IN24XX_FLAG_NPHDL_VALID 0x4 #define IN24XX_FLAG_N2N_PRLI 0x8 #define IN24XX_FLAG_PN_NN_VALID 0x10 #define IN24XX_LIP_RESET 0x0E #define IN24XX_LINK_RESET 0x0F #define IN24XX_PORT_LOGOUT 0x29 #define IN24XX_PORT_CHANGED 0x2A #define IN24XX_LINK_FAILED 0x2E #define IN24XX_SRR_RCVD 0x45 #define IN24XX_ELS_RCVD 0x46 /* * login-affectin ELS received- check * subcode for specific opcode */ /* * For f/w > 4.0.25, these offsets in the Immediate Notify contain * the WWNN/WWPN if the ELS is PLOGI, PDISC or ADISC. The WWN is in * Big Endian format. */ #define IN24XX_PRLI_WWNN_OFF 0x18 #define IN24XX_PRLI_WWPN_OFF 0x28 #define IN24XX_PLOGI_WWNN_OFF 0x20 #define IN24XX_PLOGI_WWPN_OFF 0x28 /* * For f/w > 4.0.25, this offset in the Immediate Notify contain * the WWPN if the ELS is LOGO. The WWN is in Big Endian format. */ #define IN24XX_LOGO_WWPN_OFF 0x28 /* * Immediate Notify Status Subcodes for IN24XX_PORT_LOGOUT */ #define IN24XX_PORT_LOGOUT_PDISC_TMO 0x00 #define IN24XX_PORT_LOGOUT_UXPR_DISC 0x01 #define IN24XX_PORT_LOGOUT_OWN_OPN 0x02 #define IN24XX_PORT_LOGOUT_OWN_OPN_SFT 0x03 #define IN24XX_PORT_LOGOUT_ABTS_TMO 0x04 #define IN24XX_PORT_LOGOUT_DISC_RJT 0x05 #define IN24XX_PORT_LOGOUT_LOGIN_NEEDED 0x06 #define IN24XX_PORT_LOGOUT_BAD_DISC 0x07 #define IN24XX_PORT_LOGOUT_LOST_ALPA 0x08 #define IN24XX_PORT_LOGOUT_XMIT_FAILURE 0x09 /* * Immediate Notify Status Subcodes for IN24XX_PORT_CHANGED */ #define IN24XX_PORT_CHANGED_BADFAN 0x00 #define IN24XX_PORT_CHANGED_TOPO_CHANGE 0x01 #define IN24XX_PORT_CHANGED_FLOGI_ACC 0x02 #define IN24XX_PORT_CHANGED_FLOGI_RJT 0x03 #define IN24XX_PORT_CHANGED_TIMEOUT 0x04 #define IN24XX_PORT_CHANGED_PORT_CHANGE 0x05 /* * Notify Acknowledge Entry structure */ #define NA_RSVDLEN 22 typedef struct { isphdr_t na_header; uint32_t na_reserved; uint8_t na_lun; /* lun */ uint8_t na_iid; /* initiator */ uint8_t na_reserved2; uint8_t na_tgt; /* target */ uint32_t na_flags; uint8_t na_status; uint8_t na_event; uint16_t na_seqid; /* sequence id */ uint16_t na_reserved3[NA_RSVDLEN]; } na_entry_t; /* * Value for the na_event field */ #define NA_RST_CLRD 0x80 /* Clear an async event notification */ #define NA_OK 0x01 /* Notify Acknowledge Succeeded */ #define NA_INVALID 0x06 /* Invalid Notify Acknowledge */ #define NA2_RSVDLEN 21 typedef struct { isphdr_t na_header; uint32_t na_reserved; uint8_t na_reserved1; uint8_t na_iid; /* initiator loop id */ uint16_t na_response; uint16_t na_flags; uint16_t na_reserved2; uint16_t na_status; uint16_t na_task_flags; uint16_t na_seqid; /* sequence id */ uint16_t na_reserved3[NA2_RSVDLEN]; } na_fcentry_t; typedef struct { isphdr_t na_header; uint32_t na_reserved; uint16_t na_iid; /* initiator loop id */ uint16_t na_response; /* response code */ uint16_t na_flags; uint16_t na_reserved2; uint16_t na_status; uint16_t na_task_flags; uint16_t na_seqid; /* sequence id */ uint16_t na_reserved3[NA2_RSVDLEN]; } na_fcentry_e_t; #define NAFC_RCOUNT 0x80 /* increment resource count */ #define NAFC_RST_CLRD 0x20 /* Clear LIP Reset */ #define NAFC_TVALID 0x10 /* task mangement response code is valid */ /* * ISP24XX Notify Acknowledge */ typedef struct { isphdr_t na_header; uint32_t na_handle; uint16_t na_nphdl; uint16_t na_reserved1; uint16_t na_flags; uint16_t na_srr_rxid; uint16_t na_status; uint8_t na_status_subcode; uint8_t na_fwhandle; uint32_t na_rxid; uint16_t na_srr_reloff_lo; uint16_t na_srr_reloff_hi; uint16_t na_srr_iu; uint16_t na_srr_flags; uint8_t na_reserved3[18]; uint8_t na_reserved4; uint8_t na_vpidx; uint8_t na_srr_reject_vunique; uint8_t na_srr_reject_explanation; uint8_t na_srr_reject_code; uint8_t na_reserved5; uint8_t na_reserved6[6]; uint16_t na_oxid; } na_fcentry_24xx_t; /* * Accept Target I/O Entry structure */ #define ATIO_CDBLEN 26 typedef struct { isphdr_t at_header; uint16_t at_reserved; uint16_t at_handle; uint8_t at_lun; /* lun */ uint8_t at_iid; /* initiator */ uint8_t at_cdblen; /* cdb length */ uint8_t at_tgt; /* target */ uint32_t at_flags; uint8_t at_status; /* firmware status */ uint8_t at_scsi_status; /* scsi status */ uint8_t at_tag_val; /* tag value */ uint8_t at_tag_type; /* tag type */ uint8_t at_cdb[ATIO_CDBLEN]; /* received CDB */ uint8_t at_sense[QLTM_SENSELEN];/* suggested sense data */ } at_entry_t; /* * at_flags values */ #define AT_NODISC 0x00008000 /* disconnect disabled */ #define AT_TQAE 0x00000002 /* Tagged Queue Action enabled */ /* * at_status values */ #define AT_PATH_INVALID 0x07 /* ATIO sent to firmware for disabled lun */ #define AT_RESET 0x0E /* SCSI Bus Reset Occurred */ #define AT_PHASE_ERROR 0x14 /* Bus phase sequence error */ #define AT_NOCAP 0x16 /* Requested capability not available */ #define AT_BDR_MSG 0x17 /* Bus Device Reset msg received */ #define AT_CDB 0x3D /* CDB received */ /* * Macros to create and fetch and test concatenated handle and tag value macros * (SPI only) */ #define AT_MAKE_TAGID(tid, aep) \ tid = aep->at_handle; \ if (aep->at_flags & AT_TQAE) { \ tid |= (aep->at_tag_val << 16); \ tid |= (1 << 24); \ } #define CT_MAKE_TAGID(tid, ct) \ tid = ct->ct_fwhandle; \ if (ct->ct_flags & CT_TQAE) { \ tid |= (ct->ct_tag_val << 16); \ tid |= (1 << 24); \ } #define AT_HAS_TAG(val) ((val) & (1 << 24)) #define AT_GET_TAG(val) (((val) >> 16) & 0xff) #define AT_GET_HANDLE(val) ((val) & 0xffff) #define IN_MAKE_TAGID(tid, inp) \ tid = inp->in_seqid; \ tid |= (inp->in_tag_val << 16); \ tid |= (1 << 24) /* * Accept Target I/O Entry structure, Type 2 */ #define ATIO2_CDBLEN 16 typedef struct { isphdr_t at_header; uint32_t at_reserved; uint8_t at_lun; /* lun or reserved */ uint8_t at_iid; /* initiator */ uint16_t at_rxid; /* response ID */ uint16_t at_flags; uint16_t at_status; /* firmware status */ uint8_t at_crn; /* command reference number */ uint8_t at_taskcodes; uint8_t at_taskflags; uint8_t at_execodes; uint8_t at_cdb[ATIO2_CDBLEN]; /* received CDB */ uint32_t at_datalen; /* allocated data len */ uint16_t at_scclun; /* SCC Lun or reserved */ uint16_t at_wwpn[4]; /* WWPN of initiator */ uint16_t at_reserved2[6]; uint16_t at_oxid; } at2_entry_t; typedef struct { isphdr_t at_header; uint32_t at_reserved; uint16_t at_iid; /* initiator */ uint16_t at_rxid; /* response ID */ uint16_t at_flags; uint16_t at_status; /* firmware status */ uint8_t at_crn; /* command reference number */ uint8_t at_taskcodes; uint8_t at_taskflags; uint8_t at_execodes; uint8_t at_cdb[ATIO2_CDBLEN]; /* received CDB */ uint32_t at_datalen; /* allocated data len */ uint16_t at_scclun; /* SCC Lun or reserved */ uint16_t at_wwpn[4]; /* WWPN of initiator */ uint16_t at_reserved2[6]; uint16_t at_oxid; } at2e_entry_t; #define ATIO2_WWPN_OFFSET 0x2A #define ATIO2_OXID_OFFSET 0x3E #define ATIO2_TC_ATTR_MASK 0x7 #define ATIO2_TC_ATTR_SIMPLEQ 0 #define ATIO2_TC_ATTR_HEADOFQ 1 #define ATIO2_TC_ATTR_ORDERED 2 #define ATIO2_TC_ATTR_ACAQ 4 #define ATIO2_TC_ATTR_UNTAGGED 5 #define ATIO2_EX_WRITE 0x1 #define ATIO2_EX_READ 0x2 /* * Macros to create and fetch and test concatenated handle and tag value macros */ #define AT2_MAKE_TAGID(tid, bus, inst, aep) \ tid = aep->at_rxid; \ tid |= (((uint64_t)inst) << 32); \ tid |= (((uint64_t)bus) << 48) #define CT2_MAKE_TAGID(tid, bus, inst, ct) \ tid = ct->ct_rxid; \ tid |= (((uint64_t)inst) << 32); \ tid |= (((uint64_t)(bus & 0xff)) << 48) #define AT2_HAS_TAG(val) 1 #define AT2_GET_TAG(val) ((val) & 0xffffffff) #define AT2_GET_INST(val) (((val) >> 32) & 0xffff) #define AT2_GET_HANDLE AT2_GET_TAG #define AT2_GET_BUS(val) (((val) >> 48) & 0xff) #define FC_HAS_TAG AT2_HAS_TAG #define FC_GET_TAG AT2_GET_TAG #define FC_GET_INST AT2_GET_INST #define FC_GET_HANDLE AT2_GET_HANDLE #define IN_FC_MAKE_TAGID(tid, bus, inst, seqid) \ tid = seqid; \ tid |= (((uint64_t)inst) << 32); \ tid |= (((uint64_t)(bus & 0xff)) << 48) #define FC_TAG_INSERT_INST(tid, inst) \ tid &= ~0x0000ffff00000000ull; \ tid |= (((uint64_t)inst) << 32) /* * 24XX ATIO Definition * * This is *quite* different from other entry types. * First of all, it has its own queue it comes in on. * * Secondly, it doesn't have a normal header. * * Thirdly, it's just a passthru of the FCP CMND IU * which is recorded in big endian mode. */ typedef struct { uint8_t at_type; uint8_t at_count; /* * Task attribute in high four bits, * the rest is the FCP CMND IU Length. * NB: the command can extend past the * length for a single queue entry. */ uint16_t at_ta_len; uint32_t at_rxid; fc_hdr_t at_hdr; fcp_cmnd_iu_t at_cmnd; } at7_entry_t; #define AT7_NORESRC_RXID 0xffffffff /* * Continue Target I/O Entry structure * Request from driver. The response from the * ISP firmware is the same except that the last 18 * bytes are overwritten by suggested sense data if * the 'autosense valid' bit is set in the status byte. */ typedef struct { isphdr_t ct_header; uint16_t ct_syshandle; uint16_t ct_fwhandle; /* required by f/w */ uint8_t ct_lun; /* lun */ uint8_t ct_iid; /* initiator id */ uint8_t ct_reserved2; uint8_t ct_tgt; /* our target id */ uint32_t ct_flags; uint8_t ct_status; /* isp status */ uint8_t ct_scsi_status; /* scsi status */ uint8_t ct_tag_val; /* tag value */ uint8_t ct_tag_type; /* tag type */ uint32_t ct_xfrlen; /* transfer length */ uint32_t ct_resid; /* residual length */ uint16_t ct_timeout; uint16_t ct_seg_count; ispds_t ct_dataseg[ISP_RQDSEG]; } ct_entry_t; /* * For some of the dual port SCSI adapters, port (bus #) is reported * in the MSbit of ct_iid. Bit fields are a bit too awkward here. * * Note that this does not apply to FC adapters at all which can and * do report IIDs between 0x81 && 0xfe (or 0x7ff) which represent devices * that have logged in across a SCSI fabric. */ #define GET_IID_VAL(x) (x & 0x3f) #define GET_BUS_VAL(x) ((x >> 7) & 0x1) #define SET_IID_VAL(y, x) y = ((y & ~0x3f) | (x & 0x3f)) #define SET_BUS_VAL(y, x) y = ((y & 0x3f) | ((x & 0x1) << 7)) /* * ct_flags values */ #define CT_TQAE 0x00000002 /* bit 1, Tagged Queue Action enable */ #define CT_DATA_IN 0x00000040 /* bits 6&7, Data direction - *to* initiator */ #define CT_DATA_OUT 0x00000080 /* bits 6&7, Data direction - *from* initiator */ #define CT_NO_DATA 0x000000C0 /* bits 6&7, Data direction */ #define CT_CCINCR 0x00000100 /* bit 8, autoincrement atio count */ #define CT_DATAMASK 0x000000C0 /* bits 6&7, Data direction */ #define CT_INISYNCWIDE 0x00004000 /* bit 14, Do Sync/Wide Negotiation */ #define CT_NODISC 0x00008000 /* bit 15, Disconnects disabled */ #define CT_DSDP 0x01000000 /* bit 24, Disable Save Data Pointers */ #define CT_SENDRDP 0x04000000 /* bit 26, Send Restore Pointers msg */ #define CT_SENDSTATUS 0x80000000 /* bit 31, Send SCSI status byte */ /* * ct_status values * - set by the firmware when it returns the CTIO */ #define CT_OK 0x01 /* completed without error */ #define CT_ABORTED 0x02 /* aborted by host */ #define CT_ERR 0x04 /* see sense data for error */ #define CT_INVAL 0x06 /* request for disabled lun */ #define CT_NOPATH 0x07 /* invalid ITL nexus */ #define CT_INVRXID 0x08 /* (FC only) Invalid RX_ID */ #define CT_DATA_OVER 0x09 /* (FC only) Data Overrun */ #define CT_RSELTMO 0x0A /* reselection timeout after 2 tries */ #define CT_TIMEOUT 0x0B /* timed out */ #define CT_RESET 0x0E /* SCSI Bus Reset occurred */ #define CT_PARITY 0x0F /* Uncorrectable Parity Error */ #define CT_BUS_ERROR 0x10 /* (FC Only) DMA PCI Error */ #define CT_PANIC 0x13 /* Unrecoverable Error */ #define CT_PHASE_ERROR 0x14 /* Bus phase sequence error */ #define CT_DATA_UNDER 0x15 /* (FC only) Data Underrun */ #define CT_BDR_MSG 0x17 /* Bus Device Reset msg received */ #define CT_TERMINATED 0x19 /* due to Terminate Transfer mbox cmd */ #define CT_PORTUNAVAIL 0x28 /* port not available */ #define CT_LOGOUT 0x29 /* port logout */ #define CT_PORTCHANGED 0x2A /* port changed */ #define CT_IDE 0x33 /* Initiator Detected Error */ #define CT_NOACK 0x35 /* Outstanding Immed. Notify. entry */ #define CT_SRR 0x45 /* SRR Received */ #define CT_LUN_RESET 0x48 /* Lun Reset Received */ #define CT_HBA_RESET 0xffff /* pseudo error - command destroyed by HBA reset*/ /* * When the firmware returns a CTIO entry, it may overwrite the last * part of the structure with sense data. This starts at offset 0x2E * into the entry, which is in the middle of ct_dataseg[1]. Rather * than define a new struct for this, I'm just using the sense data * offset. */ #define CTIO_SENSE_OFFSET 0x2E /* * Entry length in u_longs. All entries are the same size so * any one will do as the numerator. */ #define UINT32_ENTRY_SIZE (sizeof(at_entry_t)/sizeof(uint32_t)) /* * QLA2100 CTIO (type 2) entry */ #define MAXRESPLEN 26 typedef struct { isphdr_t ct_header; uint32_t ct_syshandle; uint8_t ct_lun; /* lun */ uint8_t ct_iid; /* initiator id */ uint16_t ct_rxid; /* response ID */ uint16_t ct_flags; uint16_t ct_status; /* isp status */ uint16_t ct_timeout; uint16_t ct_seg_count; uint32_t ct_reloff; /* relative offset */ uint32_t ct_resid; /* residual length */ union { /* * The three different modes that the target driver * can set the CTIO{2,3,4} up as. * * The first is for sending FCP_DATA_IUs as well as * (optionally) sending a terminal SCSI status FCP_RSP_IU. * * The second is for sending SCSI sense data in an FCP_RSP_IU. * Note that no FCP_DATA_IUs will be sent. * * The third is for sending FCP_RSP_IUs as built specifically * in system memory as located by the isp_dataseg. */ struct { uint32_t _reserved; uint16_t _reserved2; uint16_t ct_scsi_status; uint32_t ct_xfrlen; union { ispds_t ct_dataseg[ISP_RQDSEG_T2]; ispds64_t ct_dataseg64[ISP_RQDSEG_T3]; ispdslist_t ct_dslist; } u; } m0; struct { uint16_t _reserved; uint16_t _reserved2; uint16_t ct_senselen; uint16_t ct_scsi_status; uint16_t ct_resplen; uint8_t ct_resp[MAXRESPLEN]; } m1; struct { uint32_t _reserved; uint16_t _reserved2; uint16_t _reserved3; uint32_t ct_datalen; union { ispds_t ct_fcp_rsp_iudata_32; ispds64_t ct_fcp_rsp_iudata_64; } u; } m2; } rsp; } ct2_entry_t; typedef struct { isphdr_t ct_header; uint32_t ct_syshandle; uint16_t ct_iid; /* initiator id */ uint16_t ct_rxid; /* response ID */ uint16_t ct_flags; uint16_t ct_status; /* isp status */ uint16_t ct_timeout; uint16_t ct_seg_count; uint32_t ct_reloff; /* relative offset */ uint32_t ct_resid; /* residual length */ union { struct { uint32_t _reserved; uint16_t _reserved2; uint16_t ct_scsi_status; uint32_t ct_xfrlen; union { ispds_t ct_dataseg[ISP_RQDSEG_T2]; ispds64_t ct_dataseg64[ISP_RQDSEG_T3]; ispdslist_t ct_dslist; } u; } m0; struct { uint16_t _reserved; uint16_t _reserved2; uint16_t ct_senselen; uint16_t ct_scsi_status; uint16_t ct_resplen; uint8_t ct_resp[MAXRESPLEN]; } m1; struct { uint32_t _reserved; uint16_t _reserved2; uint16_t _reserved3; uint32_t ct_datalen; union { ispds_t ct_fcp_rsp_iudata_32; ispds64_t ct_fcp_rsp_iudata_64; } u; } m2; } rsp; } ct2e_entry_t; /* * ct_flags values for CTIO2 */ #define CT2_FLAG_MODE0 0x0000 #define CT2_FLAG_MODE1 0x0001 #define CT2_FLAG_MODE2 0x0002 #define CT2_FLAG_MMASK 0x0003 #define CT2_DATA_IN 0x0040 /* *to* initiator */ #define CT2_DATA_OUT 0x0080 /* *from* initiator */ #define CT2_NO_DATA 0x00C0 #define CT2_DATAMASK 0x00C0 #define CT2_CCINCR 0x0100 #define CT2_FASTPOST 0x0200 #define CT2_CONFIRM 0x2000 #define CT2_TERMINATE 0x4000 #define CT2_SENDSTATUS 0x8000 /* * ct_status values are (mostly) the same as that for ct_entry. */ /* * ct_scsi_status values- the low 8 bits are the normal SCSI status * we know and love. The upper 8 bits are validity markers for FCP_RSP_IU * fields. */ #define CT2_RSPLEN_VALID 0x0100 #define CT2_SNSLEN_VALID 0x0200 #define CT2_DATA_OVER 0x0400 #define CT2_DATA_UNDER 0x0800 /* * ISP24XX CTIO */ #define MAXRESPLEN_24XX 24 typedef struct { isphdr_t ct_header; uint32_t ct_syshandle; uint16_t ct_nphdl; /* status on returned CTIOs */ uint16_t ct_timeout; uint16_t ct_seg_count; uint8_t ct_vpidx; uint8_t ct_xflags; uint16_t ct_iid_lo; /* low 16 bits of portid */ uint8_t ct_iid_hi; /* hi 8 bits of portid */ uint8_t ct_reserved; uint32_t ct_rxid; uint16_t ct_senselen; /* mode 1 only */ uint16_t ct_flags; uint32_t ct_resid; /* residual length */ uint16_t ct_oxid; uint16_t ct_scsi_status; /* modes 0 && 1 only */ union { struct { uint32_t reloff; uint32_t reserved0; uint32_t ct_xfrlen; uint32_t reserved1; ispds64_t ds; } m0; struct { uint16_t ct_resplen; uint16_t reserved; uint8_t ct_resp[MAXRESPLEN_24XX]; } m1; struct { uint32_t reserved0; uint32_t reserved1; uint32_t ct_datalen; uint32_t reserved2; ispds64_t ct_fcp_rsp_iudata; } m2; } rsp; } ct7_entry_t; /* * ct_flags values for CTIO7 */ #define CT7_NO_DATA 0x0000 #define CT7_DATA_OUT 0x0001 /* *from* initiator */ #define CT7_DATA_IN 0x0002 /* *to* initiator */ #define CT7_DATAMASK 0x3 #define CT7_DSD_ENABLE 0x0004 #define CT7_CONF_STSFD 0x0010 #define CT7_EXPLCT_CONF 0x0020 #define CT7_FLAG_MODE0 0x0000 #define CT7_FLAG_MODE1 0x0040 #define CT7_FLAG_MODE2 0x0080 #define CT7_FLAG_MMASK 0x00C0 #define CT7_NOACK 0x0100 #define CT7_TASK_ATTR_SHIFT 9 #define CT7_CONFIRM 0x2000 #define CT7_TERMINATE 0x4000 #define CT7_SENDSTATUS 0x8000 /* * Type 7 CTIO status codes */ #define CT7_OK 0x01 /* completed without error */ #define CT7_ABORTED 0x02 /* aborted by host */ #define CT7_ERR 0x04 /* see sense data for error */ #define CT7_INVAL 0x06 /* request for disabled lun */ #define CT7_INVRXID 0x08 /* Invalid RX_ID */ #define CT7_DATA_OVER 0x09 /* Data Overrun */ #define CT7_TIMEOUT 0x0B /* timed out */ #define CT7_RESET 0x0E /* LIP Rset Received */ #define CT7_BUS_ERROR 0x10 /* DMA PCI Error */ #define CT7_REASSY_ERR 0x11 /* DMA reassembly error */ #define CT7_DATA_UNDER 0x15 /* Data Underrun */ #define CT7_PORTUNAVAIL 0x28 /* port not available */ #define CT7_LOGOUT 0x29 /* port logout */ #define CT7_PORTCHANGED 0x2A /* port changed */ #define CT7_SRR 0x45 /* SRR Received */ /* * Other 24XX related target IOCBs */ /* * ABTS Received */ typedef struct { isphdr_t abts_header; uint8_t abts_reserved0[6]; uint16_t abts_nphdl; uint16_t abts_reserved1; uint16_t abts_sof; uint32_t abts_rxid_abts; uint16_t abts_did_lo; uint8_t abts_did_hi; uint8_t abts_r_ctl; uint16_t abts_sid_lo; uint8_t abts_sid_hi; uint8_t abts_cs_ctl; uint16_t abts_fs_ctl; uint8_t abts_f_ctl; uint8_t abts_type; uint16_t abts_seq_cnt; uint8_t abts_df_ctl; uint8_t abts_seq_id; uint16_t abts_rx_id; uint16_t abts_ox_id; uint32_t abts_param; uint8_t abts_reserved2[16]; uint32_t abts_rxid_task; } abts_t; typedef struct { isphdr_t abts_rsp_header; uint32_t abts_rsp_handle; uint16_t abts_rsp_status; uint16_t abts_rsp_nphdl; uint16_t abts_rsp_ctl_flags; uint16_t abts_rsp_sof; uint32_t abts_rsp_rxid_abts; uint16_t abts_rsp_did_lo; uint8_t abts_rsp_did_hi; uint8_t abts_rsp_r_ctl; uint16_t abts_rsp_sid_lo; uint8_t abts_rsp_sid_hi; uint8_t abts_rsp_cs_ctl; uint16_t abts_rsp_f_ctl_lo; uint8_t abts_rsp_f_ctl_hi; uint8_t abts_rsp_type; uint16_t abts_rsp_seq_cnt; uint8_t abts_rsp_df_ctl; uint8_t abts_rsp_seq_id; uint16_t abts_rsp_rx_id; uint16_t abts_rsp_ox_id; uint32_t abts_rsp_param; union { struct { uint16_t reserved; uint8_t last_seq_id; uint8_t seq_id_valid; uint16_t aborted_rx_id; uint16_t aborted_ox_id; uint16_t high_seq_cnt; uint16_t low_seq_cnt; uint8_t reserved2[4]; } ba_acc; struct { uint8_t vendor_unique; uint8_t explanation; uint8_t reason; uint8_t reserved; uint8_t reserved2[12]; } ba_rjt; struct { uint8_t reserved[8]; uint32_t subcode1; uint32_t subcode2; } rsp; uint8_t reserved[16]; } abts_rsp_payload; uint32_t abts_rsp_rxid_task; } abts_rsp_t; /* terminate this ABTS exchange */ #define ISP24XX_ABTS_RSP_TERMINATE 0x01 #define ISP24XX_ABTS_RSP_COMPLETE 0x00 #define ISP24XX_ABTS_RSP_RESET 0x04 #define ISP24XX_ABTS_RSP_ABORTED 0x05 #define ISP24XX_ABTS_RSP_TIMEOUT 0x06 #define ISP24XX_ABTS_RSP_INVXID 0x08 #define ISP24XX_ABTS_RSP_LOGOUT 0x29 #define ISP24XX_ABTS_RSP_SUBCODE 0x31 #define ISP24XX_NO_TASK 0xffffffff /* * Miscellaneous * * These are the limits of the number of dma segments we * can deal with based not on the size of the segment counter * (which is 16 bits), but on the size of the number of * queue entries field (which is 8 bits). We assume no * segments in the first queue entry, so we can either * have 7 dma segments per continuation entry or 5 * (for 64 bit dma).. multiplying out by 254.... */ #define ISP_NSEG_MAX 1778 #define ISP_NSEG64_MAX 1270 #endif /* _ISPMBOX_H */ Index: projects/import-googletest-1.8.1/sys/dev/isp/ispvar.h =================================================================== --- projects/import-googletest-1.8.1/sys/dev/isp/ispvar.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/isp/ispvar.h (revision 345026) @@ -1,1142 +1,1147 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2018 Alexander Motin * Copyright (c) 1997-2009 by Matthew Jacob * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Soft Definitions for for Qlogic ISP SCSI adapters. */ #ifndef _ISPVAR_H #define _ISPVAR_H #if defined(__NetBSD__) || defined(__OpenBSD__) #include #include #endif #ifdef __FreeBSD__ #include #include #endif #ifdef __linux__ #include "isp_stds.h" #include "ispmbox.h" #endif #ifdef __svr4__ #include "isp_stds.h" #include "ispmbox.h" #endif #define ISP_CORE_VERSION_MAJOR 7 #define ISP_CORE_VERSION_MINOR 0 /* * Vector for bus specific code to provide specific services. */ typedef struct ispsoftc ispsoftc_t; struct ispmdvec { void (*dv_run_isr) (ispsoftc_t *); uint32_t (*dv_rd_reg) (ispsoftc_t *, int); void (*dv_wr_reg) (ispsoftc_t *, int, uint32_t); int (*dv_mbxdma) (ispsoftc_t *); int (*dv_dmaset) (ispsoftc_t *, XS_T *, void *); void (*dv_dmaclr) (ispsoftc_t *, XS_T *, uint32_t); int (*dv_irqsetup) (ispsoftc_t *); void (*dv_dregs) (ispsoftc_t *, const char *); const void * dv_ispfw; /* ptr to f/w */ uint16_t dv_conf1; uint16_t dv_clock; /* clock frequency */ }; /* * Overall parameters */ #define MAX_TARGETS 16 #ifndef MAX_FC_TARG #define MAX_FC_TARG 1024 #endif #define ISP_MAX_TARGETS(isp) (IS_FC(isp)? MAX_FC_TARG : MAX_TARGETS) #define ISP_MAX_LUNS(isp) (isp)->isp_maxluns #define ISP_MAX_IRQS 3 /* * Macros to access ISP registers through bus specific layers- * mostly wrappers to vector through the mdvec structure. */ #define ISP_RUN_ISR(isp) \ (*(isp)->isp_mdvec->dv_run_isr)(isp) #define ISP_READ(isp, reg) \ (*(isp)->isp_mdvec->dv_rd_reg)((isp), (reg)) #define ISP_WRITE(isp, reg, val) \ (*(isp)->isp_mdvec->dv_wr_reg)((isp), (reg), (val)) #define ISP_MBOXDMASETUP(isp) \ (*(isp)->isp_mdvec->dv_mbxdma)((isp)) #define ISP_DMASETUP(isp, xs, req) \ (*(isp)->isp_mdvec->dv_dmaset)((isp), (xs), (req)) #define ISP_DMAFREE(isp, xs, hndl) \ if ((isp)->isp_mdvec->dv_dmaclr) \ (*(isp)->isp_mdvec->dv_dmaclr)((isp), (xs), (hndl)) #define ISP_IRQSETUP(isp) \ (((isp)->isp_mdvec->dv_irqsetup) ? (*(isp)->isp_mdvec->dv_irqsetup)(isp) : 0) #define ISP_DUMPREGS(isp, m) \ if ((isp)->isp_mdvec->dv_dregs) (*(isp)->isp_mdvec->dv_dregs)((isp),(m)) #define ISP_SETBITS(isp, reg, val) \ (*(isp)->isp_mdvec->dv_wr_reg)((isp), (reg), ISP_READ((isp), (reg)) | (val)) #define ISP_CLRBITS(isp, reg, val) \ (*(isp)->isp_mdvec->dv_wr_reg)((isp), (reg), ISP_READ((isp), (reg)) & ~(val)) /* * The MEMORYBARRIER macro is defined per platform (to provide synchronization * on Request and Response Queues, Scratch DMA areas, and Registers) * * Defined Memory Barrier Synchronization Types */ #define SYNC_REQUEST 0 /* request queue synchronization */ #define SYNC_RESULT 1 /* result queue synchronization */ #define SYNC_SFORDEV 2 /* scratch, sync for ISP */ #define SYNC_SFORCPU 3 /* scratch, sync for CPU */ #define SYNC_REG 4 /* for registers */ #define SYNC_ATIOQ 5 /* atio result queue (24xx) */ #define SYNC_IFORDEV 6 /* synchrounous IOCB, sync for ISP */ #define SYNC_IFORCPU 7 /* synchrounous IOCB, sync for CPU */ /* * Request/Response Queue defines and macros. * The maximum is defined per platform (and can be based on board type). */ /* This is the size of a queue entry (request and response) */ #define QENTRY_LEN 64 /* Both request and result queue length must be a power of two */ #define RQUEST_QUEUE_LEN(x) MAXISPREQUEST(x) #ifdef ISP_TARGET_MODE #define RESULT_QUEUE_LEN(x) MAXISPREQUEST(x) #else #define RESULT_QUEUE_LEN(x) \ (((MAXISPREQUEST(x) >> 2) < 64)? 64 : MAXISPREQUEST(x) >> 2) #endif #define ISP_QUEUE_ENTRY(q, idx) (((uint8_t *)q) + ((idx) * QENTRY_LEN)) #define ISP_QUEUE_SIZE(n) ((n) * QENTRY_LEN) #define ISP_NXT_QENTRY(idx, qlen) (((idx) + 1) & ((qlen)-1)) #define ISP_QFREE(in, out, qlen) \ ((in == out)? (qlen - 1) : ((in > out)? \ ((qlen - 1) - (in - out)) : (out - in - 1))) #define ISP_QAVAIL(isp) \ ISP_QFREE(isp->isp_reqidx, isp->isp_reqodx, RQUEST_QUEUE_LEN(isp)) #define ISP_ADD_REQUEST(isp, nxti) \ MEMORYBARRIER(isp, SYNC_REQUEST, isp->isp_reqidx, QENTRY_LEN, -1); \ ISP_WRITE(isp, isp->isp_rqstinrp, nxti); \ isp->isp_reqidx = nxti #define ISP_SYNC_REQUEST(isp) \ MEMORYBARRIER(isp, SYNC_REQUEST, isp->isp_reqidx, QENTRY_LEN, -1); \ isp->isp_reqidx = ISP_NXT_QENTRY(isp->isp_reqidx, RQUEST_QUEUE_LEN(isp)); \ ISP_WRITE(isp, isp->isp_rqstinrp, isp->isp_reqidx) /* * SCSI Specific Host Adapter Parameters- per bus, per target */ typedef struct { uint32_t : 8, update : 1, sendmarker : 1, isp_req_ack_active_neg : 1, isp_data_line_active_neg: 1, isp_cmd_dma_burst_enable: 1, isp_data_dma_burst_enabl: 1, isp_fifo_threshold : 3, isp_ptisp : 1, isp_ultramode : 1, isp_diffmode : 1, isp_lvdmode : 1, isp_fast_mttr : 1, /* fast sram */ isp_initiator_id : 4, isp_async_data_setup : 4; uint16_t isp_selection_timeout; uint16_t isp_max_queue_depth; uint8_t isp_tag_aging; uint8_t isp_bus_reset_delay; uint8_t isp_retry_count; uint8_t isp_retry_delay; struct { uint32_t exc_throttle : 8, : 1, dev_enable : 1, /* ignored */ dev_update : 1, dev_refresh : 1, actv_offset : 4, goal_offset : 4, nvrm_offset : 4; uint8_t actv_period; /* current sync period */ uint8_t goal_period; /* goal sync period */ uint8_t nvrm_period; /* nvram sync period */ uint16_t actv_flags; /* current device flags */ uint16_t goal_flags; /* goal device flags */ uint16_t nvrm_flags; /* nvram device flags */ } isp_devparam[MAX_TARGETS]; } sdparam; /* * Device Flags */ #define DPARM_DISC 0x8000 #define DPARM_PARITY 0x4000 #define DPARM_WIDE 0x2000 #define DPARM_SYNC 0x1000 #define DPARM_TQING 0x0800 #define DPARM_ARQ 0x0400 #define DPARM_QFRZ 0x0200 #define DPARM_RENEG 0x0100 #define DPARM_NARROW 0x0080 #define DPARM_ASYNC 0x0040 #define DPARM_PPR 0x0020 #define DPARM_DEFAULT (0xFF00 & ~DPARM_QFRZ) #define DPARM_SAFE_DFLT (DPARM_DEFAULT & ~(DPARM_WIDE|DPARM_SYNC|DPARM_TQING)) /* technically, not really correct, as they need to be rated based upon clock */ #define ISP_80M_SYNCPARMS 0x0c09 #define ISP_40M_SYNCPARMS 0x0c0a #define ISP_20M_SYNCPARMS 0x0c0c #define ISP_20M_SYNCPARMS_1040 0x080c #define ISP_10M_SYNCPARMS 0x0c19 #define ISP_08M_SYNCPARMS 0x0c25 #define ISP_05M_SYNCPARMS 0x0c32 #define ISP_04M_SYNCPARMS 0x0c41 /* * Fibre Channel Specifics */ /* These are for non-2K Login Firmware cards */ #define FL_ID 0x7e /* FL_Port Special ID */ #define SNS_ID 0x80 /* SNS Server Special ID */ #define NPH_MAX 0xfe /* These are for 2K Login Firmware cards */ #define NPH_RESERVED 0x7F0 /* begin of reserved N-port handles */ #define NPH_MGT_ID 0x7FA /* Management Server Special ID */ #define NPH_SNS_ID 0x7FC /* SNS Server Special ID */ #define NPH_FABRIC_CTLR 0x7FD /* Fabric Controller (0xFFFFFD) */ #define NPH_FL_ID 0x7FE /* F Port Special ID (0xFFFFFE) */ #define NPH_IP_BCST 0x7FF /* IP Broadcast Special ID (0xFFFFFF) */ #define NPH_MAX_2K 0x800 /* * "Unassigned" handle to be used internally */ #define NIL_HANDLE 0xffff /* * Limit for devices on an arbitrated loop. */ #define LOCAL_LOOP_LIM 126 /* * Limit for (2K login) N-port handle amounts */ #define MAX_NPORT_HANDLE 2048 /* * Special Constants */ #define INI_NONE ((uint64_t) 0) #define ISP_NOCHAN 0xff /* * Special Port IDs */ #define MANAGEMENT_PORT_ID 0xFFFFFA #define SNS_PORT_ID 0xFFFFFC #define FABRIC_PORT_ID 0xFFFFFE #define PORT_ANY 0xFFFFFF #define PORT_NONE 0 #define VALID_PORT(port) (port != PORT_NONE && port != PORT_ANY) #define DOMAIN_CONTROLLER_BASE 0xFFFC00 #define DOMAIN_CONTROLLER_END 0xFFFCFF /* * Command Handles * * Most QLogic initiator or target have 32 bit handles associated with them. * We want to have a quick way to index back and forth between a local SCSI * command context and what the firmware is passing back to us. We also * want to avoid working on stale information. This structure handles both * at the expense of some local memory. * * The handle is architected thusly: * * 0 means "free handle" * bits 0..12 index commands * bits 13..15 bits index usage * bits 16..31 contain a rolling sequence * * */ typedef struct { void * cmd; /* associated command context */ uint32_t handle; /* handle associated with this command */ } isp_hdl_t; #define ISP_HANDLE_FREE 0x00000000 #define ISP_HANDLE_CMD_MASK 0x00001fff #define ISP_HANDLE_USAGE_MASK 0x0000e000 #define ISP_HANDLE_USAGE_SHIFT 13 #define ISP_H2HT(hdl) ((hdl & ISP_HANDLE_USAGE_MASK) >> ISP_HANDLE_USAGE_SHIFT) # define ISP_HANDLE_NONE 0 # define ISP_HANDLE_INITIATOR 1 # define ISP_HANDLE_TARGET 2 # define ISP_HANDLE_CTRL 3 #define ISP_HANDLE_SEQ_MASK 0xffff0000 #define ISP_HANDLE_SEQ_SHIFT 16 #define ISP_H2SEQ(hdl) ((hdl & ISP_HANDLE_SEQ_MASK) >> ISP_HANDLE_SEQ_SHIFT) #define ISP_VALID_HANDLE(c, hdl) \ ((ISP_H2HT(hdl) == ISP_HANDLE_INITIATOR || \ ISP_H2HT(hdl) == ISP_HANDLE_TARGET || \ ISP_H2HT(hdl) == ISP_HANDLE_CTRL) && \ ((hdl) & ISP_HANDLE_CMD_MASK) < (c)->isp_maxcmds && \ (hdl) == ((c)->isp_xflist[(hdl) & ISP_HANDLE_CMD_MASK].handle)) #define ISP_BAD_HANDLE_INDEX 0xffffffff /* * FC Port Database entry. * * It has a handle that the f/w uses to address commands to a device. * This handle's value may be assigned by the firmware (e.g., for local loop * devices) or by the driver (e.g., for fabric devices). * * It has a state. If the state if VALID, that means that we've logged into * the device. * * Local loop devices the firmware automatically performs PLOGI on for us * (which is why that handle is imposed upon us). Fabric devices we assign * a handle to and perform the PLOGI on. * * When a PORT DATABASE CHANGED asynchronous event occurs, we mark all VALID * entries as PROBATIONAL. This allows us, if policy says to, just keep track * of devices whose handles change but are otherwise the same device (and * thus keep 'target' constant). * * In any case, we search all possible local loop handles. For each one that * has a port database entity returned, we search for any PROBATIONAL entry * that matches it and update as appropriate. Otherwise, as a new entry, we * find room for it in the Port Database. We *try* and use the handle as the * index to put it into the Database, but that's just an optimization. We mark * the entry VALID and make sure that the target index is updated and correct. * * When we get done searching the local loop, we then search similarly for * a list of devices we've gotten from the fabric name controller (if we're * on a fabric). VALID marking is also done similarly. * * When all of this is done, we can march through the database and clean up * any entry that is still PROBATIONAL (these represent devices which have * departed). Then we're done and can resume normal operations. * * Negative invariants that we try and test for are: * * + There can never be two non-NIL entries with the same { Port, Node } WWN * duples. * * + There can never be two non-NIL entries with the same handle. */ typedef struct { /* * This is the handle that the firmware needs in order for us to * send commands to the device. For pre-24XX cards, this would be * the 'loopid'. */ uint16_t handle; /* + * PRLI word 0 contains the Establish Image Pair bit, which is + * important for knowing when to reset the CRN. + * * PRLI word 3 parameters contains role as well as other things. * * The state is the current state of this entry. * * The is_target is the current state of target on this port. * * The is_initiator is the current state of initiator on this port. * * Portid is obvious, as are node && port WWNs. The new_role and * new_portid is for when we are pending a change. */ + uint16_t prli_word0; /* PRLI parameters */ uint16_t prli_word3; /* PRLI parameters */ + uint16_t new_prli_word0; /* Incoming new PRLI parameters */ uint16_t new_prli_word3; /* Incoming new PRLI parameters */ uint16_t : 12, probational : 1, state : 3; uint32_t : 6, is_target : 1, is_initiator : 1, portid : 24; uint32_t : 8, new_portid : 24; uint64_t node_wwn; uint64_t port_wwn; uint32_t gone_timer; } fcportdb_t; #define FC_PORTDB_STATE_NIL 0 /* Empty DB slot */ #define FC_PORTDB_STATE_DEAD 1 /* Was valid, but no more. */ #define FC_PORTDB_STATE_CHANGED 2 /* Was valid, but changed. */ #define FC_PORTDB_STATE_NEW 3 /* Logged in, not announced. */ #define FC_PORTDB_STATE_ZOMBIE 4 /* Invalid, but announced. */ #define FC_PORTDB_STATE_VALID 5 /* Valid */ #define FC_PORTDB_TGT(isp, bus, pdb) (int)(lp - FCPARAM(isp, bus)->portdb) /* * FC card specific information * * This structure is replicated across multiple channels for multi-id * capapble chipsets, with some entities different on a per-channel basis. */ typedef struct { int isp_gbspeed; /* Connection speed */ int isp_linkstate; /* Link state */ int isp_fwstate; /* ISP F/W state */ int isp_loopstate; /* Loop State */ int isp_topo; /* Connection Type */ uint32_t : 4, fctape_enabled : 1, sendmarker : 1, role : 2, isp_portid : 24; /* S_ID */ uint16_t isp_fwoptions; uint16_t isp_xfwoptions; uint16_t isp_zfwoptions; uint16_t isp_loopid; /* hard loop id */ uint16_t isp_sns_hdl; /* N-port handle for SNS */ uint16_t isp_lasthdl; /* only valid for channel 0 */ uint16_t isp_maxalloc; uint16_t isp_fabric_params; uint16_t isp_login_hdl; /* Logging in handle */ uint8_t isp_retry_delay; uint8_t isp_retry_count; int isp_use_gft_id; /* Use GFT_ID */ int isp_use_gff_id; /* Use GFF_ID */ /* * Current active WWNN/WWPN */ uint64_t isp_wwnn; uint64_t isp_wwpn; /* * NVRAM WWNN/WWPN */ uint64_t isp_wwnn_nvram; uint64_t isp_wwpn_nvram; /* * Our Port Data Base */ fcportdb_t portdb[MAX_FC_TARG]; /* * Scratch DMA mapped in area to fetch Port Database stuff, etc. */ void * isp_scratch; XS_DMA_ADDR_T isp_scdma; uint8_t isp_scanscratch[ISP_FC_SCRLEN]; } fcparam; #define FW_CONFIG_WAIT 0 #define FW_WAIT_LINK 1 #define FW_WAIT_LOGIN 2 #define FW_READY 3 #define FW_LOSS_OF_SYNC 4 #define FW_ERROR 5 #define FW_REINIT 6 #define FW_NON_PART 7 #define LOOP_NIL 0 #define LOOP_HAVE_LINK 1 #define LOOP_HAVE_ADDR 2 #define LOOP_TESTING_LINK 3 #define LOOP_LTEST_DONE 4 #define LOOP_SCANNING_LOOP 5 #define LOOP_LSCAN_DONE 6 #define LOOP_SCANNING_FABRIC 7 #define LOOP_FSCAN_DONE 8 #define LOOP_SYNCING_PDB 9 #define LOOP_READY 10 #define TOPO_NL_PORT 0 #define TOPO_FL_PORT 1 #define TOPO_N_PORT 2 #define TOPO_F_PORT 3 #define TOPO_PTP_STUB 4 #define TOPO_IS_FABRIC(x) ((x) == TOPO_FL_PORT || (x) == TOPO_F_PORT) #define FCP_AL_DA_ALL 0xFF #define FCP_AL_PA(fcp) ((uint8_t)(fcp->isp_portid)) #define FCP_IS_DEST_ALPD(fcp, alpd) (FCP_AL_PA((fcp)) == FCP_AL_DA_ALL || FCP_AL_PA((fcp)) == alpd) /* * Soft Structure per host adapter */ struct ispsoftc { /* * Platform (OS) specific data */ struct isposinfo isp_osinfo; /* * Pointer to bus specific functions and data */ struct ispmdvec * isp_mdvec; /* * (Mostly) nonvolatile state. Board specific parameters * may contain some volatile state (e.g., current loop state). */ void * isp_param; /* type specific */ uint64_t isp_fwattr; /* firmware attributes */ uint16_t isp_fwrev[3]; /* Loaded F/W revision */ uint16_t isp_maxcmds; /* max possible I/O cmds */ uint8_t isp_type; /* HBA Chip Type */ uint8_t isp_revision; /* HBA Chip H/W Revision */ uint8_t isp_nirq; /* number of IRQs */ uint16_t isp_nchan; /* number of channels */ uint32_t isp_maxluns; /* maximum luns supported */ uint32_t isp_clock : 8, /* input clock */ : 5, isp_port : 1, /* 23XX/24XX only */ isp_bustype : 1, /* SBus or PCI */ isp_loaded_fw : 1, /* loaded firmware */ isp_dblev : 16; /* debug log mask */ uint32_t isp_confopts; /* config options */ uint32_t isp_rqstinrp; /* register for REQINP */ uint32_t isp_rqstoutrp; /* register for REQOUTP */ uint32_t isp_respinrp; /* register for RESINP */ uint32_t isp_respoutrp; /* register for RESOUTP */ /* * Volatile state */ volatile u_int isp_mboxbsy; /* mailbox command active */ volatile u_int isp_state; volatile mbreg_t isp_curmbx; /* currently active mailbox command */ volatile uint32_t isp_reqodx; /* index of last ISP pickup */ volatile uint32_t isp_reqidx; /* index of next request */ volatile uint32_t isp_residx; /* index of last ISP write */ volatile uint32_t isp_resodx; /* index of next result */ volatile uint32_t isp_atioodx; /* index of next ATIO */ volatile uint32_t isp_obits; /* mailbox command output */ volatile uint32_t isp_serno; /* rolling serial number */ volatile uint16_t isp_mboxtmp[MAX_MAILBOX]; volatile uint16_t isp_lastmbxcmd; /* last mbox command sent */ volatile uint16_t isp_seqno; /* running sequence number */ /* * Active commands are stored here, indexed by handle functions. */ isp_hdl_t *isp_xflist; isp_hdl_t *isp_xffree; /* * DMA mapped in area for synchronous IOCB requests. */ void * isp_iocb; XS_DMA_ADDR_T isp_iocb_dma; /* * request/result queue pointers and DMA handles for them. */ void * isp_rquest; void * isp_result; XS_DMA_ADDR_T isp_rquest_dma; XS_DMA_ADDR_T isp_result_dma; #ifdef ISP_TARGET_MODE /* for 24XX only */ void * isp_atioq; XS_DMA_ADDR_T isp_atioq_dma; #endif }; #define SDPARAM(isp, chan) (&((sdparam *)(isp)->isp_param)[(chan)]) #define FCPARAM(isp, chan) (&((fcparam *)(isp)->isp_param)[(chan)]) #define ISP_SET_SENDMARKER(isp, chan, val) \ if (IS_FC(isp)) { \ FCPARAM(isp, chan)->sendmarker = val; \ } else { \ SDPARAM(isp, chan)->sendmarker = val; \ } #define ISP_TST_SENDMARKER(isp, chan) \ (IS_FC(isp)? \ FCPARAM(isp, chan)->sendmarker != 0 : \ SDPARAM(isp, chan)->sendmarker != 0) /* * ISP Driver Run States */ #define ISP_NILSTATE 0 #define ISP_CRASHED 1 #define ISP_RESETSTATE 2 #define ISP_INITSTATE 3 #define ISP_RUNSTATE 4 /* * ISP Runtime Configuration Options */ #define ISP_CFG_FULL_DUPLEX 0x01 /* Full Duplex (Fibre Channel only) */ #define ISP_CFG_PORT_PREF 0x0e /* Mask for Port Prefs (all FC except 2100) */ #define ISP_CFG_PORT_DEF 0x00 /* prefer connection type from NVRAM */ #define ISP_CFG_LPORT_ONLY 0x02 /* insist on {N/F}L-Port connection */ #define ISP_CFG_NPORT_ONLY 0x04 /* insist on {N/F}-Port connection */ #define ISP_CFG_LPORT 0x06 /* prefer {N/F}L-Port connection */ #define ISP_CFG_NPORT 0x08 /* prefer {N/F}-Port connection */ #define ISP_CFG_1GB 0x10 /* force 1Gb connection (23XX only) */ #define ISP_CFG_2GB 0x20 /* force 2Gb connection (23XX only) */ #define ISP_CFG_NORELOAD 0x80 /* don't download f/w */ #define ISP_CFG_NONVRAM 0x40 /* ignore NVRAM */ #define ISP_CFG_NOFCTAPE 0x100 /* disable FC-Tape */ #define ISP_CFG_FCTAPE 0x200 /* enable FC-Tape */ #define ISP_CFG_OWNFSZ 0x400 /* override NVRAM frame size */ #define ISP_CFG_OWNLOOPID 0x800 /* override NVRAM loopid */ #define ISP_CFG_OWNEXCTHROTTLE 0x1000 /* override NVRAM execution throttle */ #define ISP_CFG_4GB 0x2000 /* force 4Gb connection (24XX only) */ #define ISP_CFG_8GB 0x4000 /* force 8Gb connection (25XX only) */ #define ISP_CFG_16GB 0x8000 /* force 16Gb connection (26XX only) */ #define ISP_CFG_32GB 0x10000 /* force 32Gb connection (27XX only) */ /* * For each channel, the outer layers should know what role that channel * will take: ISP_ROLE_NONE, ISP_ROLE_INITIATOR, ISP_ROLE_TARGET, * ISP_ROLE_BOTH. * * If you set ISP_ROLE_NONE, the cards will be reset, new firmware loaded, * NVRAM read, and defaults set, but any further initialization (e.g. * INITIALIZE CONTROL BLOCK commands for 2X00 cards) won't be done. * * If INITIATOR MODE isn't set, attempts to run commands will be stopped * at isp_start and completed with the equivalent of SELECTION TIMEOUT. * * If TARGET MODE is set, it doesn't mean that the rest of target mode support * needs to be enabled, or will even work. What happens with the 2X00 cards * here is that if you have enabled it with TARGET MODE as part of the ICB * options, but you haven't given the f/w any ram resources for ATIOs or * Immediate Notifies, the f/w just handles what it can and you never see * anything. Basically, it sends a single byte of data (the first byte, * which you can set as part of the INITIALIZE CONTROL BLOCK command) for * INQUIRY, and sends back QUEUE FULL status for any other command. * */ #define ISP_ROLE_NONE 0x0 #define ISP_ROLE_TARGET 0x1 #define ISP_ROLE_INITIATOR 0x2 #define ISP_ROLE_BOTH (ISP_ROLE_TARGET|ISP_ROLE_INITIATOR) #define ISP_ROLE_EITHER ISP_ROLE_BOTH #ifndef ISP_DEFAULT_ROLES /* * Counterintuitively, we prefer to default to role 'none' * if we are enable target mode support. This gives us the * maximum flexibility as to which port will do what. */ #ifdef ISP_TARGET_MODE #define ISP_DEFAULT_ROLES ISP_ROLE_NONE #else #define ISP_DEFAULT_ROLES ISP_ROLE_INITIATOR #endif #endif /* * Firmware related defines */ #define ISP_CODE_ORG 0x1000 /* default f/w code start */ #define ISP_CODE_ORG_2300 0x0800 /* ..except for 2300s */ #define ISP_CODE_ORG_2400 0x100000 /* ..and 2400s */ #define ISP_FW_REV(maj, min, mic) ((maj << 24) | (min << 16) | mic) #define ISP_FW_MAJOR(code) ((code >> 24) & 0xff) #define ISP_FW_MINOR(code) ((code >> 16) & 0xff) #define ISP_FW_MICRO(code) ((code >> 8) & 0xff) #define ISP_FW_REVX(xp) ((xp[0]<<24) | (xp[1] << 16) | xp[2]) #define ISP_FW_MAJORX(xp) (xp[0]) #define ISP_FW_MINORX(xp) (xp[1]) #define ISP_FW_MICROX(xp) (xp[2]) #define ISP_FW_NEWER_THAN(i, major, minor, micro) \ (ISP_FW_REVX((i)->isp_fwrev) > ISP_FW_REV(major, minor, micro)) #define ISP_FW_OLDER_THAN(i, major, minor, micro) \ (ISP_FW_REVX((i)->isp_fwrev) < ISP_FW_REV(major, minor, micro)) /* * Bus (implementation) types */ #define ISP_BT_PCI 0 /* PCI Implementations */ #define ISP_BT_SBUS 1 /* SBus Implementations */ /* * If we have not otherwise defined SBus support away make sure * it is defined here such that the code is included as default */ #ifndef ISP_SBUS_SUPPORTED #define ISP_SBUS_SUPPORTED 1 #endif /* * Chip Types */ #define ISP_HA_SCSI 0xf #define ISP_HA_SCSI_UNKNOWN 0x1 #define ISP_HA_SCSI_1020 0x2 #define ISP_HA_SCSI_1020A 0x3 #define ISP_HA_SCSI_1040 0x4 #define ISP_HA_SCSI_1040A 0x5 #define ISP_HA_SCSI_1040B 0x6 #define ISP_HA_SCSI_1040C 0x7 #define ISP_HA_SCSI_1240 0x8 #define ISP_HA_SCSI_1080 0x9 #define ISP_HA_SCSI_1280 0xa #define ISP_HA_SCSI_10160 0xb #define ISP_HA_SCSI_12160 0xc #define ISP_HA_FC 0xf0 #define ISP_HA_FC_2100 0x10 #define ISP_HA_FC_2200 0x20 #define ISP_HA_FC_2300 0x30 #define ISP_HA_FC_2312 0x40 #define ISP_HA_FC_2322 0x50 #define ISP_HA_FC_2400 0x60 #define ISP_HA_FC_2500 0x70 #define ISP_HA_FC_2600 0x80 #define ISP_HA_FC_2700 0x90 #define IS_SCSI(isp) (isp->isp_type & ISP_HA_SCSI) #define IS_1020(isp) (isp->isp_type < ISP_HA_SCSI_1240) #define IS_1240(isp) (isp->isp_type == ISP_HA_SCSI_1240) #define IS_1080(isp) (isp->isp_type == ISP_HA_SCSI_1080) #define IS_1280(isp) (isp->isp_type == ISP_HA_SCSI_1280) #define IS_10160(isp) (isp->isp_type == ISP_HA_SCSI_10160) #define IS_12160(isp) (isp->isp_type == ISP_HA_SCSI_12160) #define IS_12X0(isp) (IS_1240(isp) || IS_1280(isp)) #define IS_1X160(isp) (IS_10160(isp) || IS_12160(isp)) #define IS_DUALBUS(isp) (IS_12X0(isp) || IS_12160(isp)) #define IS_ULTRA2(isp) (IS_1080(isp) || IS_1280(isp) || IS_1X160(isp)) #define IS_ULTRA3(isp) (IS_1X160(isp)) #define IS_FC(isp) ((isp)->isp_type & ISP_HA_FC) #define IS_2100(isp) ((isp)->isp_type == ISP_HA_FC_2100) #define IS_2200(isp) ((isp)->isp_type == ISP_HA_FC_2200) #define IS_23XX(isp) ((isp)->isp_type >= ISP_HA_FC_2300 && \ (isp)->isp_type < ISP_HA_FC_2400) #define IS_2300(isp) ((isp)->isp_type == ISP_HA_FC_2300) #define IS_2312(isp) ((isp)->isp_type == ISP_HA_FC_2312) #define IS_2322(isp) ((isp)->isp_type == ISP_HA_FC_2322) #define IS_24XX(isp) ((isp)->isp_type >= ISP_HA_FC_2400) #define IS_25XX(isp) ((isp)->isp_type >= ISP_HA_FC_2500) #define IS_26XX(isp) ((isp)->isp_type >= ISP_HA_FC_2600) #define IS_27XX(isp) ((isp)->isp_type >= ISP_HA_FC_2700) /* * DMA related macros */ #define DMA_WD3(x) (((uint16_t)(((uint64_t)x) >> 48)) & 0xffff) #define DMA_WD2(x) (((uint16_t)(((uint64_t)x) >> 32)) & 0xffff) #define DMA_WD1(x) ((uint16_t)((x) >> 16) & 0xffff) #define DMA_WD0(x) ((uint16_t)((x) & 0xffff)) #define DMA_LO32(x) ((uint32_t) (x)) #define DMA_HI32(x) ((uint32_t)(((uint64_t)x) >> 32)) /* * Core System Function Prototypes */ /* * Reset Hardware. Totally. Assumes that you'll follow this with a call to isp_init. */ void isp_reset(ispsoftc_t *, int); /* * Initialize Hardware to known state */ void isp_init(ispsoftc_t *); /* * Reset the ISP and call completion for any orphaned commands. */ int isp_reinit(ispsoftc_t *, int); /* * Shutdown hardware after use. */ void isp_shutdown(ispsoftc_t *); /* * Internal Interrupt Service Routine */ #ifdef ISP_TARGET_MODE void isp_intr_atioq(ispsoftc_t *); #endif void isp_intr_async(ispsoftc_t *, uint16_t event); void isp_intr_mbox(ispsoftc_t *, uint16_t mbox0); void isp_intr_respq(ispsoftc_t *); /* * Command Entry Point- Platform Dependent layers call into this */ int isp_start(XS_T *); /* these values are what isp_start returns */ #define CMD_COMPLETE 101 /* command completed */ #define CMD_EAGAIN 102 /* busy- maybe retry later */ #define CMD_QUEUED 103 /* command has been queued for execution */ #define CMD_RQLATER 104 /* requeue this command later */ /* * Command Completion Point- Core layers call out from this with completed cmds */ void isp_done(XS_T *); /* * Platform Dependent to External to Internal Control Function * * Assumes locks are held on entry. You should note that with many of * these commands locks may be released while this function is called. * * ... ISPCTL_RESET_BUS, int channel); * Reset BUS on this channel * ... ISPCTL_RESET_DEV, int channel, int target); * Reset Device on this channel at this target. * ... ISPCTL_ABORT_CMD, XS_T *xs); * Abort active transaction described by xs. * ... IPCTL_UPDATE_PARAMS); * Update any operating parameters (speed, etc.) * ... ISPCTL_FCLINK_TEST, int channel); * Test FC link status on this channel * ... ISPCTL_SCAN_LOOP, int channel); * Scan local loop on this channel * ... ISPCTL_SCAN_FABRIC, int channel); * Scan fabric on this channel * ... ISPCTL_PDB_SYNC, int channel); * Synchronize port database on this channel * ... ISPCTL_SEND_LIP, int channel); * Send a LIP on this channel * ... ISPCTL_GET_NAMES, int channel, int np, uint64_t *wwnn, uint64_t *wwpn) * Get a WWNN/WWPN for this N-port handle on this channel * ... ISPCTL_RUN_MBOXCMD, mbreg_t *mbp) * Run this mailbox command * ... ISPCTL_GET_PDB, int channel, int nphandle, isp_pdb_t *pdb) * Get PDB on this channel for this N-port handle * ... ISPCTL_PLOGX, isp_plcmd_t *) * Performa a port login/logout * ... ISPCTL_CHANGE_ROLE, int channel, int role); * Change role of specified channel * * ISPCTL_PDB_SYNC is somewhat misnamed. It actually is the final step, in * order, of ISPCTL_FCLINK_TEST, ISPCTL_SCAN_LOOP, and ISPCTL_SCAN_FABRIC. * The main purpose of ISPCTL_PDB_SYNC is to complete management of logging * and logging out of fabric devices (if one is on a fabric) and then marking * the 'loop state' as being ready to now be used for sending commands to * devices. */ typedef enum { ISPCTL_RESET_BUS, ISPCTL_RESET_DEV, ISPCTL_ABORT_CMD, ISPCTL_UPDATE_PARAMS, ISPCTL_FCLINK_TEST, ISPCTL_SCAN_FABRIC, ISPCTL_SCAN_LOOP, ISPCTL_PDB_SYNC, ISPCTL_SEND_LIP, ISPCTL_GET_NAMES, ISPCTL_RUN_MBOXCMD, ISPCTL_GET_PDB, ISPCTL_PLOGX, ISPCTL_CHANGE_ROLE } ispctl_t; int isp_control(ispsoftc_t *, ispctl_t, ...); /* * Platform Dependent to Internal to External Control Function */ typedef enum { ISPASYNC_NEW_TGT_PARAMS, /* SPI New Target Parameters */ ISPASYNC_BUS_RESET, /* All Bus Was Reset */ ISPASYNC_LOOP_DOWN, /* FC Loop Down */ ISPASYNC_LOOP_UP, /* FC Loop Up */ ISPASYNC_LIP, /* FC LIP Received */ ISPASYNC_LOOP_RESET, /* FC Loop Reset Received */ ISPASYNC_CHANGE_NOTIFY, /* FC Change Notification */ ISPASYNC_DEV_ARRIVED, /* FC Device Arrived */ ISPASYNC_DEV_CHANGED, /* FC Device Changed */ ISPASYNC_DEV_STAYED, /* FC Device Stayed */ ISPASYNC_DEV_GONE, /* FC Device Departure */ ISPASYNC_TARGET_NOTIFY, /* All target async notification */ ISPASYNC_TARGET_NOTIFY_ACK, /* All target notify ack required */ ISPASYNC_TARGET_ACTION, /* All target action requested */ ISPASYNC_FW_CRASH, /* All Firmware has crashed */ ISPASYNC_FW_RESTARTED /* All Firmware has been restarted */ } ispasync_t; void isp_async(ispsoftc_t *, ispasync_t, ...); #define ISPASYNC_CHANGE_PDB 0 #define ISPASYNC_CHANGE_SNS 1 #define ISPASYNC_CHANGE_OTHER 2 /* * Platform Dependent Error and Debug Printout * * Two required functions for each platform must be provided: * * void isp_prt(ispsoftc_t *, int level, const char *, ...) * void isp_xs_prt(ispsoftc_t *, XS_T *, int level, const char *, ...) * * but due to compiler differences on different platforms this won't be * formally defined here. Instead, they go in each platform definition file. */ #define ISP_LOGALL 0x0 /* log always */ #define ISP_LOGCONFIG 0x1 /* log configuration messages */ #define ISP_LOGINFO 0x2 /* log informational messages */ #define ISP_LOGWARN 0x4 /* log warning messages */ #define ISP_LOGERR 0x8 /* log error messages */ #define ISP_LOGDEBUG0 0x10 /* log simple debug messages */ #define ISP_LOGDEBUG1 0x20 /* log intermediate debug messages */ #define ISP_LOGDEBUG2 0x40 /* log most debug messages */ #define ISP_LOGDEBUG3 0x80 /* log high frequency debug messages */ #define ISP_LOG_SANCFG 0x100 /* log SAN configuration */ #define ISP_LOG_CWARN 0x200 /* log SCSI command "warnings" (e.g., check conditions) */ #define ISP_LOG_WARN1 0x400 /* log WARNS we might be interested at some time */ #define ISP_LOGTINFO 0x1000 /* log informational messages (target mode) */ #define ISP_LOGTDEBUG0 0x2000 /* log simple debug messages (target mode) */ #define ISP_LOGTDEBUG1 0x4000 /* log intermediate debug messages (target) */ #define ISP_LOGTDEBUG2 0x8000 /* log all debug messages (target) */ /* * Each Platform provides it's own isposinfo substructure of the ispsoftc * defined above. * * Each platform must also provide the following macros/defines: * * * ISP_FC_SCRLEN FC scratch area DMA length * * ISP_MEMZERO(dst, src) platform zeroing function * ISP_MEMCPY(dst, src, count) platform copying function * ISP_SNPRINTF(buf, bufsize, fmt, ...) snprintf * ISP_DELAY(usecs) microsecond spindelay function * ISP_SLEEP(isp, usecs) microsecond sleep function * * ISP_INLINE ___inline or not- depending on how * good your debugger is * ISP_MIN shorthand for ((a) < (b))? (a) : (b) * * NANOTIME_T nanosecond time type * * GET_NANOTIME(NANOTIME_T *) get current nanotime. * * GET_NANOSEC(NANOTIME_T *) get uint64_t from NANOTIME_T * * NANOTIME_SUB(NANOTIME_T *, NANOTIME_T *) * subtract two NANOTIME_T values * * MAXISPREQUEST(ispsoftc_t *) maximum request queue size * for this particular board type * * MEMORYBARRIER(ispsoftc_t *, barrier_type, offset, size, chan) * * Function/Macro the provides memory synchronization on * various objects so that the ISP's and the system's view * of the same object is consistent. * * MBOX_ACQUIRE(ispsoftc_t *) acquire lock on mailbox regs * MBOX_WAIT_COMPLETE(ispsoftc_t *, mbreg_t *) wait for cmd to be done * MBOX_NOTIFY_COMPLETE(ispsoftc_t *) notification of mbox cmd donee * MBOX_RELEASE(ispsoftc_t *) release lock on mailbox regs * * FC_SCRATCH_ACQUIRE(ispsoftc_t *, chan) acquire lock on FC scratch area * return -1 if you cannot * FC_SCRATCH_RELEASE(ispsoftc_t *, chan) acquire lock on FC scratch area * * FCP_NEXT_CRN(ispsoftc_t *, XS_T *, rslt, channel, target, lun) generate the next command reference number. XS_T * may be null. * * SCSI_GOOD SCSI 'Good' Status * SCSI_CHECK SCSI 'Check Condition' Status * SCSI_BUSY SCSI 'Busy' Status * SCSI_QFULL SCSI 'Queue Full' Status * * XS_T Platform SCSI transaction type (i.e., command for HBA) * XS_DMA_ADDR_T Platform PCI DMA Address Type * XS_GET_DMA_SEG(..) Get 32 bit dma segment list value * XS_GET_DMA64_SEG(..) Get 64 bit dma segment list value * XS_NEED_DMA64_SEG(..) dma segment needs 64 bit storage * XS_ISP(xs) gets an instance out of an XS_T * XS_CHANNEL(xs) gets the channel (bus # for DUALBUS cards) "" * XS_TGT(xs) gets the target "" * XS_LUN(xs) gets the lun "" * XS_CDBP(xs) gets a pointer to the scsi CDB "" * XS_CDBLEN(xs) gets the CDB's length "" * XS_XFRLEN(xs) gets the associated data transfer length "" * XS_TIME(xs) gets the time (in seconds) for this command * XS_GET_RESID(xs) gets the current residual count * XS_GET_RESID(xs, resid) sets the current residual count * XS_STSP(xs) gets a pointer to the SCSI status byte "" * XS_SNSP(xs) gets a pointer to the associate sense data * XS_TOT_SNSLEN(xs) gets the total length of sense data storage * XS_CUR_SNSLEN(xs) gets the currently used length of sense data storage * XS_SNSKEY(xs) dereferences XS_SNSP to get the current stored Sense Key * XS_SNSASC(xs) dereferences XS_SNSP to get the current stored Additional Sense Code * XS_SNSASCQ(xs) dereferences XS_SNSP to get the current stored Additional Sense Code Qualifier * XS_TAG_P(xs) predicate of whether this command should be tagged * XS_TAG_TYPE(xs) which type of tag to use * XS_SETERR(xs) set error state * * HBA_NOERROR command has no erros * HBA_BOTCH hba botched something * HBA_CMDTIMEOUT command timed out * HBA_SELTIMEOUT selection timed out (also port logouts for FC) * HBA_TGTBSY target returned a BUSY status * HBA_BUSRESET bus reset destroyed command * HBA_ABORTED command was aborted (by request) * HBA_DATAOVR a data overrun was detected * HBA_ARQFAIL Automatic Request Sense failed * * XS_ERR(xs) return current error state * XS_NOERR(xs) there is no error currently set * XS_INITERR(xs) initialize error state * * XS_SAVE_SENSE(xs, sp, len) save sense data * XS_APPEND_SENSE(xs, sp, len) append more sense data * * XS_SENSE_VALID(xs) indicates whether sense is valid * * DEFAULT_FRAMESIZE(ispsoftc_t *) Default Frame Size * DEFAULT_EXEC_THROTTLE(ispsoftc_t *) Default Execution Throttle * * DEFAULT_ROLE(ispsoftc_t *, int) Get Default Role for a channel * DEFAULT_IID(ispsoftc_t *, int) Default SCSI initiator ID * DEFAULT_LOOPID(ispsoftc_t *, int) Default FC Loop ID * * These establish reasonable defaults for each platform. * These must be available independent of card NVRAM and are * to be used should NVRAM not be readable. * * DEFAULT_NODEWWN(ispsoftc_t *, chan) Default FC Node WWN to use * DEFAULT_PORTWWN(ispsoftc_t *, chan) Default FC Port WWN to use * * These defines are hooks to allow the setting of node and * port WWNs when NVRAM cannot be read or is to be overriden. * * ACTIVE_NODEWWN(ispsoftc_t *, chan) FC Node WWN to use * ACTIVE_PORTWWN(ispsoftc_t *, chan) FC Port WWN to use * * After NVRAM is read, these will be invoked to get the * node and port WWNs that will actually be used for this * channel. * * * ISP_IOXPUT_8(ispsoftc_t *, uint8_t srcval, uint8_t *dstptr) * ISP_IOXPUT_16(ispsoftc_t *, uint16_t srcval, uint16_t *dstptr) * ISP_IOXPUT_32(ispsoftc_t *, uint32_t srcval, uint32_t *dstptr) * * ISP_IOXGET_8(ispsoftc_t *, uint8_t *srcptr, uint8_t dstrval) * ISP_IOXGET_16(ispsoftc_t *, uint16_t *srcptr, uint16_t dstrval) * ISP_IOXGET_32(ispsoftc_t *, uint32_t *srcptr, uint32_t dstrval) * * ISP_SWIZZLE_NVRAM_WORD(ispsoftc_t *, uint16_t *) * ISP_SWIZZLE_NVRAM_LONG(ispsoftc_t *, uint32_t *) * ISP_SWAP16(ispsoftc_t *, uint16_t srcval) * ISP_SWAP32(ispsoftc_t *, uint32_t srcval) */ #ifdef ISP_TARGET_MODE /* * The functions below are for the publicly available * target mode functions that are internal to the Qlogic driver. */ /* * This function handles new response queue entry appropriate for target mode. */ int isp_target_notify(ispsoftc_t *, void *, uint32_t *); /* * This function externalizes the ability to acknowledge an Immediate Notify request. */ int isp_notify_ack(ispsoftc_t *, void *); /* * This function externalized acknowledging (success/fail) an ABTS frame */ int isp_acknak_abts(ispsoftc_t *, void *, int); /* * General request queue 'put' routine for target mode entries. */ int isp_target_put_entry(ispsoftc_t *isp, void *); /* * General routine to put back an ATIO entry- * used for replenishing f/w resource counts. * The argument is a pointer to a source ATIO * or ATIO2. */ int isp_target_put_atio(ispsoftc_t *, void *); /* * General routine to send a final CTIO for a command- used mostly for * local responses. */ int isp_endcmd(ispsoftc_t *, ...); #define ECMD_SVALID 0x100 #define ECMD_RVALID 0x200 #define ECMD_TERMINATE 0x400 /* * Handle an asynchronous event */ void isp_target_async(ispsoftc_t *, int, int); #endif #endif /* _ISPVAR_H */ Index: projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm.c (revision 345026) @@ -1,6412 +1,6414 @@ /* $OpenBSD: if_iwm.c,v 1.167 2017/04/04 00:40:52 claudio Exp $ */ /* * Copyright (c) 2014 genua mbh * Copyright (c) 2014 Fixup Software Ltd. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * * Driver version we are currently based off of is * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd) * *********************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * * BSD LICENSE * * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2007-2010 Damien Bergamini * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* From DragonflyBSD */ #define mtodoff(m, t, off) ((t)((m)->m_data + (off))) const uint8_t iwm_nvm_channels[] = { /* 2.4 GHz */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 5 GHz */ 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165 }; _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS, "IWM_NUM_CHANNELS is too small"); const uint8_t iwm_nvm_channels_8000[] = { /* 2.4 GHz */ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 5 GHz */ 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, 181 }; _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000, "IWM_NUM_CHANNELS_8000 is too small"); #define IWM_NUM_2GHZ_CHANNELS 14 #define IWM_N_HW_ADDR_MASK 0xF /* * XXX For now, there's simply a fixed set of rate table entries * that are populated. */ const struct iwm_rate { uint8_t rate; uint8_t plcp; } iwm_rates[] = { { 2, IWM_RATE_1M_PLCP }, { 4, IWM_RATE_2M_PLCP }, { 11, IWM_RATE_5M_PLCP }, { 22, IWM_RATE_11M_PLCP }, { 12, IWM_RATE_6M_PLCP }, { 18, IWM_RATE_9M_PLCP }, { 24, IWM_RATE_12M_PLCP }, { 36, IWM_RATE_18M_PLCP }, { 48, IWM_RATE_24M_PLCP }, { 72, IWM_RATE_36M_PLCP }, { 96, IWM_RATE_48M_PLCP }, { 108, IWM_RATE_54M_PLCP }, }; #define IWM_RIDX_CCK 0 #define IWM_RIDX_OFDM 4 #define IWM_RIDX_MAX (nitems(iwm_rates)-1) #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM) #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM) struct iwm_nvm_section { uint16_t length; uint8_t *data; }; #define IWM_MVM_UCODE_ALIVE_TIMEOUT hz #define IWM_MVM_UCODE_CALIB_TIMEOUT (2*hz) struct iwm_mvm_alive_data { int valid; uint32_t scd_base_addr; }; static int iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t); static int iwm_firmware_store_section(struct iwm_softc *, enum iwm_ucode_type, const uint8_t *, size_t); static int iwm_set_default_calib(struct iwm_softc *, const void *); static void iwm_fw_info_free(struct iwm_fw_info *); static int iwm_read_firmware(struct iwm_softc *); static int iwm_alloc_fwmem(struct iwm_softc *); static int iwm_alloc_sched(struct iwm_softc *); static int iwm_alloc_kw(struct iwm_softc *); static int iwm_alloc_ict(struct iwm_softc *); static int iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static void iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static void iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *); static int iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *, int); static void iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *); static void iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *); static void iwm_enable_interrupts(struct iwm_softc *); static void iwm_restore_interrupts(struct iwm_softc *); static void iwm_disable_interrupts(struct iwm_softc *); static void iwm_ict_reset(struct iwm_softc *); static int iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *); static void iwm_stop_device(struct iwm_softc *); static void iwm_mvm_nic_config(struct iwm_softc *); static int iwm_nic_rx_init(struct iwm_softc *); static int iwm_nic_tx_init(struct iwm_softc *); static int iwm_nic_init(struct iwm_softc *); static int iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t); static int iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t, uint16_t, uint8_t *, uint16_t *); static int iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *, uint16_t *, uint32_t); static uint32_t iwm_eeprom_channel_flags(uint16_t); static void iwm_add_channel_band(struct iwm_softc *, struct ieee80211_channel[], int, int *, int, size_t, const uint8_t[]); static void iwm_init_channel_map(struct ieee80211com *, int, int *, struct ieee80211_channel[]); static struct iwm_nvm_data * iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *, const uint16_t *); static void iwm_free_nvm_data(struct iwm_nvm_data *); static void iwm_set_hw_address_family_8000(struct iwm_softc *, struct iwm_nvm_data *, const uint16_t *, const uint16_t *); static int iwm_get_sku(const struct iwm_softc *, const uint16_t *, const uint16_t *); static int iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *); static int iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *, const uint16_t *); static int iwm_get_n_hw_addrs(const struct iwm_softc *, const uint16_t *); static void iwm_set_radio_cfg(const struct iwm_softc *, struct iwm_nvm_data *, uint32_t); static struct iwm_nvm_data * iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *); static int iwm_nvm_init(struct iwm_softc *); static int iwm_pcie_load_section(struct iwm_softc *, uint8_t, const struct iwm_fw_desc *); static int iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t, bus_addr_t, uint32_t); static int iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc, const struct iwm_fw_img *, int, int *); static int iwm_pcie_load_cpu_sections(struct iwm_softc *, const struct iwm_fw_img *, int, int *); static int iwm_pcie_load_given_ucode_8000(struct iwm_softc *, const struct iwm_fw_img *); static int iwm_pcie_load_given_ucode(struct iwm_softc *, const struct iwm_fw_img *); static int iwm_start_fw(struct iwm_softc *, const struct iwm_fw_img *); static int iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t); static int iwm_send_phy_cfg_cmd(struct iwm_softc *); static int iwm_mvm_load_ucode_wait_alive(struct iwm_softc *, enum iwm_ucode_type); static int iwm_run_init_mvm_ucode(struct iwm_softc *, int); static int iwm_mvm_config_ltr(struct iwm_softc *sc); static int iwm_rx_addbuf(struct iwm_softc *, int, int); static int iwm_mvm_get_signal_strength(struct iwm_softc *, struct iwm_rx_phy_info *); static void iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *, struct iwm_rx_packet *); static int iwm_get_noise(struct iwm_softc *, const struct iwm_mvm_statistics_rx_non_phy *); static void iwm_mvm_handle_rx_statistics(struct iwm_softc *, struct iwm_rx_packet *); static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *, uint32_t, boolean_t); static int iwm_mvm_rx_tx_cmd_single(struct iwm_softc *, struct iwm_rx_packet *, struct iwm_node *); static void iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *); static void iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *); #if 0 static void iwm_update_sched(struct iwm_softc *, int, int, uint8_t, uint16_t); #endif static const struct iwm_rate * iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *, struct mbuf *, struct iwm_tx_cmd *); static int iwm_tx(struct iwm_softc *, struct mbuf *, struct ieee80211_node *, int); static int iwm_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static int iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_vap *); static int iwm_auth(struct ieee80211vap *, struct iwm_softc *); static struct ieee80211_node * iwm_node_alloc(struct ieee80211vap *, const uint8_t[IEEE80211_ADDR_LEN]); static uint8_t iwm_rate_from_ucode_rate(uint32_t); static int iwm_rate2ridx(struct iwm_softc *, uint8_t); static void iwm_setrates(struct iwm_softc *, struct iwm_node *, int); static int iwm_media_change(struct ifnet *); static int iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int); static void iwm_endscan_cb(void *, int); static int iwm_send_bt_init_conf(struct iwm_softc *); static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *); static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *); static int iwm_send_update_mcc_cmd(struct iwm_softc *, const char *); static void iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t); static int iwm_init_hw(struct iwm_softc *); static void iwm_init(struct iwm_softc *); static void iwm_start(struct iwm_softc *); static void iwm_stop(struct iwm_softc *); static void iwm_watchdog(void *); static void iwm_parent(struct ieee80211com *); #ifdef IWM_DEBUG static const char * iwm_desc_lookup(uint32_t); static void iwm_nic_error(struct iwm_softc *); static void iwm_nic_umac_error(struct iwm_softc *); #endif static void iwm_handle_rxb(struct iwm_softc *, struct mbuf *); static void iwm_notif_intr(struct iwm_softc *); static void iwm_intr(void *); static int iwm_attach(device_t); static int iwm_is_valid_ether_addr(uint8_t *); static void iwm_preinit(void *); static int iwm_detach_local(struct iwm_softc *sc, int); static void iwm_init_task(void *); static void iwm_radiotap_attach(struct iwm_softc *); static struct ieee80211vap * iwm_vap_create(struct ieee80211com *, const char [IFNAMSIZ], int, enum ieee80211_opmode, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); static void iwm_vap_delete(struct ieee80211vap *); static void iwm_xmit_queue_drain(struct iwm_softc *); static void iwm_scan_start(struct ieee80211com *); static void iwm_scan_end(struct ieee80211com *); static void iwm_update_mcast(struct ieee80211com *); static void iwm_set_channel(struct ieee80211com *); static void iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long); static void iwm_scan_mindwell(struct ieee80211_scan_state *); static int iwm_detach(device_t); static int iwm_lar_disable = 0; TUNABLE_INT("hw.iwm.lar.disable", &iwm_lar_disable); /* * Firmware parser. */ static int iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen) { const struct iwm_fw_cscheme_list *l = (const void *)data; if (dlen < sizeof(*l) || dlen < sizeof(l->size) + l->size * sizeof(*l->cs)) return EINVAL; /* we don't actually store anything for now, always use s/w crypto */ return 0; } static int iwm_firmware_store_section(struct iwm_softc *sc, enum iwm_ucode_type type, const uint8_t *data, size_t dlen) { struct iwm_fw_img *fws; struct iwm_fw_desc *fwone; if (type >= IWM_UCODE_TYPE_MAX) return EINVAL; if (dlen < sizeof(uint32_t)) return EINVAL; fws = &sc->sc_fw.img[type]; if (fws->fw_count >= IWM_UCODE_SECTION_MAX) return EINVAL; fwone = &fws->sec[fws->fw_count]; /* first 32bit are device load offset */ memcpy(&fwone->offset, data, sizeof(uint32_t)); /* rest is data */ fwone->data = data + sizeof(uint32_t); fwone->len = dlen - sizeof(uint32_t); fws->fw_count++; return 0; } #define IWM_DEFAULT_SCAN_CHANNELS 40 /* iwlwifi: iwl-drv.c */ struct iwm_tlv_calib_data { uint32_t ucode_type; struct iwm_tlv_calib_ctrl calib; } __packed; static int iwm_set_default_calib(struct iwm_softc *sc, const void *data) { const struct iwm_tlv_calib_data *def_calib = data; uint32_t ucode_type = le32toh(def_calib->ucode_type); if (ucode_type >= IWM_UCODE_TYPE_MAX) { device_printf(sc->sc_dev, "Wrong ucode_type %u for default " "calibration.\n", ucode_type); return EINVAL; } sc->sc_default_calib[ucode_type].flow_trigger = def_calib->calib.flow_trigger; sc->sc_default_calib[ucode_type].event_trigger = def_calib->calib.event_trigger; return 0; } static int iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data, struct iwm_ucode_capabilities *capa) { const struct iwm_ucode_api *ucode_api = (const void *)data; uint32_t api_index = le32toh(ucode_api->api_index); uint32_t api_flags = le32toh(ucode_api->api_flags); int i; if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) { device_printf(sc->sc_dev, "api flags index %d larger than supported by driver\n", api_index); /* don't return an error so we can load FW that has more bits */ return 0; } for (i = 0; i < 32; i++) { if (api_flags & (1U << i)) setbit(capa->enabled_api, i + 32 * api_index); } return 0; } static int iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data, struct iwm_ucode_capabilities *capa) { const struct iwm_ucode_capa *ucode_capa = (const void *)data; uint32_t api_index = le32toh(ucode_capa->api_index); uint32_t api_flags = le32toh(ucode_capa->api_capa); int i; if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) { device_printf(sc->sc_dev, "capa flags index %d larger than supported by driver\n", api_index); /* don't return an error so we can load FW that has more bits */ return 0; } for (i = 0; i < 32; i++) { if (api_flags & (1U << i)) setbit(capa->enabled_capa, i + 32 * api_index); } return 0; } static void iwm_fw_info_free(struct iwm_fw_info *fw) { firmware_put(fw->fw_fp, FIRMWARE_UNLOAD); fw->fw_fp = NULL; memset(fw->img, 0, sizeof(fw->img)); } static int iwm_read_firmware(struct iwm_softc *sc) { struct iwm_fw_info *fw = &sc->sc_fw; const struct iwm_tlv_ucode_header *uhdr; const struct iwm_ucode_tlv *tlv; struct iwm_ucode_capabilities *capa = &sc->sc_fw.ucode_capa; enum iwm_ucode_tlv_type tlv_type; const struct firmware *fwp; const uint8_t *data; uint32_t tlv_len; uint32_t usniffer_img; const uint8_t *tlv_data; uint32_t paging_mem_size; int num_of_cpus; int error = 0; size_t len; /* * Load firmware into driver memory. * fw_fp will be set. */ fwp = firmware_get(sc->cfg->fw_name); if (fwp == NULL) { device_printf(sc->sc_dev, "could not read firmware %s (error %d)\n", sc->cfg->fw_name, error); goto out; } fw->fw_fp = fwp; /* (Re-)Initialize default values. */ capa->flags = 0; capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH; capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS; memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa)); memset(capa->enabled_api, 0, sizeof(capa->enabled_api)); memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc)); /* * Parse firmware contents */ uhdr = (const void *)fw->fw_fp->data; if (*(const uint32_t *)fw->fw_fp->data != 0 || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) { device_printf(sc->sc_dev, "invalid firmware %s\n", sc->cfg->fw_name); error = EINVAL; goto out; } snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%u.%u (API ver %u)", IWM_UCODE_MAJOR(le32toh(uhdr->ver)), IWM_UCODE_MINOR(le32toh(uhdr->ver)), IWM_UCODE_API(le32toh(uhdr->ver))); data = uhdr->data; len = fw->fw_fp->datasize - sizeof(*uhdr); while (len >= sizeof(*tlv)) { len -= sizeof(*tlv); tlv = (const void *)data; tlv_len = le32toh(tlv->length); tlv_type = le32toh(tlv->type); tlv_data = tlv->data; if (len < tlv_len) { device_printf(sc->sc_dev, "firmware too short: %zu bytes\n", len); error = EINVAL; goto parse_out; } len -= roundup2(tlv_len, 4); data += sizeof(*tlv) + roundup2(tlv_len, 4); switch ((int)tlv_type) { case IWM_UCODE_TLV_PROBE_MAX_LEN: if (tlv_len != sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: PROBE_MAX_LEN (%u) != sizeof(uint32_t)\n", __func__, tlv_len); error = EINVAL; goto parse_out; } capa->max_probe_length = le32_to_cpup((const uint32_t *)tlv_data); /* limit it to something sensible */ if (capa->max_probe_length > IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) { IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "%s: IWM_UCODE_TLV_PROBE_MAX_LEN " "ridiculous\n", __func__); error = EINVAL; goto parse_out; } break; case IWM_UCODE_TLV_PAN: if (tlv_len) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_PAN: tlv_len (%u) > 0\n", __func__, tlv_len); error = EINVAL; goto parse_out; } capa->flags |= IWM_UCODE_TLV_FLAGS_PAN; break; case IWM_UCODE_TLV_FLAGS: if (tlv_len < sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) < sizeof(uint32_t)\n", __func__, tlv_len); error = EINVAL; goto parse_out; } if (tlv_len % sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) %% sizeof(uint32_t)\n", __func__, tlv_len); error = EINVAL; goto parse_out; } /* * Apparently there can be many flags, but Linux driver * parses only the first one, and so do we. * * XXX: why does this override IWM_UCODE_TLV_PAN? * Intentional or a bug? Observations from * current firmware file: * 1) TLV_PAN is parsed first * 2) TLV_FLAGS contains TLV_FLAGS_PAN * ==> this resets TLV_PAN to itself... hnnnk */ capa->flags = le32_to_cpup((const uint32_t *)tlv_data); break; case IWM_UCODE_TLV_CSCHEME: if ((error = iwm_store_cscheme(sc, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: iwm_store_cscheme(): returned %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_NUM_OF_CPU: if (tlv_len != sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%u) != sizeof(uint32_t)\n", __func__, tlv_len); error = EINVAL; goto parse_out; } num_of_cpus = le32_to_cpup((const uint32_t *)tlv_data); if (num_of_cpus == 2) { fw->img[IWM_UCODE_REGULAR].is_dual_cpus = TRUE; fw->img[IWM_UCODE_INIT].is_dual_cpus = TRUE; fw->img[IWM_UCODE_WOWLAN].is_dual_cpus = TRUE; } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) { device_printf(sc->sc_dev, "%s: Driver supports only 1 or 2 CPUs\n", __func__); error = EINVAL; goto parse_out; } break; case IWM_UCODE_TLV_SEC_RT: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_SEC_INIT: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_SEC_WOWLAN: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) { device_printf(sc->sc_dev, "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_DEF_CALIB: if (tlv_len != sizeof(struct iwm_tlv_calib_data)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%u) < sizeof(iwm_tlv_calib_data) (%zu)\n", __func__, tlv_len, sizeof(struct iwm_tlv_calib_data)); error = EINVAL; goto parse_out; } if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) { device_printf(sc->sc_dev, "%s: iwm_set_default_calib() failed: %d\n", __func__, error); goto parse_out; } break; case IWM_UCODE_TLV_PHY_SKU: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%u) < sizeof(uint32_t)\n", __func__, tlv_len); goto parse_out; } sc->sc_fw.phy_config = le32_to_cpup((const uint32_t *)tlv_data); sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config & IWM_FW_PHY_CFG_TX_CHAIN) >> IWM_FW_PHY_CFG_TX_CHAIN_POS; sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config & IWM_FW_PHY_CFG_RX_CHAIN) >> IWM_FW_PHY_CFG_RX_CHAIN_POS; break; case IWM_UCODE_TLV_API_CHANGES_SET: { if (tlv_len != sizeof(struct iwm_ucode_api)) { error = EINVAL; goto parse_out; } if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) { error = EINVAL; goto parse_out; } break; } case IWM_UCODE_TLV_ENABLED_CAPABILITIES: { if (tlv_len != sizeof(struct iwm_ucode_capa)) { error = EINVAL; goto parse_out; } if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) { error = EINVAL; goto parse_out; } break; } case 48: /* undocumented TLV */ case IWM_UCODE_TLV_SDIO_ADMA_ADDR: case IWM_UCODE_TLV_FW_GSCAN_CAPA: /* ignore, not used by current driver */ break; case IWM_UCODE_TLV_SEC_RT_USNIFFER: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_REGULAR_USNIFFER, tlv_data, tlv_len)) != 0) goto parse_out; break; case IWM_UCODE_TLV_PAGING: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; goto parse_out; } paging_mem_size = le32_to_cpup((const uint32_t *)tlv_data); IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "%s: Paging: paging enabled (size = %u bytes)\n", __func__, paging_mem_size); if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) { device_printf(sc->sc_dev, "%s: Paging: driver supports up to %u bytes for paging image\n", __func__, IWM_MAX_PAGING_IMAGE_SIZE); error = EINVAL; goto out; } if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) { device_printf(sc->sc_dev, "%s: Paging: image isn't multiple %u\n", __func__, IWM_FW_PAGING_SIZE); error = EINVAL; goto out; } sc->sc_fw.img[IWM_UCODE_REGULAR].paging_mem_size = paging_mem_size; usniffer_img = IWM_UCODE_REGULAR_USNIFFER; sc->sc_fw.img[usniffer_img].paging_mem_size = paging_mem_size; break; case IWM_UCODE_TLV_N_SCAN_CHANNELS: if (tlv_len != sizeof(uint32_t)) { error = EINVAL; goto parse_out; } capa->n_scan_channels = le32_to_cpup((const uint32_t *)tlv_data); break; case IWM_UCODE_TLV_FW_VERSION: if (tlv_len != sizeof(uint32_t) * 3) { error = EINVAL; goto parse_out; } snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d.%d", le32toh(((const uint32_t *)tlv_data)[0]), le32toh(((const uint32_t *)tlv_data)[1]), le32toh(((const uint32_t *)tlv_data)[2])); break; case IWM_UCODE_TLV_FW_MEM_SEG: break; default: device_printf(sc->sc_dev, "%s: unknown firmware section %d, abort\n", __func__, tlv_type); error = EINVAL; goto parse_out; } } KASSERT(error == 0, ("unhandled error")); parse_out: if (error) { device_printf(sc->sc_dev, "firmware parse error %d, " "section type %d\n", error, tlv_type); } out: if (error) { if (fw->fw_fp != NULL) iwm_fw_info_free(fw); } return error; } /* * DMA resource routines */ /* fwmem is used to load firmware onto the card */ static int iwm_alloc_fwmem(struct iwm_softc *sc) { /* Must be aligned on a 16-byte boundary. */ return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma, IWM_FH_MEM_TB_MAX_LENGTH, 16); } /* tx scheduler rings. not used? */ static int iwm_alloc_sched(struct iwm_softc *sc) { /* TX scheduler rings must be aligned on a 1KB boundary. */ return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma, nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024); } /* keep-warm page is used internally by the card. see iwl-fh.h for more info */ static int iwm_alloc_kw(struct iwm_softc *sc) { return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096); } /* interrupt cause table */ static int iwm_alloc_ict(struct iwm_softc *sc) { return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma, IWM_ICT_SIZE, 1<cur = 0; /* Allocate RX descriptors (256-byte aligned). */ size = IWM_RX_RING_COUNT * sizeof(uint32_t); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256); if (error != 0) { device_printf(sc->sc_dev, "could not allocate RX ring DMA memory\n"); goto fail; } ring->desc = ring->desc_dma.vaddr; /* Allocate RX status area (16-byte aligned). */ error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma, sizeof(*ring->stat), 16); if (error != 0) { device_printf(sc->sc_dev, "could not allocate RX status DMA memory\n"); goto fail; } ring->stat = ring->stat_dma.vaddr; /* Create RX buffer DMA tag. */ error = bus_dma_tag_create(sc->sc_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA tag, error %d\n", __func__, error); goto fail; } /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */ error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA map, error %d\n", __func__, error); goto fail; } /* * Allocate and map RX buffers. */ for (i = 0; i < IWM_RX_RING_COUNT; i++) { struct iwm_rx_data *data = &ring->data[i]; error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA map, error %d\n", __func__, error); goto fail; } data->m = NULL; if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) { goto fail; } } return 0; fail: iwm_free_rx_ring(sc, ring); return error; } static void iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring) { /* Reset the ring state */ ring->cur = 0; /* * The hw rx ring index in shared memory must also be cleared, * otherwise the discrepancy can cause reprocessing chaos. */ if (sc->rxq.stat) memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat)); } static void iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring) { int i; iwm_dma_contig_free(&ring->desc_dma); iwm_dma_contig_free(&ring->stat_dma); for (i = 0; i < IWM_RX_RING_COUNT; i++) { struct iwm_rx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->map != NULL) { bus_dmamap_destroy(ring->data_dmat, data->map); data->map = NULL; } } if (ring->spare_map != NULL) { bus_dmamap_destroy(ring->data_dmat, ring->spare_map); ring->spare_map = NULL; } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } static int iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid) { bus_addr_t paddr; bus_size_t size; size_t maxsize; int nsegments; int i, error; ring->qid = qid; ring->queued = 0; ring->cur = 0; /* Allocate TX descriptors (256-byte aligned). */ size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256); if (error != 0) { device_printf(sc->sc_dev, "could not allocate TX ring DMA memory\n"); goto fail; } ring->desc = ring->desc_dma.vaddr; /* * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need * to allocate commands space for other rings. */ if (qid > IWM_MVM_CMD_QUEUE) return 0; size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd); error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4); if (error != 0) { device_printf(sc->sc_dev, "could not allocate TX cmd DMA memory\n"); goto fail; } ring->cmd = ring->cmd_dma.vaddr; /* FW commands may require more mapped space than packets. */ if (qid == IWM_MVM_CMD_QUEUE) { maxsize = IWM_RBUF_SIZE; nsegments = 1; } else { maxsize = MCLBYTES; nsegments = IWM_MAX_SCATTER - 2; } error = bus_dma_tag_create(sc->sc_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize, nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "could not create TX buf DMA tag\n"); goto fail; } paddr = ring->cmd_dma.paddr; for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; data->cmd_paddr = paddr; data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header) + offsetof(struct iwm_tx_cmd, scratch); paddr += sizeof(struct iwm_device_cmd); error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "could not create TX buf DMA map\n"); goto fail; } } KASSERT(paddr == ring->cmd_dma.paddr + size, ("invalid physical address")); return 0; fail: iwm_free_tx_ring(sc, ring); return error; } static void iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring) { int i; for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } } /* Clear TX descriptors. */ memset(ring->desc, 0, ring->desc_dma.size); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); sc->qfullmsk &= ~(1 << ring->qid); ring->queued = 0; ring->cur = 0; if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake) iwm_pcie_clear_cmd_in_flight(sc); } static void iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring) { int i; iwm_dma_contig_free(&ring->desc_dma); iwm_dma_contig_free(&ring->cmd_dma); for (i = 0; i < IWM_TX_RING_COUNT; i++) { struct iwm_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->map != NULL) { bus_dmamap_destroy(ring->data_dmat, data->map); data->map = NULL; } } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } /* * High-level hardware frobbing routines */ static void iwm_enable_interrupts(struct iwm_softc *sc) { sc->sc_intmask = IWM_CSR_INI_SET_MASK; IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } static void iwm_restore_interrupts(struct iwm_softc *sc) { IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } static void iwm_disable_interrupts(struct iwm_softc *sc) { /* disable interrupts */ IWM_WRITE(sc, IWM_CSR_INT_MASK, 0); /* acknowledge all interrupts */ IWM_WRITE(sc, IWM_CSR_INT, ~0); IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0); } static void iwm_ict_reset(struct iwm_softc *sc) { iwm_disable_interrupts(sc); /* Reset ICT table. */ memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE); sc->ict_cur = 0; /* Set physical address of ICT table (4KB aligned). */ IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG, IWM_CSR_DRAM_INT_TBL_ENABLE | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT); /* Switch to ICT interrupt mode in driver. */ sc->sc_flags |= IWM_FLAG_USE_ICT; /* Re-enable interrupts. */ IWM_WRITE(sc, IWM_CSR_INT, ~0); iwm_enable_interrupts(sc); } /* iwlwifi pcie/trans.c */ /* * Since this .. hard-resets things, it's time to actually * mark the first vap (if any) as having no mac context. * It's annoying, but since the driver is potentially being * stop/start'ed whilst active (thanks openbsd port!) we * have to correctly track this. */ static void iwm_stop_device(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); int chnl, qid; uint32_t mask = 0; /* tell the device to stop sending interrupts */ iwm_disable_interrupts(sc); /* * FreeBSD-local: mark the first vap as not-uploaded, * so the next transition through auth/assoc * will correctly populate the MAC context. */ if (vap) { struct iwm_vap *iv = IWM_VAP(vap); iv->phy_ctxt = NULL; iv->is_uploaded = 0; } sc->sc_firmware_state = 0; sc->sc_flags &= ~IWM_FLAG_TE_ACTIVE; /* device going down, Stop using ICT table */ sc->sc_flags &= ~IWM_FLAG_USE_ICT; /* stop tx and rx. tx and rx bits, as usual, are from if_iwn */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_SCD_TXFACT, 0); /* Stop each Tx DMA channel */ for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) { IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0); mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl); } /* Wait for DMA channels to be idle */ if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask, 5000)) { device_printf(sc->sc_dev, "Failing on timeout while stopping DMA channel: [0x%08x]\n", IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG)); } iwm_nic_unlock(sc); } iwm_pcie_rx_stop(sc); /* Stop RX ring. */ iwm_reset_rx_ring(sc, &sc->rxq); /* Reset all TX rings. */ for (qid = 0; qid < nitems(sc->txq); qid++) iwm_reset_tx_ring(sc, &sc->txq[qid]); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { /* Power-down device's busmaster DMA clocks */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG, IWM_APMG_CLK_VAL_DMA_CLK_RQT); iwm_nic_unlock(sc); } DELAY(5); } /* Make sure (redundant) we've released our request to stay awake */ IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); /* Stop the device, and put it in low power state */ iwm_apm_stop(sc); /* Upon stop, the APM issues an interrupt if HW RF kill is set. * Clean again the interrupt here */ iwm_disable_interrupts(sc); /* stop and reset the on-board processor */ IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET); /* * Even if we stop the HW, we still want the RF kill * interrupt */ iwm_enable_rfkill_int(sc); iwm_check_rfkill(sc); } /* iwlwifi: mvm/ops.c */ static void iwm_mvm_nic_config(struct iwm_softc *sc) { uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash; uint32_t reg_val = 0; uint32_t phy_config = iwm_mvm_get_phy_config(sc); radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >> IWM_FW_PHY_CFG_RADIO_TYPE_POS; radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >> IWM_FW_PHY_CFG_RADIO_STEP_POS; radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >> IWM_FW_PHY_CFG_RADIO_DASH_POS; /* SKU control */ reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) << IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP; reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) << IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH; /* radio configuration */ reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE; reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP; reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH; IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val); IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type, radio_cfg_step, radio_cfg_dash); /* * W/A : NIC is stuck in a reset state after Early PCIe power off * (PCIe power is lost before PERST# is asserted), causing ME FW * to lose ownership and not being able to obtain it back. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG, IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS, ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS); } } static int iwm_nic_rx_init(struct iwm_softc *sc) { /* * Initialize RX ring. This is from the iwn driver. */ memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat)); /* Stop Rx DMA */ iwm_pcie_rx_stop(sc); if (!iwm_nic_lock(sc)) return EBUSY; /* reset and flush pointers */ IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0); IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); /* Set physical address of RX ring (256-byte aligned). */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8); /* Set physical address of RX status (16-byte aligned). */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4); /* Enable Rx DMA * XXX 5000 HW isn't supported by the iwm(4) driver. * IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in * the credit mechanism in 5000 HW RX FIFO * Direct rx interrupts to hosts * Rx buffer size 4 or 8k or 12k * RB timeout 0x10 * 256 RBDs */ IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG, IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | /* HW bug */ IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K | (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS); IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF); /* W/A for interrupt coalescing bug in 7260 and 3160 */ if (sc->cfg->host_interrupt_operation_mode) IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE); /* * Thus sayeth el jefe (iwlwifi) via a comment: * * This value should initially be 0 (before preparing any * RBs), should be 8 after preparing the first 8 RBs (for example) */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8); iwm_nic_unlock(sc); return 0; } static int iwm_nic_tx_init(struct iwm_softc *sc) { int qid; if (!iwm_nic_lock(sc)) return EBUSY; /* Deactivate TX scheduler. */ iwm_write_prph(sc, IWM_SCD_TXFACT, 0); /* Set physical address of "keep warm" page (16-byte aligned). */ IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4); /* Initialize TX rings. */ for (qid = 0; qid < nitems(sc->txq); qid++) { struct iwm_tx_ring *txq = &sc->txq[qid]; /* Set physical address of TX ring (256-byte aligned). */ IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid), txq->desc_dma.paddr >> 8); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: loading ring %d descriptors (%p) at %lx\n", __func__, qid, txq->desc, (unsigned long) (txq->desc_dma.paddr >> 8)); } iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE); iwm_nic_unlock(sc); return 0; } static int iwm_nic_init(struct iwm_softc *sc) { int error; iwm_apm_init(sc); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) iwm_set_pwr(sc); iwm_mvm_nic_config(sc); if ((error = iwm_nic_rx_init(sc)) != 0) return error; /* * Ditto for TX, from iwn */ if ((error = iwm_nic_tx_init(sc)) != 0) return error; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: shadow registers enabled\n", __func__); IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff); return 0; } int iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo) { if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0); if (qid == IWM_MVM_CMD_QUEUE) { /* unactivate before configuration */ iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN)); iwm_nic_unlock(sc); iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid)); if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0); iwm_nic_unlock(sc); iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0); /* Set scheduler window size and frame limit. */ iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) + sizeof(uint32_t), ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) & IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) | ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) & IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK)); if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); return EBUSY; } iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) | (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) | IWM_SCD_QUEUE_STTS_REG_MSK); } else { struct iwm_scd_txq_cfg_cmd cmd; int error; iwm_nic_unlock(sc); memset(&cmd, 0, sizeof(cmd)); cmd.scd_queue = qid; cmd.enable = 1; cmd.sta_id = sta_id; cmd.tx_fifo = fifo; cmd.aggregate = 0; cmd.window = IWM_FRAME_LIMIT; error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC, sizeof(cmd), &cmd); if (error) { device_printf(sc->sc_dev, "cannot enable txq %d\n", qid); return error; } if (!iwm_nic_lock(sc)) return EBUSY; } iwm_write_prph(sc, IWM_SCD_EN_CTRL, iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid); iwm_nic_unlock(sc); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n", __func__, qid, fifo); return 0; } static int iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr) { int error, chnl; int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND - IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t); if (!iwm_nic_lock(sc)) return EBUSY; iwm_ict_reset(sc); sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR); if (scd_base_addr != 0 && scd_base_addr != sc->scd_base_addr) { device_printf(sc->sc_dev, "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n", __func__, sc->scd_base_addr, scd_base_addr); } iwm_nic_unlock(sc); /* reset context data, TX status and translation data */ error = iwm_write_mem(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND, NULL, clear_dwords); if (error) return EBUSY; if (!iwm_nic_lock(sc)) return EBUSY; /* Set physical address of TX scheduler rings (1KB aligned). */ iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10); iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0); iwm_nic_unlock(sc); /* enable command channel */ error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7); if (error) return error; if (!iwm_nic_lock(sc)) return EBUSY; iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff); /* Enable DMA channels. */ for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) { IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); } IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG, IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); iwm_nic_unlock(sc); /* Enable L1-Active */ if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); } return error; } /* * NVM read access and content parsing. We do not support * external NVM or writing NVM. * iwlwifi/mvm/nvm.c */ /* Default NVM size to read */ #define IWM_NVM_DEFAULT_CHUNK_SIZE (2*1024) #define IWM_NVM_WRITE_OPCODE 1 #define IWM_NVM_READ_OPCODE 0 /* load nvm chunk response */ enum { IWM_READ_NVM_CHUNK_SUCCEED = 0, IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1 }; static int iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len) { struct iwm_nvm_access_cmd nvm_access_cmd = { .offset = htole16(offset), .length = htole16(length), .type = htole16(section), .op_code = IWM_NVM_READ_OPCODE, }; struct iwm_nvm_access_resp *nvm_resp; struct iwm_rx_packet *pkt; struct iwm_host_cmd cmd = { .id = IWM_NVM_ACCESS_CMD, .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL, .data = { &nvm_access_cmd, }, }; int ret, bytes_read, offset_read; uint8_t *resp_data; cmd.len[0] = sizeof(struct iwm_nvm_access_cmd); ret = iwm_send_cmd(sc, &cmd); if (ret) { device_printf(sc->sc_dev, "Could not send NVM_ACCESS command (error=%d)\n", ret); return ret; } pkt = cmd.resp_pkt; /* Extract NVM response */ nvm_resp = (void *)pkt->data; ret = le16toh(nvm_resp->status); bytes_read = le16toh(nvm_resp->length); offset_read = le16toh(nvm_resp->offset); resp_data = nvm_resp->data; if (ret) { if ((offset != 0) && (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) { /* * meaning of NOT_VALID_ADDRESS: * driver try to read chunk from address that is * multiple of 2K and got an error since addr is empty. * meaning of (offset != 0): driver already * read valid data from another chunk so this case * is not an error. */ IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM access command failed on offset 0x%x since that section size is multiple 2K\n", offset); *len = 0; ret = 0; } else { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM access command failed with status %d\n", ret); ret = EIO; } goto exit; } if (offset_read != offset) { device_printf(sc->sc_dev, "NVM ACCESS response with invalid offset %d\n", offset_read); ret = EINVAL; goto exit; } if (bytes_read > length) { device_printf(sc->sc_dev, "NVM ACCESS response with too much data " "(%d bytes requested, %d bytes received)\n", length, bytes_read); ret = EINVAL; goto exit; } /* Write data to NVM */ memcpy(data + offset, resp_data, bytes_read); *len = bytes_read; exit: iwm_free_resp(sc, &cmd); return ret; } /* * Reads an NVM section completely. * NICs prior to 7000 family don't have a real NVM, but just read * section 0 which is the EEPROM. Because the EEPROM reading is unlimited * by uCode, we need to manually check in this case that we don't * overflow and try to read more than the EEPROM size. * For 7000 family NICs, we supply the maximal size we can read, and * the uCode fills the response with as much data as we can, * without overflowing, so no check is needed. */ static int iwm_nvm_read_section(struct iwm_softc *sc, uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read) { uint16_t seglen, length, offset = 0; int ret; /* Set nvm section read length */ length = IWM_NVM_DEFAULT_CHUNK_SIZE; seglen = length; /* Read the NVM until exhausted (reading less than requested) */ while (seglen == length) { /* Check no memory assumptions fail and cause an overflow */ if ((size_read + offset + length) > sc->cfg->eeprom_size) { device_printf(sc->sc_dev, "EEPROM size is too small for NVM\n"); return ENOBUFS; } ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen); if (ret) { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "Cannot read NVM from section %d offset %d, length %d\n", section, offset, length); return ret; } offset += seglen; } IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "NVM section %d read completed\n", section); *len = offset; return 0; } /* * BEGIN IWM_NVM_PARSE */ /* iwlwifi/iwl-nvm-parse.c */ /* NVM offsets (in words) definitions */ enum iwm_nvm_offsets { /* NVM HW-Section offset (in words) definitions */ IWM_HW_ADDR = 0x15, /* NVM SW-Section offset (in words) definitions */ IWM_NVM_SW_SECTION = 0x1C0, IWM_NVM_VERSION = 0, IWM_RADIO_CFG = 1, IWM_SKU = 2, IWM_N_HW_ADDRS = 3, IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION, /* NVM calibration section offset (in words) definitions */ IWM_NVM_CALIB_SECTION = 0x2B8, IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION }; enum iwm_8000_nvm_offsets { /* NVM HW-Section offset (in words) definitions */ IWM_HW_ADDR0_WFPM_8000 = 0x12, IWM_HW_ADDR1_WFPM_8000 = 0x16, IWM_HW_ADDR0_PCIE_8000 = 0x8A, IWM_HW_ADDR1_PCIE_8000 = 0x8E, IWM_MAC_ADDRESS_OVERRIDE_8000 = 1, /* NVM SW-Section offset (in words) definitions */ IWM_NVM_SW_SECTION_8000 = 0x1C0, IWM_NVM_VERSION_8000 = 0, IWM_RADIO_CFG_8000 = 0, IWM_SKU_8000 = 2, IWM_N_HW_ADDRS_8000 = 3, /* NVM REGULATORY -Section offset (in words) definitions */ IWM_NVM_CHANNELS_8000 = 0, IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7, IWM_NVM_LAR_OFFSET_8000 = 0x507, IWM_NVM_LAR_ENABLED_8000 = 0x7, /* NVM calibration section offset (in words) definitions */ IWM_NVM_CALIB_SECTION_8000 = 0x2B8, IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000 }; /* SKU Capabilities (actual values from NVM definition) */ enum nvm_sku_bits { IWM_NVM_SKU_CAP_BAND_24GHZ = (1 << 0), IWM_NVM_SKU_CAP_BAND_52GHZ = (1 << 1), IWM_NVM_SKU_CAP_11N_ENABLE = (1 << 2), IWM_NVM_SKU_CAP_11AC_ENABLE = (1 << 3), }; /* radio config bits (actual values from NVM definition) */ #define IWM_NVM_RF_CFG_DASH_MSK(x) (x & 0x3) /* bits 0-1 */ #define IWM_NVM_RF_CFG_STEP_MSK(x) ((x >> 2) & 0x3) /* bits 2-3 */ #define IWM_NVM_RF_CFG_TYPE_MSK(x) ((x >> 4) & 0x3) /* bits 4-5 */ #define IWM_NVM_RF_CFG_PNUM_MSK(x) ((x >> 6) & 0x3) /* bits 6-7 */ #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8) & 0xF) /* bits 8-11 */ #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */ #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x) (x & 0xF) #define IWM_NVM_RF_CFG_DASH_MSK_8000(x) ((x >> 4) & 0xF) #define IWM_NVM_RF_CFG_STEP_MSK_8000(x) ((x >> 8) & 0xF) #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x) ((x >> 12) & 0xFFF) #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x) ((x >> 24) & 0xF) #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x) ((x >> 28) & 0xF) /** * enum iwm_nvm_channel_flags - channel flags in NVM * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed * @IWM_NVM_CHANNEL_RADAR: radar detection required * XXX cannot find this (DFS) flag in iwm-nvm-parse.c * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?) * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?) * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?) * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?) */ enum iwm_nvm_channel_flags { IWM_NVM_CHANNEL_VALID = (1 << 0), IWM_NVM_CHANNEL_IBSS = (1 << 1), IWM_NVM_CHANNEL_ACTIVE = (1 << 3), IWM_NVM_CHANNEL_RADAR = (1 << 4), IWM_NVM_CHANNEL_DFS = (1 << 7), IWM_NVM_CHANNEL_WIDE = (1 << 8), IWM_NVM_CHANNEL_40MHZ = (1 << 9), IWM_NVM_CHANNEL_80MHZ = (1 << 10), IWM_NVM_CHANNEL_160MHZ = (1 << 11), }; /* * Translate EEPROM flags to net80211. */ static uint32_t iwm_eeprom_channel_flags(uint16_t ch_flags) { uint32_t nflags; nflags = 0; if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0) nflags |= IEEE80211_CHAN_PASSIVE; if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0) nflags |= IEEE80211_CHAN_NOADHOC; if (ch_flags & IWM_NVM_CHANNEL_RADAR) { nflags |= IEEE80211_CHAN_DFS; /* Just in case. */ nflags |= IEEE80211_CHAN_NOADHOC; } return (nflags); } static void iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[], int maxchans, int *nchans, int ch_idx, size_t ch_num, const uint8_t bands[]) { const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags; uint32_t nflags; uint16_t ch_flags; uint8_t ieee; int error; for (; ch_idx < ch_num; ch_idx++) { ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) ieee = iwm_nvm_channels[ch_idx]; else ieee = iwm_nvm_channels_8000[ch_idx]; if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Ch. %d Flags %x [%sGHz] - No traffic\n", ieee, ch_flags, (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ? "5.2" : "2.4"); continue; } nflags = iwm_eeprom_channel_flags(ch_flags); error = ieee80211_add_channel(chans, maxchans, nchans, ieee, 0, 0, nflags, bands); if (error != 0) break; IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Ch. %d Flags %x [%sGHz] - Added\n", ieee, ch_flags, (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ? "5.2" : "2.4"); } } static void iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans, struct ieee80211_channel chans[]) { struct iwm_softc *sc = ic->ic_softc; struct iwm_nvm_data *data = sc->nvm_data; uint8_t bands[IEEE80211_MODE_BYTES]; size_t ch_num; memset(bands, 0, sizeof(bands)); /* 1-13: 11b/g channels. */ setbit(bands, IEEE80211_MODE_11B); setbit(bands, IEEE80211_MODE_11G); iwm_add_channel_band(sc, chans, maxchans, nchans, 0, IWM_NUM_2GHZ_CHANNELS - 1, bands); /* 14: 11b channel only. */ clrbit(bands, IEEE80211_MODE_11G); iwm_add_channel_band(sc, chans, maxchans, nchans, IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands); if (data->sku_cap_band_52GHz_enable) { if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) ch_num = nitems(iwm_nvm_channels); else ch_num = nitems(iwm_nvm_channels_8000); memset(bands, 0, sizeof(bands)); setbit(bands, IEEE80211_MODE_11A); iwm_add_channel_band(sc, chans, maxchans, nchans, IWM_NUM_2GHZ_CHANNELS, ch_num, bands); } } static void iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data, const uint16_t *mac_override, const uint16_t *nvm_hw) { const uint8_t *hw_addr; if (mac_override) { static const uint8_t reserved_mac[] = { 0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00 }; hw_addr = (const uint8_t *)(mac_override + IWM_MAC_ADDRESS_OVERRIDE_8000); /* * Store the MAC address from MAO section. * No byte swapping is required in MAO section */ IEEE80211_ADDR_COPY(data->hw_addr, hw_addr); /* * Force the use of the OTP MAC address in case of reserved MAC * address in the NVM, or if address is given but invalid. */ if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) && !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) && iwm_is_valid_ether_addr(data->hw_addr) && !IEEE80211_IS_MULTICAST(data->hw_addr)) return; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: mac address from nvm override section invalid\n", __func__); } if (nvm_hw) { /* read the mac address from WFMP registers */ uint32_t mac_addr0 = htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0)); uint32_t mac_addr1 = htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1)); hw_addr = (const uint8_t *)&mac_addr0; data->hw_addr[0] = hw_addr[3]; data->hw_addr[1] = hw_addr[2]; data->hw_addr[2] = hw_addr[1]; data->hw_addr[3] = hw_addr[0]; hw_addr = (const uint8_t *)&mac_addr1; data->hw_addr[4] = hw_addr[1]; data->hw_addr[5] = hw_addr[0]; return; } device_printf(sc->sc_dev, "%s: mac address not found\n", __func__); memset(data->hw_addr, 0, sizeof(data->hw_addr)); } static int iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw, const uint16_t *phy_sku) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_SKU); return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000)); } static int iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_NVM_VERSION); else return le32_to_cpup((const uint32_t *)(nvm_sw + IWM_NVM_VERSION_8000)); } static int iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw, const uint16_t *phy_sku) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_RADIO_CFG); return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000)); } static int iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw) { int n_hw_addr; if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS); n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000)); return n_hw_addr & IWM_N_HW_ADDR_MASK; } static void iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data, uint32_t radio_cfg) { if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg); data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg); data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg); data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg); return; } /* set the radio configuration for family 8000 */ data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg); data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg); data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg); data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg); data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg); data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg); } static int iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data, const uint16_t *nvm_hw, const uint16_t *mac_override) { #ifdef notyet /* for FAMILY 9000 */ if (cfg->mac_addr_from_csr) { iwm_set_hw_address_from_csr(sc, data); } else #endif if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR); /* The byte order is little endian 16 bit, meaning 214365 */ data->hw_addr[0] = hw_addr[1]; data->hw_addr[1] = hw_addr[0]; data->hw_addr[2] = hw_addr[3]; data->hw_addr[3] = hw_addr[2]; data->hw_addr[4] = hw_addr[5]; data->hw_addr[5] = hw_addr[4]; } else { iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw); } if (!iwm_is_valid_ether_addr(data->hw_addr)) { device_printf(sc->sc_dev, "no valid mac address was found\n"); return EINVAL; } return 0; } static struct iwm_nvm_data * iwm_parse_nvm_data(struct iwm_softc *sc, const uint16_t *nvm_hw, const uint16_t *nvm_sw, const uint16_t *nvm_calib, const uint16_t *mac_override, const uint16_t *phy_sku, const uint16_t *regulatory) { struct iwm_nvm_data *data; uint32_t sku, radio_cfg; uint16_t lar_config; if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) { data = malloc(sizeof(*data) + IWM_NUM_CHANNELS * sizeof(uint16_t), M_DEVBUF, M_NOWAIT | M_ZERO); } else { data = malloc(sizeof(*data) + IWM_NUM_CHANNELS_8000 * sizeof(uint16_t), M_DEVBUF, M_NOWAIT | M_ZERO); } if (!data) return NULL; data->nvm_version = iwm_get_nvm_version(sc, nvm_sw); radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku); iwm_set_radio_cfg(sc, data, radio_cfg); sku = iwm_get_sku(sc, nvm_sw, phy_sku); data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ; data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ; data->sku_cap_11n_enable = 0; data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw); if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { uint16_t lar_offset = data->nvm_version < 0xE39 ? IWM_NVM_LAR_OFFSET_8000_OLD : IWM_NVM_LAR_OFFSET_8000; lar_config = le16_to_cpup(regulatory + lar_offset); data->lar_enabled = !!(lar_config & IWM_NVM_LAR_ENABLED_8000); } /* If no valid mac address was found - bail out */ if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) { free(data, M_DEVBUF); return NULL; } if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { - memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS], + memcpy(data->nvm_ch_flags, sc->cfg->nvm_type == IWM_NVM_SDP ? + ®ulatory[0] : &nvm_sw[IWM_NVM_CHANNELS], IWM_NUM_CHANNELS * sizeof(uint16_t)); } else { memcpy(data->nvm_ch_flags, ®ulatory[IWM_NVM_CHANNELS_8000], IWM_NUM_CHANNELS_8000 * sizeof(uint16_t)); } return data; } static void iwm_free_nvm_data(struct iwm_nvm_data *data) { if (data != NULL) free(data, M_DEVBUF); } static struct iwm_nvm_data * iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections) { const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku; /* Checking for required sections */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) { if (!sections[IWM_NVM_SECTION_TYPE_SW].data || !sections[sc->cfg->nvm_hw_section_num].data) { device_printf(sc->sc_dev, "Can't parse empty OTP/NVM sections\n"); return NULL; } } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { /* SW and REGULATORY sections are mandatory */ if (!sections[IWM_NVM_SECTION_TYPE_SW].data || !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) { device_printf(sc->sc_dev, "Can't parse empty OTP/NVM sections\n"); return NULL; } /* MAC_OVERRIDE or at least HW section must exist */ if (!sections[sc->cfg->nvm_hw_section_num].data && !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) { device_printf(sc->sc_dev, "Can't parse mac_address, empty sections\n"); return NULL; } /* PHY_SKU section is mandatory in B0 */ if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) { device_printf(sc->sc_dev, "Can't parse phy_sku in B0, empty sections\n"); return NULL; } } else { panic("unknown device family %d\n", sc->cfg->device_family); } hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data; sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data; calib = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data; - regulatory = (const uint16_t *) - sections[IWM_NVM_SECTION_TYPE_REGULATORY].data; + regulatory = sc->cfg->nvm_type == IWM_NVM_SDP ? + (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_REGULATORY_SDP].data : + (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_REGULATORY].data; mac_override = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data; phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data; return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override, phy_sku, regulatory); } static int iwm_nvm_init(struct iwm_softc *sc) { struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS]; int i, ret, section; uint32_t size_read = 0; uint8_t *nvm_buffer, *temp; uint16_t len; memset(nvm_sections, 0, sizeof(nvm_sections)); if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS) return EINVAL; /* load NVM values from nic */ /* Read From FW NVM */ IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n"); nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO); if (!nvm_buffer) return ENOMEM; for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) { /* we override the constness for initial read */ ret = iwm_nvm_read_section(sc, section, nvm_buffer, &len, size_read); if (ret) continue; size_read += len; temp = malloc(len, M_DEVBUF, M_NOWAIT); if (!temp) { ret = ENOMEM; break; } memcpy(temp, nvm_buffer, len); nvm_sections[section].data = temp; nvm_sections[section].length = len; } if (!size_read) device_printf(sc->sc_dev, "OTP is blank\n"); free(nvm_buffer, M_DEVBUF); sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections); if (!sc->nvm_data) return EINVAL; IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET, "nvm version = %x\n", sc->nvm_data->nvm_version); for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) { if (nvm_sections[i].data != NULL) free(nvm_sections[i].data, M_DEVBUF); } return 0; } static int iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num, const struct iwm_fw_desc *section) { struct iwm_dma_info *dma = &sc->fw_dma; uint8_t *v_addr; bus_addr_t p_addr; uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len); int ret = 0; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "%s: [%d] uCode section being loaded...\n", __func__, section_num); v_addr = dma->vaddr; p_addr = dma->paddr; for (offset = 0; offset < section->len; offset += chunk_sz) { uint32_t copy_size, dst_addr; int extended_addr = FALSE; copy_size = MIN(chunk_sz, section->len - offset); dst_addr = section->offset + offset; if (dst_addr >= IWM_FW_MEM_EXTENDED_START && dst_addr <= IWM_FW_MEM_EXTENDED_END) extended_addr = TRUE; if (extended_addr) iwm_set_bits_prph(sc, IWM_LMPM_CHICK, IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); memcpy(v_addr, (const uint8_t *)section->data + offset, copy_size); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr, copy_size); if (extended_addr) iwm_clear_bits_prph(sc, IWM_LMPM_CHICK, IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); if (ret) { device_printf(sc->sc_dev, "%s: Could not load the [%d] uCode section\n", __func__, section_num); break; } } return ret; } /* * ucode */ static int iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr, bus_addr_t phy_addr, uint32_t byte_cnt) { sc->sc_fw_chunk_done = 0; if (!iwm_nic_lock(sc)) return EBUSY; IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE); IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL), dst_addr); IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL), phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK); IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL), (iwm_get_dma_hi_addr(phy_addr) << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt); IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL), 1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM | 1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX | IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID); IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE | IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD); iwm_nic_unlock(sc); /* wait up to 5s for this segment to load */ msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz * 5); if (!sc->sc_fw_chunk_done) { device_printf(sc->sc_dev, "fw chunk addr 0x%x len %d failed to load\n", dst_addr, byte_cnt); return ETIMEDOUT; } return 0; } static int iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc, const struct iwm_fw_img *image, int cpu, int *first_ucode_section) { int shift_param; int i, ret = 0, sec_num = 0x1; uint32_t val, last_read_idx = 0; if (cpu == 1) { shift_param = 0; *first_ucode_section = 0; } else { shift_param = 16; (*first_ucode_section)++; } for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) { last_read_idx = i; /* * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between * CPU1 to CPU2. * PAGING_SEPARATOR_SECTION delimiter - separate between * CPU2 non paged to CPU2 paging sec. */ if (!image->sec[i].data || image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION || image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) { IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Break since Data not valid or Empty section, sec = %d\n", i); break; } ret = iwm_pcie_load_section(sc, i, &image->sec[i]); if (ret) return ret; /* Notify the ucode of the loaded section number and status */ if (iwm_nic_lock(sc)) { val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS); val = val | (sec_num << shift_param); IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val); sec_num = (sec_num << 1) | 0x1; iwm_nic_unlock(sc); } } *first_ucode_section = last_read_idx; iwm_enable_interrupts(sc); if (iwm_nic_lock(sc)) { if (cpu == 1) IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF); else IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF); iwm_nic_unlock(sc); } return 0; } static int iwm_pcie_load_cpu_sections(struct iwm_softc *sc, const struct iwm_fw_img *image, int cpu, int *first_ucode_section) { int shift_param; int i, ret = 0; uint32_t last_read_idx = 0; if (cpu == 1) { shift_param = 0; *first_ucode_section = 0; } else { shift_param = 16; (*first_ucode_section)++; } for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) { last_read_idx = i; /* * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between * CPU1 to CPU2. * PAGING_SEPARATOR_SECTION delimiter - separate between * CPU2 non paged to CPU2 paging sec. */ if (!image->sec[i].data || image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION || image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) { IWM_DPRINTF(sc, IWM_DEBUG_RESET, "Break since Data not valid or Empty section, sec = %d\n", i); break; } ret = iwm_pcie_load_section(sc, i, &image->sec[i]); if (ret) return ret; } *first_ucode_section = last_read_idx; return 0; } static int iwm_pcie_load_given_ucode(struct iwm_softc *sc, const struct iwm_fw_img *image) { int ret = 0; int first_ucode_section; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n", image->is_dual_cpus ? "Dual" : "Single"); /* load to FW the binary non secured sections of CPU1 */ ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section); if (ret) return ret; if (image->is_dual_cpus) { /* set CPU2 header address */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR, IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE); iwm_nic_unlock(sc); } /* load to FW the binary sections of CPU2 */ ret = iwm_pcie_load_cpu_sections(sc, image, 2, &first_ucode_section); if (ret) return ret; } iwm_enable_interrupts(sc); /* release CPU reset */ IWM_WRITE(sc, IWM_CSR_RESET, 0); return 0; } int iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc, const struct iwm_fw_img *image) { int ret = 0; int first_ucode_section; IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n", image->is_dual_cpus ? "Dual" : "Single"); /* configure the ucode to be ready to get the secured image */ /* release CPU reset */ if (iwm_nic_lock(sc)) { iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT); iwm_nic_unlock(sc); } /* load to FW the binary Secured sections of CPU1 */ ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1, &first_ucode_section); if (ret) return ret; /* load to FW the binary sections of CPU2 */ return iwm_pcie_load_cpu_sections_8000(sc, image, 2, &first_ucode_section); } /* XXX Get rid of this definition */ static inline void iwm_enable_fw_load_int(struct iwm_softc *sc) { IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n"); sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX; IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask); } /* XXX Add proper rfkill support code */ static int iwm_start_fw(struct iwm_softc *sc, const struct iwm_fw_img *fw) { int ret; /* This may fail if AMT took ownership of the device */ if (iwm_prepare_card_hw(sc)) { device_printf(sc->sc_dev, "%s: Exit HW not ready\n", __func__); ret = EIO; goto out; } IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF); iwm_disable_interrupts(sc); /* make sure rfkill handshake bits are cleared */ IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED); /* clear (again), then enable host interrupts */ IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF); ret = iwm_nic_init(sc); if (ret) { device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__); goto out; } /* * Now, we load the firmware and don't want to be interrupted, even * by the RF-Kill interrupt (hence mask all the interrupt besides the * FH_TX interrupt which is needed to load the firmware). If the * RF-Kill switch is toggled, we will find out after having loaded * the firmware and return the proper value to the caller. */ iwm_enable_fw_load_int(sc); /* really make sure rfkill handshake bits are cleared */ /* maybe we should write a few times more? just to make sure */ IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL); /* Load the given image to the HW */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) ret = iwm_pcie_load_given_ucode_8000(sc, fw); else ret = iwm_pcie_load_given_ucode(sc, fw); /* XXX re-check RF-Kill state */ out: return ret; } static int iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant) { struct iwm_tx_ant_cfg_cmd tx_ant_cmd = { .valid = htole32(valid_tx_ant), }; return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD, IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd); } /* iwlwifi: mvm/fw.c */ static int iwm_send_phy_cfg_cmd(struct iwm_softc *sc) { struct iwm_phy_cfg_cmd phy_cfg_cmd; enum iwm_ucode_type ucode_type = sc->cur_ucode; /* Set parameters */ phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc)); phy_cfg_cmd.calib_control.event_trigger = sc->sc_default_calib[ucode_type].event_trigger; phy_cfg_cmd.calib_control.flow_trigger = sc->sc_default_calib[ucode_type].flow_trigger; IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET, "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg); return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC, sizeof(phy_cfg_cmd), &phy_cfg_cmd); } static int iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data) { struct iwm_mvm_alive_data *alive_data = data; struct iwm_mvm_alive_resp_v3 *palive3; struct iwm_mvm_alive_resp *palive; struct iwm_umac_alive *umac; struct iwm_lmac_alive *lmac1; struct iwm_lmac_alive *lmac2 = NULL; uint16_t status; if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) { palive = (void *)pkt->data; umac = &palive->umac_data; lmac1 = &palive->lmac_data[0]; lmac2 = &palive->lmac_data[1]; status = le16toh(palive->status); } else { palive3 = (void *)pkt->data; umac = &palive3->umac_data; lmac1 = &palive3->lmac_data; status = le16toh(palive3->status); } sc->error_event_table[0] = le32toh(lmac1->error_event_table_ptr); if (lmac2) sc->error_event_table[1] = le32toh(lmac2->error_event_table_ptr); sc->log_event_table = le32toh(lmac1->log_event_table_ptr); sc->umac_error_event_table = le32toh(umac->error_info_addr); alive_data->scd_base_addr = le32toh(lmac1->scd_base_ptr); alive_data->valid = status == IWM_ALIVE_STATUS_OK; if (sc->umac_error_event_table) sc->support_umac_log = TRUE; IWM_DPRINTF(sc, IWM_DEBUG_FW, "Alive ucode status 0x%04x revision 0x%01X 0x%01X\n", status, lmac1->ver_type, lmac1->ver_subtype); if (lmac2) IWM_DPRINTF(sc, IWM_DEBUG_FW, "Alive ucode CDB\n"); IWM_DPRINTF(sc, IWM_DEBUG_FW, "UMAC version: Major - 0x%x, Minor - 0x%x\n", le32toh(umac->umac_major), le32toh(umac->umac_minor)); return TRUE; } static int iwm_wait_phy_db_entry(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data) { struct iwm_phy_db *phy_db = data; if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) { if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) { device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n", __func__, pkt->hdr.code); } return TRUE; } if (iwm_phy_db_set_section(phy_db, pkt)) { device_printf(sc->sc_dev, "%s: iwm_phy_db_set_section failed\n", __func__); } return FALSE; } static int iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) { struct iwm_notification_wait alive_wait; struct iwm_mvm_alive_data alive_data; const struct iwm_fw_img *fw; enum iwm_ucode_type old_type = sc->cur_ucode; int error; static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE }; fw = &sc->sc_fw.img[ucode_type]; sc->cur_ucode = ucode_type; sc->ucode_loaded = FALSE; memset(&alive_data, 0, sizeof(alive_data)); iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait, alive_cmd, nitems(alive_cmd), iwm_alive_fn, &alive_data); error = iwm_start_fw(sc, fw); if (error) { device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error); sc->cur_ucode = old_type; iwm_remove_notification(sc->sc_notif_wait, &alive_wait); return error; } /* * Some things may run in the background now, but we * just wait for the ALIVE notification here. */ IWM_UNLOCK(sc); error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait, IWM_MVM_UCODE_ALIVE_TIMEOUT); IWM_LOCK(sc); if (error) { if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { uint32_t a = 0x5a5a5a5a, b = 0x5a5a5a5a; if (iwm_nic_lock(sc)) { a = iwm_read_prph(sc, IWM_SB_CPU_1_STATUS); b = iwm_read_prph(sc, IWM_SB_CPU_2_STATUS); iwm_nic_unlock(sc); } device_printf(sc->sc_dev, "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n", a, b); } sc->cur_ucode = old_type; return error; } if (!alive_data.valid) { device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n", __func__); sc->cur_ucode = old_type; return EIO; } iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr); /* * configure and operate fw paging mechanism. * driver configures the paging flow only once, CPU2 paging image * included in the IWM_UCODE_INIT image. */ if (fw->paging_mem_size) { error = iwm_save_fw_paging(sc, fw); if (error) { device_printf(sc->sc_dev, "%s: failed to save the FW paging image\n", __func__); return error; } error = iwm_send_paging_cmd(sc, fw); if (error) { device_printf(sc->sc_dev, "%s: failed to send the paging cmd\n", __func__); iwm_free_fw_paging(sc); return error; } } if (!error) sc->ucode_loaded = TRUE; return error; } /* * mvm misc bits */ /* * follows iwlwifi/fw.c */ static int iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm) { struct iwm_notification_wait calib_wait; static const uint16_t init_complete[] = { IWM_INIT_COMPLETE_NOTIF, IWM_CALIB_RES_NOTIF_PHY_DB }; int ret; /* do not operate with rfkill switch turned on */ if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) { device_printf(sc->sc_dev, "radio is disabled by hardware switch\n"); return EPERM; } iwm_init_notification_wait(sc->sc_notif_wait, &calib_wait, init_complete, nitems(init_complete), iwm_wait_phy_db_entry, sc->sc_phy_db); /* Will also start the device */ ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT); if (ret) { device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n", ret); goto error; } if (justnvm) { /* Read nvm */ ret = iwm_nvm_init(sc); if (ret) { device_printf(sc->sc_dev, "failed to read nvm\n"); goto error; } IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr); goto error; } ret = iwm_send_bt_init_conf(sc); if (ret) { device_printf(sc->sc_dev, "failed to send bt coex configuration: %d\n", ret); goto error; } /* Send TX valid antennas before triggering calibrations */ ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc)); if (ret) { device_printf(sc->sc_dev, "failed to send antennas before calibration: %d\n", ret); goto error; } /* * Send phy configurations command to init uCode * to start the 16.0 uCode init image internal calibrations. */ ret = iwm_send_phy_cfg_cmd(sc); if (ret) { device_printf(sc->sc_dev, "%s: Failed to run INIT calibrations: %d\n", __func__, ret); goto error; } /* * Nothing to do but wait for the init complete notification * from the firmware. */ IWM_UNLOCK(sc); ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait, IWM_MVM_UCODE_CALIB_TIMEOUT); IWM_LOCK(sc); goto out; error: iwm_remove_notification(sc->sc_notif_wait, &calib_wait); out: return ret; } static int iwm_mvm_config_ltr(struct iwm_softc *sc) { struct iwm_ltr_config_cmd cmd = { .flags = htole32(IWM_LTR_CFG_FLAG_FEATURE_ENABLE), }; if (!sc->sc_ltr_enabled) return 0; return iwm_mvm_send_cmd_pdu(sc, IWM_LTR_CONFIG, 0, sizeof(cmd), &cmd); } /* * receive side */ /* (re)stock rx ring, called at init-time and at runtime */ static int iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx) { struct iwm_rx_ring *ring = &sc->rxq; struct iwm_rx_data *data = &ring->data[idx]; struct mbuf *m; bus_dmamap_t dmamap; bus_dma_segment_t seg; int nsegs, error; m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE); if (m == NULL) return ENOBUFS; m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m, &seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); m_freem(m); return error; } if (data->m != NULL) bus_dmamap_unload(ring->data_dmat, data->map); /* Swap ring->spare_map with data->map */ dmamap = data->map; data->map = ring->spare_map; ring->spare_map = dmamap; bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD); data->m = m; /* Update RX descriptor. */ KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned")); ring->desc[idx] = htole32(seg.ds_addr >> 8); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); return 0; } /* iwlwifi: mvm/rx.c */ /* * iwm_mvm_get_signal_strength - use new rx PHY INFO API * values are reported by the fw as positive values - need to negate * to obtain their dBM. Account for missing antennas by replacing 0 * values by -256dBm: practically 0 power and a non-feasible 8 bit value. */ static int iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info) { int energy_a, energy_b, energy_c, max_energy; uint32_t val; val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]); energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >> IWM_RX_INFO_ENERGY_ANT_A_POS; energy_a = energy_a ? -energy_a : -256; energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >> IWM_RX_INFO_ENERGY_ANT_B_POS; energy_b = energy_b ? -energy_b : -256; energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >> IWM_RX_INFO_ENERGY_ANT_C_POS; energy_c = energy_c ? -energy_c : -256; max_energy = MAX(energy_a, energy_b); max_energy = MAX(max_energy, energy_c); IWM_DPRINTF(sc, IWM_DEBUG_RECV, "energy In A %d B %d C %d , and max %d\n", energy_a, energy_b, energy_c, max_energy); return max_energy; } static void iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_rx_phy_info *phy_info = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n"); memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info)); } /* * Retrieve the average noise (in dBm) among receivers. */ static int iwm_get_noise(struct iwm_softc *sc, const struct iwm_mvm_statistics_rx_non_phy *stats) { int i, total, nbant, noise; total = nbant = noise = 0; for (i = 0; i < 3; i++) { noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n", __func__, i, noise); if (noise) { total += noise; nbant++; } } IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n", __func__, nbant, total); #if 0 /* There should be at least one antenna but check anyway. */ return (nbant == 0) ? -127 : (total / nbant) - 107; #else /* For now, just hard-code it to -96 to be safe */ return (-96); #endif } static void iwm_mvm_handle_rx_statistics(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_notif_statistics_v10 *stats = (void *)&pkt->data; memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats)); sc->sc_noise = iwm_get_noise(sc, &stats->rx.general); } /* * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler * * Handles the actual data of the Rx packet from the fw */ static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset, boolean_t stolen) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct ieee80211_frame *wh; struct ieee80211_node *ni; struct ieee80211_rx_stats rxs; struct iwm_rx_phy_info *phy_info; struct iwm_rx_mpdu_res_start *rx_res; struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset); uint32_t len; uint32_t rx_pkt_status; int rssi; phy_info = &sc->sc_last_phy_info; rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data; wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res)); len = le16toh(rx_res->byte_count); rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len)); if (__predict_false(phy_info->cfg_phy_cnt > 20)) { device_printf(sc->sc_dev, "dsp size out of range [0,20]: %d\n", phy_info->cfg_phy_cnt); goto fail; } if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) || !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status); goto fail; } rssi = iwm_mvm_get_signal_strength(sc, phy_info); /* Map it to relative value */ rssi = rssi - sc->sc_noise; /* replenish ring for the buffer we're going to feed to the sharks */ if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) { device_printf(sc->sc_dev, "%s: unable to add more buffers\n", __func__); goto fail; } m->m_data = pkt->data + sizeof(*rx_res); m->m_pkthdr.len = m->m_len = len; IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise); ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh); IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: phy_info: channel=%d, flags=0x%08x\n", __func__, le16toh(phy_info->channel), le16toh(phy_info->phy_flags)); /* * Populate an RX state struct with the provided information. */ bzero(&rxs, sizeof(rxs)); rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ; rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI; rxs.c_ieee = le16toh(phy_info->channel); if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) { rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ); } else { rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ); } /* rssi is in 1/2db units */ rxs.c_rssi = rssi * 2; rxs.c_nf = sc->sc_noise; if (ieee80211_add_rx_params(m, &rxs) == 0) { if (ni) ieee80211_free_node(ni); goto fail; } if (ieee80211_radiotap_active_vap(vap)) { struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap; tap->wr_flags = 0; if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE)) tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE; tap->wr_chan_freq = htole16(rxs.c_freq); /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */ tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags); tap->wr_dbm_antsignal = (int8_t)rssi; tap->wr_dbm_antnoise = (int8_t)sc->sc_noise; tap->wr_tsft = phy_info->system_timestamp; switch (phy_info->rate) { /* CCK rates. */ case 10: tap->wr_rate = 2; break; case 20: tap->wr_rate = 4; break; case 55: tap->wr_rate = 11; break; case 110: tap->wr_rate = 22; break; /* OFDM rates. */ case 0xd: tap->wr_rate = 12; break; case 0xf: tap->wr_rate = 18; break; case 0x5: tap->wr_rate = 24; break; case 0x7: tap->wr_rate = 36; break; case 0x9: tap->wr_rate = 48; break; case 0xb: tap->wr_rate = 72; break; case 0x1: tap->wr_rate = 96; break; case 0x3: tap->wr_rate = 108; break; /* Unknown rate: should not happen. */ default: tap->wr_rate = 0; } } IWM_UNLOCK(sc); if (ni != NULL) { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m); ieee80211_input_mimo(ni, m); ieee80211_free_node(ni); } else { IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m); ieee80211_input_mimo_all(ic, m); } IWM_LOCK(sc); return TRUE; fail: counter_u64_add(ic->ic_ierrors, 1); return FALSE; } static int iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt, struct iwm_node *in) { struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data; struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs; struct ieee80211_node *ni = &in->in_ni; struct ieee80211vap *vap = ni->ni_vap; int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK; int new_rate, cur_rate = vap->iv_bss->ni_txrate; boolean_t rate_matched; uint8_t tx_resp_rate; KASSERT(tx_resp->frame_count == 1, ("too many frames")); /* Update rate control statistics. */ IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n", __func__, (int) le16toh(tx_resp->status.status), (int) le16toh(tx_resp->status.sequence), tx_resp->frame_count, tx_resp->bt_kill_count, tx_resp->failure_rts, tx_resp->failure_frame, le32toh(tx_resp->initial_rate), (int) le16toh(tx_resp->wireless_media_time)); tx_resp_rate = iwm_rate_from_ucode_rate(le32toh(tx_resp->initial_rate)); /* For rate control, ignore frames sent at different initial rate */ rate_matched = (tx_resp_rate != 0 && tx_resp_rate == cur_rate); if (tx_resp_rate != 0 && cur_rate != 0 && !rate_matched) { IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "tx_resp_rate doesn't match ni_txrate (tx_resp_rate=%u " "ni_txrate=%d)\n", tx_resp_rate, cur_rate); } txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY | IEEE80211_RATECTL_STATUS_LONG_RETRY; txs->short_retries = tx_resp->failure_rts; txs->long_retries = tx_resp->failure_frame; if (status != IWM_TX_STATUS_SUCCESS && status != IWM_TX_STATUS_DIRECT_DONE) { switch (status) { case IWM_TX_STATUS_FAIL_SHORT_LIMIT: txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT; break; case IWM_TX_STATUS_FAIL_LONG_LIMIT: txs->status = IEEE80211_RATECTL_TX_FAIL_LONG; break; case IWM_TX_STATUS_FAIL_LIFE_EXPIRE: txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED; break; default: txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED; break; } } else { txs->status = IEEE80211_RATECTL_TX_SUCCESS; } if (rate_matched) { ieee80211_ratectl_tx_complete(ni, txs); int rix = ieee80211_ratectl_rate(vap->iv_bss, NULL, 0); new_rate = vap->iv_bss->ni_txrate; if (new_rate != 0 && new_rate != cur_rate) { struct iwm_node *in = IWM_NODE(vap->iv_bss); iwm_setrates(sc, in, rix); iwm_mvm_send_lq_cmd(sc, &in->in_lq, FALSE); } } return (txs->status != IEEE80211_RATECTL_TX_SUCCESS); } static void iwm_mvm_rx_tx_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_cmd_header *cmd_hdr = &pkt->hdr; int idx = cmd_hdr->idx; int qid = cmd_hdr->qid; struct iwm_tx_ring *ring = &sc->txq[qid]; struct iwm_tx_data *txd = &ring->data[idx]; struct iwm_node *in = txd->in; struct mbuf *m = txd->m; int status; KASSERT(txd->done == 0, ("txd not done")); KASSERT(txd->in != NULL, ("txd without node")); KASSERT(txd->m != NULL, ("txd without mbuf")); sc->sc_tx_timer = 0; status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in); /* Unmap and free mbuf. */ bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, txd->map); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "free txd %p, in %p\n", txd, txd->in); txd->done = 1; txd->m = NULL; txd->in = NULL; ieee80211_tx_complete(&in->in_ni, m, status); if (--ring->queued < IWM_TX_RING_LOMARK) { sc->qfullmsk &= ~(1 << ring->qid); if (sc->qfullmsk == 0) { iwm_start(sc); } } } /* * transmit side */ /* * Process a "command done" firmware notification. This is where we wakeup * processes waiting for a synchronous command completion. * from if_iwn */ static void iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt) { struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE]; struct iwm_tx_data *data; if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) { return; /* Not a command ack. */ } /* XXX wide commands? */ IWM_DPRINTF(sc, IWM_DEBUG_CMD, "cmd notification type 0x%x qid %d idx %d\n", pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx); data = &ring->data[pkt->hdr.idx]; /* If the command was mapped in an mbuf, free it. */ if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } wakeup(&ring->desc[pkt->hdr.idx]); if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) { device_printf(sc->sc_dev, "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n", __func__, pkt->hdr.idx, ring->queued, ring->cur); /* XXX call iwm_force_nmi() */ } KASSERT(ring->queued > 0, ("ring->queued is empty?")); ring->queued--; if (ring->queued == 0) iwm_pcie_clear_cmd_in_flight(sc); } #if 0 /* * necessary only for block ack mode */ void iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id, uint16_t len) { struct iwm_agn_scd_bc_tbl *scd_bc_tbl; uint16_t w_val; scd_bc_tbl = sc->sched_dma.vaddr; len += 8; /* magic numbers came naturally from paris */ len = roundup(len, 4) / 4; w_val = htole16(sta_id << 12 | len); /* Update TX scheduler. */ scd_bc_tbl[qid].tfd_offset[idx] = w_val; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); /* I really wonder what this is ?!? */ if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) { scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); } } #endif static int iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate) { int i; for (i = 0; i < nitems(iwm_rates); i++) { if (iwm_rates[i].rate == rate) return (i); } /* XXX error? */ IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: couldn't find an entry for rate=%d\n", __func__, rate); return (0); } /* * Fill in the rate related information for a transmit command. */ static const struct iwm_rate * iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in, struct mbuf *m, struct iwm_tx_cmd *tx) { struct ieee80211_node *ni = &in->in_ni; struct ieee80211_frame *wh; const struct ieee80211_txparam *tp = ni->ni_txparms; const struct iwm_rate *rinfo; int type; int ridx, rate_flags; wh = mtod(m, struct ieee80211_frame *); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT; tx->data_retry_limit = IWM_DEFAULT_TX_RETRY; if (type == IEEE80211_FC0_TYPE_MGT || type == IEEE80211_FC0_TYPE_CTL || (m->m_flags & M_EAPOL) != 0) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: MGT (%d)\n", __func__, tp->mgmtrate); } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: MCAST (%d)\n", __func__, tp->mcastrate); } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) { ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate); } else { /* for data frames, use RS table */ IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__); ridx = iwm_rate2ridx(sc, ni->ni_txrate); if (ridx == -1) ridx = 0; /* This is the index into the programmed table */ tx->initial_rate_index = 0; tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE); } IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE, "%s: frame type=%d txrate %d\n", __func__, type, iwm_rates[ridx].rate); rinfo = &iwm_rates[ridx]; IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n", __func__, ridx, rinfo->rate, !! (IWM_RIDX_IS_CCK(ridx)) ); /* XXX TODO: hard-coded TX antenna? */ rate_flags = 1 << IWM_RATE_MCS_ANT_POS; if (IWM_RIDX_IS_CCK(ridx)) rate_flags |= IWM_RATE_MCS_CCK_MSK; tx->rate_n_flags = htole32(rate_flags | rinfo->plcp); return rinfo; } #define TB0_SIZE 16 static int iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_node *in = IWM_NODE(ni); struct iwm_tx_ring *ring; struct iwm_tx_data *data; struct iwm_tfd *desc; struct iwm_device_cmd *cmd; struct iwm_tx_cmd *tx; struct ieee80211_frame *wh; struct ieee80211_key *k = NULL; struct mbuf *m1; const struct iwm_rate *rinfo; uint32_t flags; u_int hdrlen; bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER]; int nsegs; uint8_t tid, type; int i, totlen, error, pad; wh = mtod(m, struct ieee80211_frame *); hdrlen = ieee80211_anyhdrsize(wh); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; tid = 0; ring = &sc->txq[ac]; desc = &ring->desc[ring->cur]; memset(desc, 0, sizeof(*desc)); data = &ring->data[ring->cur]; /* Fill out iwm_tx_cmd to send to the firmware */ cmd = &ring->cmd[ring->cur]; cmd->hdr.code = IWM_TX_CMD; cmd->hdr.flags = 0; cmd->hdr.qid = ring->qid; cmd->hdr.idx = ring->cur; tx = (void *)cmd->data; memset(tx, 0, sizeof(*tx)); rinfo = iwm_tx_fill_cmd(sc, in, m, tx); /* Encrypt the frame if need be. */ if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { /* Retrieve key for TX && do software encryption. */ k = ieee80211_crypto_encap(ni, m); if (k == NULL) { m_freem(m); return (ENOBUFS); } /* 802.11 header may have moved. */ wh = mtod(m, struct ieee80211_frame *); } if (ieee80211_radiotap_active_vap(vap)) { struct iwm_tx_radiotap_header *tap = &sc->sc_txtap; tap->wt_flags = 0; tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq); tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags); tap->wt_rate = rinfo->rate; if (k != NULL) tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP; ieee80211_radiotap_tx(vap, m); } totlen = m->m_pkthdr.len; flags = 0; if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= IWM_TX_CMD_FLG_ACK; } if (type == IEEE80211_FC0_TYPE_DATA && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold) && !IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= IWM_TX_CMD_FLG_PROT_REQUIRE; } if (IEEE80211_IS_MULTICAST(wh->i_addr1) || type != IEEE80211_FC0_TYPE_DATA) tx->sta_id = sc->sc_aux_sta.sta_id; else tx->sta_id = IWM_STATION_ID; if (type == IEEE80211_FC0_TYPE_MGT) { uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ || subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC); } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); } else { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT); } } else { tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); } if (hdrlen & 3) { /* First segment length must be a multiple of 4. */ flags |= IWM_TX_CMD_FLG_MH_PAD; pad = 4 - (hdrlen & 3); } else pad = 0; tx->driver_txop = 0; tx->next_frame_len = 0; tx->len = htole16(totlen); tx->tid_tspec = tid; tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE); /* Set physical address of "scratch area". */ tx->dram_lsb_ptr = htole32(data->scratch_paddr); tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr); /* Copy 802.11 header in TX command. */ memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen); flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL; tx->sec_ctl = 0; tx->tx_flags |= htole32(flags); /* Trim 802.11 header. */ m_adj(m, hdrlen); error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { if (error != EFBIG) { device_printf(sc->sc_dev, "can't map mbuf (error %d)\n", error); m_freem(m); return error; } /* Too many DMA segments, linearize mbuf. */ m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2); if (m1 == NULL) { device_printf(sc->sc_dev, "%s: could not defrag mbuf\n", __func__); m_freem(m); return (ENOBUFS); } m = m1; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "can't map mbuf (error %d)\n", error); m_freem(m); return error; } } data->m = m; data->in = in; data->done = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "sending txd %p, in %p\n", data, data->in); KASSERT(data->in != NULL, ("node is NULL")); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n", ring->qid, ring->cur, totlen, nsegs, le32toh(tx->tx_flags), le32toh(tx->rate_n_flags), tx->initial_rate_index ); /* Fill TX descriptor. */ desc->num_tbs = 2 + nsegs; desc->tbs[0].lo = htole32(data->cmd_paddr); desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) | (TB0_SIZE << 4); desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE); desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) | ((sizeof(struct iwm_cmd_header) + sizeof(*tx) + hdrlen + pad - TB0_SIZE) << 4); /* Other DMA segments are for data payload. */ for (i = 0; i < nsegs; i++) { seg = &segs[i]; desc->tbs[i+2].lo = htole32(seg->ds_addr); desc->tbs[i+2].hi_n_len = \ htole16(iwm_get_dma_hi_addr(seg->ds_addr)) | ((seg->ds_len) << 4); } bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); #if 0 iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len)); #endif /* Kick TX ring. */ ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT; IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); /* Mark TX ring as full if we reach a certain threshold. */ if (++ring->queued > IWM_TX_RING_HIMARK) { sc->qfullmsk |= 1 << ring->qid; } return 0; } static int iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_bpf_params *params) { struct ieee80211com *ic = ni->ni_ic; struct iwm_softc *sc = ic->ic_softc; int error = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "->%s begin\n", __func__); if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) { m_freem(m); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "<-%s not RUNNING\n", __func__); return (ENETDOWN); } IWM_LOCK(sc); /* XXX fix this */ if (params == NULL) { error = iwm_tx(sc, m, ni, 0); } else { error = iwm_tx(sc, m, ni, 0); } if (sc->sc_tx_timer == 0) callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); sc->sc_tx_timer = 5; IWM_UNLOCK(sc); return (error); } /* * mvm/tx.c */ /* * Note that there are transports that buffer frames before they reach * the firmware. This means that after flush_tx_path is called, the * queue might not be empty. The race-free way to handle this is to: * 1) set the station as draining * 2) flush the Tx path * 3) wait for the transport queues to be empty */ int iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags) { int ret; struct iwm_tx_path_flush_cmd flush_cmd = { .queues_ctl = htole32(tfd_msk), .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH), }; ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags, sizeof(flush_cmd), &flush_cmd); if (ret) device_printf(sc->sc_dev, "Flushing tx queue failed: %d\n", ret); return ret; } /* * BEGIN mvm/quota.c */ static int iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_vap *ivp) { struct iwm_time_quota_cmd cmd; int i, idx, ret, num_active_macs, quota, quota_rem; int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, }; int n_ifs[IWM_MAX_BINDINGS] = {0, }; uint16_t id; memset(&cmd, 0, sizeof(cmd)); /* currently, PHY ID == binding ID */ if (ivp) { id = ivp->phy_ctxt->id; KASSERT(id < IWM_MAX_BINDINGS, ("invalid id")); colors[id] = ivp->phy_ctxt->color; if (1) n_ifs[id] = 1; } /* * The FW's scheduling session consists of * IWM_MVM_MAX_QUOTA fragments. Divide these fragments * equally between all the bindings that require quota */ num_active_macs = 0; for (i = 0; i < IWM_MAX_BINDINGS; i++) { cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID); num_active_macs += n_ifs[i]; } quota = 0; quota_rem = 0; if (num_active_macs) { quota = IWM_MVM_MAX_QUOTA / num_active_macs; quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs; } for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) { if (colors[i] < 0) continue; cmd.quotas[idx].id_and_color = htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i])); if (n_ifs[i] <= 0) { cmd.quotas[idx].quota = htole32(0); cmd.quotas[idx].max_duration = htole32(0); } else { cmd.quotas[idx].quota = htole32(quota * n_ifs[i]); cmd.quotas[idx].max_duration = htole32(0); } idx++; } /* Give the remainder of the session to the first binding */ cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem); ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC, sizeof(cmd), &cmd); if (ret) device_printf(sc->sc_dev, "%s: Failed to send quota: %d\n", __func__, ret); return ret; } /* * END mvm/quota.c */ /* * ieee80211 routines */ /* * Change to AUTH state in 80211 state machine. Roughly matches what * Linux does in bss_info_changed(). */ static int iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc) { struct ieee80211_node *ni; struct iwm_node *in; struct iwm_vap *iv = IWM_VAP(vap); uint32_t duration; int error; /* * XXX i have a feeling that the vap node is being * freed from underneath us. Grr. */ ni = ieee80211_ref_node(vap->iv_bss); in = IWM_NODE(ni); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE, "%s: called; vap=%p, bss ni=%p\n", __func__, vap, ni); IWM_DPRINTF(sc, IWM_DEBUG_STATE, "%s: Current node bssid: %s\n", __func__, ether_sprintf(ni->ni_bssid)); in->in_assoc = 0; iv->iv_auth = 1; /* * Firmware bug - it'll crash if the beacon interval is less * than 16. We can't avoid connecting at all, so refuse the * station state change, this will cause net80211 to abandon * attempts to connect to this AP, and eventually wpa_s will * blacklist the AP... */ if (ni->ni_intval < 16) { device_printf(sc->sc_dev, "AP %s beacon interval is %d, refusing due to firmware bug!\n", ether_sprintf(ni->ni_bssid), ni->ni_intval); error = EINVAL; goto out; } error = iwm_allow_mcast(vap, sc); if (error) { device_printf(sc->sc_dev, "%s: failed to set multicast\n", __func__); goto out; } /* * This is where it deviates from what Linux does. * * Linux iwlwifi doesn't reset the nic each time, nor does it * call ctxt_add() here. Instead, it adds it during vap creation, * and always does a mac_ctx_changed(). * * The openbsd port doesn't attempt to do that - it reset things * at odd states and does the add here. * * So, until the state handling is fixed (ie, we never reset * the NIC except for a firmware failure, which should drag * the NIC back to IDLE, re-setup and re-add all the mac/phy * contexts that are required), let's do a dirty hack here. */ if (iv->is_uploaded) { if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC\n", __func__); goto out; } } else { if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to add MAC\n", __func__); goto out; } } sc->sc_firmware_state = 1; if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0], in->in_ni.ni_chan, 1, 1)) != 0) { device_printf(sc->sc_dev, "%s: failed update phy ctxt\n", __func__); goto out; } iv->phy_ctxt = &sc->sc_phyctxt[0]; if ((error = iwm_mvm_binding_add_vif(sc, iv)) != 0) { device_printf(sc->sc_dev, "%s: binding update cmd\n", __func__); goto out; } sc->sc_firmware_state = 2; /* * Authentication becomes unreliable when powersaving is left enabled * here. Powersaving will be activated again when association has * finished or is aborted. */ iv->ps_disabled = TRUE; error = iwm_mvm_power_update_mac(sc); iv->ps_disabled = FALSE; if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update power management\n", __func__); goto out; } if ((error = iwm_mvm_add_sta(sc, in)) != 0) { device_printf(sc->sc_dev, "%s: failed to add sta\n", __func__); goto out; } sc->sc_firmware_state = 3; /* * Prevent the FW from wandering off channel during association * by "protecting" the session with a time event. */ /* XXX duration is in units of TU, not MS */ duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS; iwm_mvm_protect_session(sc, iv, duration, 500 /* XXX magic number */, TRUE); error = 0; out: if (error != 0) iv->iv_auth = 0; ieee80211_free_node(ni); return (error); } static struct ieee80211_node * iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN]) { return malloc(sizeof (struct iwm_node), M_80211_NODE, M_NOWAIT | M_ZERO); } static uint8_t iwm_rate_from_ucode_rate(uint32_t rate_n_flags) { uint8_t plcp = rate_n_flags & 0xff; int i; for (i = 0; i <= IWM_RIDX_MAX; i++) { if (iwm_rates[i].plcp == plcp) return iwm_rates[i].rate; } return 0; } uint8_t iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx) { int i; uint8_t rval; for (i = 0; i < rs->rs_nrates; i++) { rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL); if (rval == iwm_rates[ridx].rate) return rs->rs_rates[i]; } return 0; } static int iwm_rate2ridx(struct iwm_softc *sc, uint8_t rate) { int i; for (i = 0; i <= IWM_RIDX_MAX; i++) { if (iwm_rates[i].rate == rate) return i; } device_printf(sc->sc_dev, "%s: WARNING: device rate for %u not found!\n", __func__, rate); return -1; } static void iwm_setrates(struct iwm_softc *sc, struct iwm_node *in, int rix) { struct ieee80211_node *ni = &in->in_ni; struct iwm_lq_cmd *lq = &in->in_lq; struct ieee80211_rateset *rs = &ni->ni_rates; int nrates = rs->rs_nrates; int i, ridx, tab = 0; // int txant = 0; KASSERT(rix >= 0 && rix < nrates, ("invalid rix")); if (nrates > nitems(lq->rs_table)) { device_printf(sc->sc_dev, "%s: node supports %d rates, driver handles " "only %zu\n", __func__, nrates, nitems(lq->rs_table)); return; } if (nrates == 0) { device_printf(sc->sc_dev, "%s: node supports 0 rates, odd!\n", __func__); return; } nrates = imin(rix + 1, nrates); IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: nrates=%d\n", __func__, nrates); /* then construct a lq_cmd based on those */ memset(lq, 0, sizeof(*lq)); lq->sta_id = IWM_STATION_ID; /* For HT, always enable RTS/CTS to avoid excessive retries. */ if (ni->ni_flags & IEEE80211_NODE_HT) lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK; /* * are these used? (we don't do SISO or MIMO) * need to set them to non-zero, though, or we get an error. */ lq->single_stream_ant_msk = 1; lq->dual_stream_ant_msk = 1; /* * Build the actual rate selection table. * The lowest bits are the rates. Additionally, * CCK needs bit 9 to be set. The rest of the bits * we add to the table select the tx antenna * Note that we add the rates in the highest rate first * (opposite of ni_rates). */ for (i = 0; i < nrates; i++) { int rate = rs->rs_rates[rix - i] & IEEE80211_RATE_VAL; int nextant; /* Map 802.11 rate to HW rate index. */ ridx = iwm_rate2ridx(sc, rate); if (ridx == -1) continue; #if 0 if (txant == 0) txant = iwm_mvm_get_valid_tx_ant(sc); nextant = 1<<(ffs(txant)-1); txant &= ~nextant; #else nextant = iwm_mvm_get_valid_tx_ant(sc); #endif tab = iwm_rates[ridx].plcp; tab |= nextant << IWM_RATE_MCS_ANT_POS; if (IWM_RIDX_IS_CCK(ridx)) tab |= IWM_RATE_MCS_CCK_MSK; IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "station rate i=%d, rate=%d, hw=%x\n", i, iwm_rates[ridx].rate, tab); lq->rs_table[i] = htole32(tab); } /* then fill the rest with the lowest possible rate */ for (i = nrates; i < nitems(lq->rs_table); i++) { KASSERT(tab != 0, ("invalid tab")); lq->rs_table[i] = htole32(tab); } } static int iwm_media_change(struct ifnet *ifp) { struct ieee80211vap *vap = ifp->if_softc; struct ieee80211com *ic = vap->iv_ic; struct iwm_softc *sc = ic->ic_softc; int error; error = ieee80211_media_change(ifp); if (error != ENETRESET) return error; IWM_LOCK(sc); if (ic->ic_nrunning > 0) { iwm_stop(sc); iwm_init(sc); } IWM_UNLOCK(sc); return error; } static void iwm_bring_down_firmware(struct iwm_softc *sc, struct ieee80211vap *vap) { struct iwm_vap *ivp = IWM_VAP(vap); int error; /* Avoid Tx watchdog triggering, when transfers get dropped here. */ sc->sc_tx_timer = 0; ivp->iv_auth = 0; if (sc->sc_firmware_state == 3) { iwm_xmit_queue_drain(sc); // iwm_mvm_flush_tx_path(sc, 0xf, IWM_CMD_SYNC); error = iwm_mvm_rm_sta(sc, vap, TRUE); if (error) { device_printf(sc->sc_dev, "%s: Failed to remove station: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error) { device_printf(sc->sc_dev, "%s: Failed to change mac context: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { error = iwm_mvm_sf_update(sc, vap, FALSE); if (error) { device_printf(sc->sc_dev, "%s: Failed to update smart FIFO: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { error = iwm_mvm_rm_sta_id(sc, vap); if (error) { device_printf(sc->sc_dev, "%s: Failed to remove station id: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { error = iwm_mvm_update_quotas(sc, NULL); if (error) { device_printf(sc->sc_dev, "%s: Failed to update PHY quota: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { /* XXX Might need to specify bssid correctly. */ error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error) { device_printf(sc->sc_dev, "%s: Failed to change mac context: %d\n", __func__, error); } } if (sc->sc_firmware_state == 3) { sc->sc_firmware_state = 2; } if (sc->sc_firmware_state > 1) { error = iwm_mvm_binding_remove_vif(sc, ivp); if (error) { device_printf(sc->sc_dev, "%s: Failed to remove channel ctx: %d\n", __func__, error); } } if (sc->sc_firmware_state > 1) { sc->sc_firmware_state = 1; } ivp->phy_ctxt = NULL; if (sc->sc_firmware_state > 0) { error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error) { device_printf(sc->sc_dev, "%s: Failed to change mac context: %d\n", __func__, error); } } if (sc->sc_firmware_state > 0) { error = iwm_mvm_power_update_mac(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update power management\n", __func__); } } sc->sc_firmware_state = 0; } static int iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct iwm_vap *ivp = IWM_VAP(vap); struct ieee80211com *ic = vap->iv_ic; struct iwm_softc *sc = ic->ic_softc; struct iwm_node *in; int error; IWM_DPRINTF(sc, IWM_DEBUG_STATE, "switching state %s -> %s arg=0x%x\n", ieee80211_state_name[vap->iv_state], ieee80211_state_name[nstate], arg); IEEE80211_UNLOCK(ic); IWM_LOCK(sc); if ((sc->sc_flags & IWM_FLAG_SCAN_RUNNING) && (nstate == IEEE80211_S_AUTH || nstate == IEEE80211_S_ASSOC || nstate == IEEE80211_S_RUN)) { /* Stop blinking for a scan, when authenticating. */ iwm_led_blink_stop(sc); } if (vap->iv_state == IEEE80211_S_RUN && nstate != IEEE80211_S_RUN) { iwm_mvm_led_disable(sc); /* disable beacon filtering if we're hopping out of RUN */ iwm_mvm_disable_beacon_filter(sc); if (((in = IWM_NODE(vap->iv_bss)) != NULL)) in->in_assoc = 0; } if ((vap->iv_state == IEEE80211_S_AUTH || vap->iv_state == IEEE80211_S_ASSOC || vap->iv_state == IEEE80211_S_RUN) && (nstate == IEEE80211_S_INIT || nstate == IEEE80211_S_SCAN || nstate == IEEE80211_S_AUTH)) { iwm_mvm_stop_session_protection(sc, ivp); } if ((vap->iv_state == IEEE80211_S_RUN || vap->iv_state == IEEE80211_S_ASSOC) && nstate == IEEE80211_S_INIT) { /* * In this case, iv_newstate() wants to send an 80211 frame on * the network that we are leaving. So we need to call it, * before tearing down all the firmware state. */ IWM_UNLOCK(sc); IEEE80211_LOCK(ic); ivp->iv_newstate(vap, nstate, arg); IEEE80211_UNLOCK(ic); IWM_LOCK(sc); iwm_bring_down_firmware(sc, vap); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return 0; } switch (nstate) { case IEEE80211_S_INIT: case IEEE80211_S_SCAN: break; case IEEE80211_S_AUTH: iwm_bring_down_firmware(sc, vap); if ((error = iwm_auth(vap, sc)) != 0) { device_printf(sc->sc_dev, "%s: could not move to auth state: %d\n", __func__, error); iwm_bring_down_firmware(sc, vap); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return 1; } break; case IEEE80211_S_ASSOC: /* * EBS may be disabled due to previous failures reported by FW. * Reset EBS status here assuming environment has been changed. */ sc->last_ebs_successful = TRUE; break; case IEEE80211_S_RUN: in = IWM_NODE(vap->iv_bss); /* Update the association state, now we have it all */ /* (eg associd comes in at this point */ error = iwm_mvm_update_sta(sc, in); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update STA\n", __func__); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return error; } in->in_assoc = 1; error = iwm_mvm_mac_ctxt_changed(sc, vap); if (error != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC: %d\n", __func__, error); } iwm_mvm_sf_update(sc, vap, FALSE); iwm_mvm_enable_beacon_filter(sc, ivp); iwm_mvm_power_update_mac(sc); iwm_mvm_update_quotas(sc, ivp); int rix = ieee80211_ratectl_rate(&in->in_ni, NULL, 0); iwm_setrates(sc, in, rix); if ((error = iwm_mvm_send_lq_cmd(sc, &in->in_lq, TRUE)) != 0) { device_printf(sc->sc_dev, "%s: IWM_LQ_CMD failed: %d\n", __func__, error); } iwm_mvm_led_enable(sc); break; default: break; } IWM_UNLOCK(sc); IEEE80211_LOCK(ic); return (ivp->iv_newstate(vap, nstate, arg)); } void iwm_endscan_cb(void *arg, int pending) { struct iwm_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE, "%s: scan ended\n", __func__); ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps)); } static int iwm_send_bt_init_conf(struct iwm_softc *sc) { struct iwm_bt_coex_cmd bt_cmd; bt_cmd.mode = htole32(IWM_BT_COEX_WIFI); bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET); return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd), &bt_cmd); } static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *sc) { boolean_t nvm_lar = sc->nvm_data->lar_enabled; boolean_t tlv_lar = fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT); if (iwm_lar_disable) return FALSE; /* * Enable LAR only if it is supported by the FW (TLV) && * enabled in the NVM */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) return nvm_lar && tlv_lar; else return tlv_lar; } static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *sc) { return fw_has_api(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) || fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC); } static int iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2) { struct iwm_mcc_update_cmd mcc_cmd; struct iwm_host_cmd hcmd = { .id = IWM_MCC_UPDATE_CMD, .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB), .data = { &mcc_cmd }, }; int ret; #ifdef IWM_DEBUG struct iwm_rx_packet *pkt; struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL; struct iwm_mcc_update_resp *mcc_resp; int n_channels; uint16_t mcc; #endif int resp_v2 = fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2); if (!iwm_mvm_is_lar_supported(sc)) { IWM_DPRINTF(sc, IWM_DEBUG_LAR, "%s: no LAR support\n", __func__); return 0; } memset(&mcc_cmd, 0, sizeof(mcc_cmd)); mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]); if (iwm_mvm_is_wifi_mcc_supported(sc)) mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT; else mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW; if (resp_v2) hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd); else hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1); IWM_DPRINTF(sc, IWM_DEBUG_LAR, "send MCC update to FW with '%c%c' src = %d\n", alpha2[0], alpha2[1], mcc_cmd.source_id); ret = iwm_send_cmd(sc, &hcmd); if (ret) return ret; #ifdef IWM_DEBUG pkt = hcmd.resp_pkt; /* Extract MCC response */ if (resp_v2) { mcc_resp = (void *)pkt->data; mcc = mcc_resp->mcc; n_channels = le32toh(mcc_resp->n_channels); } else { mcc_resp_v1 = (void *)pkt->data; mcc = mcc_resp_v1->mcc; n_channels = le32toh(mcc_resp_v1->n_channels); } /* W/A for a FW/NVM issue - returns 0x00 for the world domain */ if (mcc == 0) mcc = 0x3030; /* "00" - world */ IWM_DPRINTF(sc, IWM_DEBUG_LAR, "regulatory domain '%c%c' (%d channels available)\n", mcc >> 8, mcc & 0xff, n_channels); #endif iwm_free_resp(sc, &hcmd); return 0; } static void iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff) { struct iwm_host_cmd cmd = { .id = IWM_REPLY_THERMAL_MNG_BACKOFF, .len = { sizeof(uint32_t), }, .data = { &backoff, }, }; if (iwm_send_cmd(sc, &cmd) != 0) { device_printf(sc->sc_dev, "failed to change thermal tx backoff\n"); } } static int iwm_init_hw(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; int error, i, ac; sc->sf_state = IWM_SF_UNINIT; if ((error = iwm_start_hw(sc)) != 0) { printf("iwm_start_hw: failed %d\n", error); return error; } if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) { printf("iwm_run_init_mvm_ucode: failed %d\n", error); return error; } /* * should stop and start HW since that INIT * image just loaded */ iwm_stop_device(sc); sc->sc_ps_disabled = FALSE; if ((error = iwm_start_hw(sc)) != 0) { device_printf(sc->sc_dev, "could not initialize hardware\n"); return error; } /* omstart, this time with the regular firmware */ error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR); if (error) { device_printf(sc->sc_dev, "could not load firmware\n"); goto error; } error = iwm_mvm_sf_update(sc, NULL, FALSE); if (error) device_printf(sc->sc_dev, "Failed to initialize Smart Fifo\n"); if ((error = iwm_send_bt_init_conf(sc)) != 0) { device_printf(sc->sc_dev, "bt init conf failed\n"); goto error; } error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc)); if (error != 0) { device_printf(sc->sc_dev, "antenna config failed\n"); goto error; } /* Send phy db control command and then phy db calibration */ if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0) goto error; if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) { device_printf(sc->sc_dev, "phy_cfg_cmd failed\n"); goto error; } /* Add auxiliary station for scanning */ if ((error = iwm_mvm_add_aux_sta(sc)) != 0) { device_printf(sc->sc_dev, "add_aux_sta failed\n"); goto error; } for (i = 0; i < IWM_NUM_PHY_CTX; i++) { /* * The channel used here isn't relevant as it's * going to be overwritten in the other flows. * For now use the first channel we have. */ if ((error = iwm_mvm_phy_ctxt_add(sc, &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0) goto error; } /* Initialize tx backoffs to the minimum. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) iwm_mvm_tt_tx_backoff(sc, 0); if (iwm_mvm_config_ltr(sc) != 0) device_printf(sc->sc_dev, "PCIe LTR configuration failed\n"); error = iwm_mvm_power_update_device(sc); if (error) goto error; if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0) goto error; if (fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) { if ((error = iwm_mvm_config_umac_scan(sc)) != 0) goto error; } /* Enable Tx queues. */ for (ac = 0; ac < WME_NUM_AC; ac++) { error = iwm_enable_txq(sc, IWM_STATION_ID, ac, iwm_mvm_ac_to_tx_fifo[ac]); if (error) goto error; } if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) { device_printf(sc->sc_dev, "failed to disable beacon filter\n"); goto error; } return 0; error: iwm_stop_device(sc); return error; } /* Allow multicast from our BSSID. */ static int iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc) { struct ieee80211_node *ni = vap->iv_bss; struct iwm_mcast_filter_cmd *cmd; size_t size; int error; size = roundup(sizeof(*cmd), 4); cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); if (cmd == NULL) return ENOMEM; cmd->filter_own = 1; cmd->port_id = 0; cmd->count = 0; cmd->pass_all = 1; IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid); error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD, IWM_CMD_SYNC, size, cmd); free(cmd, M_DEVBUF); return (error); } /* * ifnet interfaces */ static void iwm_init(struct iwm_softc *sc) { int error; if (sc->sc_flags & IWM_FLAG_HW_INITED) { return; } sc->sc_generation++; sc->sc_flags &= ~IWM_FLAG_STOPPED; if ((error = iwm_init_hw(sc)) != 0) { printf("iwm_init_hw failed %d\n", error); iwm_stop(sc); return; } /* * Ok, firmware loaded and we are jogging */ sc->sc_flags |= IWM_FLAG_HW_INITED; } static int iwm_transmit(struct ieee80211com *ic, struct mbuf *m) { struct iwm_softc *sc; int error; sc = ic->ic_softc; IWM_LOCK(sc); if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) { IWM_UNLOCK(sc); return (ENXIO); } error = mbufq_enqueue(&sc->sc_snd, m); if (error) { IWM_UNLOCK(sc); return (error); } iwm_start(sc); IWM_UNLOCK(sc); return (0); } /* * Dequeue packets from sendq and call send. */ static void iwm_start(struct iwm_softc *sc) { struct ieee80211_node *ni; struct mbuf *m; int ac = 0; IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__); while (sc->qfullmsk == 0 && (m = mbufq_dequeue(&sc->sc_snd)) != NULL) { ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; if (iwm_tx(sc, m, ni, ac) != 0) { if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); ieee80211_free_node(ni); continue; } if (sc->sc_tx_timer == 0) { callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); } sc->sc_tx_timer = 15; } IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__); } static void iwm_stop(struct iwm_softc *sc) { sc->sc_flags &= ~IWM_FLAG_HW_INITED; sc->sc_flags |= IWM_FLAG_STOPPED; sc->sc_generation++; iwm_led_blink_stop(sc); sc->sc_tx_timer = 0; iwm_stop_device(sc); sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; } static void iwm_watchdog(void *arg) { struct iwm_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; if (sc->sc_attached == 0) return; if (sc->sc_tx_timer > 0) { if (--sc->sc_tx_timer == 0) { device_printf(sc->sc_dev, "device timeout\n"); #ifdef IWM_DEBUG iwm_nic_error(sc); #endif ieee80211_restart_all(ic); counter_u64_add(sc->sc_ic.ic_oerrors, 1); return; } callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); } } static void iwm_parent(struct ieee80211com *ic) { struct iwm_softc *sc = ic->ic_softc; int startall = 0; IWM_LOCK(sc); if (ic->ic_nrunning > 0) { if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) { iwm_init(sc); startall = 1; } } else if (sc->sc_flags & IWM_FLAG_HW_INITED) iwm_stop(sc); IWM_UNLOCK(sc); if (startall) ieee80211_start_all(ic); } /* * The interrupt side of things */ /* * error dumping routines are from iwlwifi/mvm/utils.c */ /* * Note: This structure is read from the device with IO accesses, * and the reading already does the endian conversion. As it is * read with uint32_t-sized accesses, any members with a different size * need to be ordered correctly though! */ struct iwm_error_event_table { uint32_t valid; /* (nonzero) valid, (0) log is empty */ uint32_t error_id; /* type of error */ uint32_t trm_hw_status0; /* TRM HW status */ uint32_t trm_hw_status1; /* TRM HW status */ uint32_t blink2; /* branch link */ uint32_t ilink1; /* interrupt link */ uint32_t ilink2; /* interrupt link */ uint32_t data1; /* error-specific data */ uint32_t data2; /* error-specific data */ uint32_t data3; /* error-specific data */ uint32_t bcon_time; /* beacon timer */ uint32_t tsf_low; /* network timestamp function timer */ uint32_t tsf_hi; /* network timestamp function timer */ uint32_t gp1; /* GP1 timer register */ uint32_t gp2; /* GP2 timer register */ uint32_t fw_rev_type; /* firmware revision type */ uint32_t major; /* uCode version major */ uint32_t minor; /* uCode version minor */ uint32_t hw_ver; /* HW Silicon version */ uint32_t brd_ver; /* HW board version */ uint32_t log_pc; /* log program counter */ uint32_t frame_ptr; /* frame pointer */ uint32_t stack_ptr; /* stack pointer */ uint32_t hcmd; /* last host command header */ uint32_t isr0; /* isr status register LMPM_NIC_ISR0: * rxtx_flag */ uint32_t isr1; /* isr status register LMPM_NIC_ISR1: * host_flag */ uint32_t isr2; /* isr status register LMPM_NIC_ISR2: * enc_flag */ uint32_t isr3; /* isr status register LMPM_NIC_ISR3: * time_flag */ uint32_t isr4; /* isr status register LMPM_NIC_ISR4: * wico interrupt */ uint32_t last_cmd_id; /* last HCMD id handled by the firmware */ uint32_t wait_event; /* wait event() caller address */ uint32_t l2p_control; /* L2pControlField */ uint32_t l2p_duration; /* L2pDurationField */ uint32_t l2p_mhvalid; /* L2pMhValidBits */ uint32_t l2p_addr_match; /* L2pAddrMatchStat */ uint32_t lmpm_pmg_sel; /* indicate which clocks are turned on * (LMPM_PMG_SEL) */ uint32_t u_timestamp; /* indicate when the date and time of the * compilation */ uint32_t flow_handler; /* FH read/write pointers, RX credit */ } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */; /* * UMAC error struct - relevant starting from family 8000 chip. * Note: This structure is read from the device with IO accesses, * and the reading already does the endian conversion. As it is * read with u32-sized accesses, any members with a different size * need to be ordered correctly though! */ struct iwm_umac_error_event_table { uint32_t valid; /* (nonzero) valid, (0) log is empty */ uint32_t error_id; /* type of error */ uint32_t blink1; /* branch link */ uint32_t blink2; /* branch link */ uint32_t ilink1; /* interrupt link */ uint32_t ilink2; /* interrupt link */ uint32_t data1; /* error-specific data */ uint32_t data2; /* error-specific data */ uint32_t data3; /* error-specific data */ uint32_t umac_major; uint32_t umac_minor; uint32_t frame_pointer; /* core register 27*/ uint32_t stack_pointer; /* core register 28 */ uint32_t cmd_header; /* latest host cmd sent to UMAC */ uint32_t nic_isr_pref; /* ISR status register */ } __packed; #define ERROR_START_OFFSET (1 * sizeof(uint32_t)) #define ERROR_ELEM_SIZE (7 * sizeof(uint32_t)) #ifdef IWM_DEBUG struct { const char *name; uint8_t num; } advanced_lookup[] = { { "NMI_INTERRUPT_WDG", 0x34 }, { "SYSASSERT", 0x35 }, { "UCODE_VERSION_MISMATCH", 0x37 }, { "BAD_COMMAND", 0x38 }, { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C }, { "FATAL_ERROR", 0x3D }, { "NMI_TRM_HW_ERR", 0x46 }, { "NMI_INTERRUPT_TRM", 0x4C }, { "NMI_INTERRUPT_BREAK_POINT", 0x54 }, { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C }, { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 }, { "NMI_INTERRUPT_HOST", 0x66 }, { "NMI_INTERRUPT_ACTION_PT", 0x7C }, { "NMI_INTERRUPT_UNKNOWN", 0x84 }, { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 }, { "ADVANCED_SYSASSERT", 0 }, }; static const char * iwm_desc_lookup(uint32_t num) { int i; for (i = 0; i < nitems(advanced_lookup) - 1; i++) if (advanced_lookup[i].num == num) return advanced_lookup[i].name; /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */ return advanced_lookup[i].name; } static void iwm_nic_umac_error(struct iwm_softc *sc) { struct iwm_umac_error_event_table table; uint32_t base; base = sc->umac_error_event_table; if (base < 0x800000) { device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n", base); return; } if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { device_printf(sc->sc_dev, "reading errlog failed\n"); return; } if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n"); device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", sc->sc_flags, table.valid); } device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id, iwm_desc_lookup(table.error_id)); device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1); device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2); device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n", table.ilink1); device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n", table.ilink2); device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1); device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2); device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3); device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major); device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor); device_printf(sc->sc_dev, "0x%08X | frame pointer\n", table.frame_pointer); device_printf(sc->sc_dev, "0x%08X | stack pointer\n", table.stack_pointer); device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header); device_printf(sc->sc_dev, "0x%08X | isr status reg\n", table.nic_isr_pref); } /* * Support for dumping the error log seemed like a good idea ... * but it's mostly hex junk and the only sensible thing is the * hw/ucode revision (which we know anyway). Since it's here, * I'll just leave it in, just in case e.g. the Intel guys want to * help us decipher some "ADVANCED_SYSASSERT" later. */ static void iwm_nic_error(struct iwm_softc *sc) { struct iwm_error_event_table table; uint32_t base; device_printf(sc->sc_dev, "dumping device error log\n"); base = sc->error_event_table[0]; if (base < 0x800000) { device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n", base); return; } if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { device_printf(sc->sc_dev, "reading errlog failed\n"); return; } if (!table.valid) { device_printf(sc->sc_dev, "errlog not found, skipping\n"); return; } if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { device_printf(sc->sc_dev, "Start Error Log Dump:\n"); device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", sc->sc_flags, table.valid); } device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id, iwm_desc_lookup(table.error_id)); device_printf(sc->sc_dev, "%08X | trm_hw_status0\n", table.trm_hw_status0); device_printf(sc->sc_dev, "%08X | trm_hw_status1\n", table.trm_hw_status1); device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2); device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1); device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2); device_printf(sc->sc_dev, "%08X | data1\n", table.data1); device_printf(sc->sc_dev, "%08X | data2\n", table.data2); device_printf(sc->sc_dev, "%08X | data3\n", table.data3); device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time); device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low); device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi); device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1); device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2); device_printf(sc->sc_dev, "%08X | uCode revision type\n", table.fw_rev_type); device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major); device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor); device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver); device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver); device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd); device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0); device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1); device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2); device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3); device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4); device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id); device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event); device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control); device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration); device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid); device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match); device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel); device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp); device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler); if (sc->umac_error_event_table) iwm_nic_umac_error(sc); } #endif static void iwm_handle_rxb(struct iwm_softc *sc, struct mbuf *m) { struct ieee80211com *ic = &sc->sc_ic; struct iwm_cmd_response *cresp; struct mbuf *m1; uint32_t offset = 0; uint32_t maxoff = IWM_RBUF_SIZE; uint32_t nextoff; boolean_t stolen = FALSE; #define HAVEROOM(a) \ ((a) + sizeof(uint32_t) + sizeof(struct iwm_cmd_header) < maxoff) while (HAVEROOM(offset)) { struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset); int qid, idx, code, len; qid = pkt->hdr.qid; idx = pkt->hdr.idx; code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code); /* * randomly get these from the firmware, no idea why. * they at least seem harmless, so just ignore them for now */ if ((pkt->hdr.code == 0 && (qid & ~0x80) == 0 && idx == 0) || pkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID)) { break; } IWM_DPRINTF(sc, IWM_DEBUG_INTR, "rx packet qid=%d idx=%d type=%x\n", qid & ~0x80, pkt->hdr.idx, code); len = iwm_rx_packet_len(pkt); len += sizeof(uint32_t); /* account for status word */ nextoff = offset + roundup2(len, IWM_FH_RSCSR_FRAME_ALIGN); iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt); switch (code) { case IWM_REPLY_RX_PHY_CMD: iwm_mvm_rx_rx_phy_cmd(sc, pkt); break; case IWM_REPLY_RX_MPDU_CMD: { /* * If this is the last frame in the RX buffer, we * can directly feed the mbuf to the sharks here. */ struct iwm_rx_packet *nextpkt = mtodoff(m, struct iwm_rx_packet *, nextoff); if (!HAVEROOM(nextoff) || (nextpkt->hdr.code == 0 && (nextpkt->hdr.qid & ~0x80) == 0 && nextpkt->hdr.idx == 0) || (nextpkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID))) { if (iwm_mvm_rx_rx_mpdu(sc, m, offset, stolen)) { stolen = FALSE; /* Make sure we abort the loop */ nextoff = maxoff; } break; } /* * Use m_copym instead of m_split, because that * makes it easier to keep a valid rx buffer in * the ring, when iwm_mvm_rx_rx_mpdu() fails. * * We need to start m_copym() at offset 0, to get the * M_PKTHDR flag preserved. */ m1 = m_copym(m, 0, M_COPYALL, M_NOWAIT); if (m1) { if (iwm_mvm_rx_rx_mpdu(sc, m1, offset, stolen)) stolen = TRUE; else m_freem(m1); } break; } case IWM_TX_CMD: iwm_mvm_rx_tx_cmd(sc, pkt); break; case IWM_MISSED_BEACONS_NOTIFICATION: { struct iwm_missed_beacons_notif *resp; int missed; /* XXX look at mac_id to determine interface ID */ struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); resp = (void *)pkt->data; missed = le32toh(resp->consec_missed_beacons); IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE, "%s: MISSED_BEACON: mac_id=%d, " "consec_since_last_rx=%d, consec=%d, num_expect=%d " "num_rx=%d\n", __func__, le32toh(resp->mac_id), le32toh(resp->consec_missed_beacons_since_last_rx), le32toh(resp->consec_missed_beacons), le32toh(resp->num_expected_beacons), le32toh(resp->num_recvd_beacons)); /* Be paranoid */ if (vap == NULL) break; /* XXX no net80211 locking? */ if (vap->iv_state == IEEE80211_S_RUN && (ic->ic_flags & IEEE80211_F_SCAN) == 0) { if (missed > vap->iv_bmissthreshold) { /* XXX bad locking; turn into task */ IWM_UNLOCK(sc); ieee80211_beacon_miss(ic); IWM_LOCK(sc); } } break; } case IWM_MFUART_LOAD_NOTIFICATION: break; case IWM_MVM_ALIVE: break; case IWM_CALIB_RES_NOTIF_PHY_DB: break; case IWM_STATISTICS_NOTIFICATION: iwm_mvm_handle_rx_statistics(sc, pkt); break; case IWM_NVM_ACCESS_CMD: case IWM_MCC_UPDATE_CMD: if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) { memcpy(sc->sc_cmd_resp, pkt, sizeof(sc->sc_cmd_resp)); } break; case IWM_MCC_CHUB_UPDATE_CMD: { struct iwm_mcc_chub_notif *notif; notif = (void *)pkt->data; sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8; sc->sc_fw_mcc[1] = notif->mcc & 0xff; sc->sc_fw_mcc[2] = '\0'; IWM_DPRINTF(sc, IWM_DEBUG_LAR, "fw source %d sent CC '%s'\n", notif->source_id, sc->sc_fw_mcc); break; } case IWM_DTS_MEASUREMENT_NOTIFICATION: case IWM_WIDE_ID(IWM_PHY_OPS_GROUP, IWM_DTS_MEASUREMENT_NOTIF_WIDE): { struct iwm_dts_measurement_notif_v1 *notif; if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) { device_printf(sc->sc_dev, "Invalid DTS_MEASUREMENT_NOTIFICATION\n"); break; } notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_TEMP, "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n", notif->temp); break; } case IWM_PHY_CONFIGURATION_CMD: case IWM_TX_ANT_CONFIGURATION_CMD: case IWM_ADD_STA: case IWM_MAC_CONTEXT_CMD: case IWM_REPLY_SF_CFG_CMD: case IWM_POWER_TABLE_CMD: case IWM_LTR_CONFIG: case IWM_PHY_CONTEXT_CMD: case IWM_BINDING_CONTEXT_CMD: case IWM_TIME_EVENT_CMD: case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD): case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC): case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_ABORT_UMAC): case IWM_SCAN_OFFLOAD_REQUEST_CMD: case IWM_SCAN_OFFLOAD_ABORT_CMD: case IWM_REPLY_BEACON_FILTERING_CMD: case IWM_MAC_PM_POWER_TABLE: case IWM_TIME_QUOTA_CMD: case IWM_REMOVE_STA: case IWM_TXPATH_FLUSH: case IWM_LQ_CMD: case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_FW_PAGING_BLOCK_CMD): case IWM_BT_CONFIG: case IWM_REPLY_THERMAL_MNG_BACKOFF: cresp = (void *)pkt->data; if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) { memcpy(sc->sc_cmd_resp, pkt, sizeof(*pkt)+sizeof(*cresp)); } break; /* ignore */ case IWM_PHY_DB_CMD: break; case IWM_INIT_COMPLETE_NOTIF: break; case IWM_SCAN_OFFLOAD_COMPLETE: iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; ieee80211_runtask(ic, &sc->sc_es_task); } break; case IWM_SCAN_ITERATION_COMPLETE: { struct iwm_lmac_scan_complete_notif *notif; notif = (void *)pkt->data; break; } case IWM_SCAN_COMPLETE_UMAC: iwm_mvm_rx_umac_scan_complete_notif(sc, pkt); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; ieee80211_runtask(ic, &sc->sc_es_task); } break; case IWM_SCAN_ITERATION_COMPLETE_UMAC: { struct iwm_umac_scan_iter_complete_notif *notif; notif = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration " "complete, status=0x%x, %d channels scanned\n", notif->status, notif->scanned_channels); break; } case IWM_REPLY_ERROR: { struct iwm_error_resp *resp; resp = (void *)pkt->data; device_printf(sc->sc_dev, "firmware error 0x%x, cmd 0x%x\n", le32toh(resp->error_type), resp->cmd_id); break; } case IWM_TIME_EVENT_NOTIFICATION: iwm_mvm_rx_time_event_notif(sc, pkt); break; /* * Firmware versions 21 and 22 generate some DEBUG_LOG_MSG * messages. Just ignore them for now. */ case IWM_DEBUG_LOG_MSG: break; case IWM_MCAST_FILTER_CMD: break; case IWM_SCD_QUEUE_CFG: { struct iwm_scd_txq_cfg_rsp *rsp; rsp = (void *)pkt->data; IWM_DPRINTF(sc, IWM_DEBUG_CMD, "queue cfg token=0x%x sta_id=%d " "tid=%d scd_queue=%d\n", rsp->token, rsp->sta_id, rsp->tid, rsp->scd_queue); break; } default: device_printf(sc->sc_dev, "frame %d/%d %x UNHANDLED (this should " "not happen)\n", qid & ~0x80, idx, pkt->len_n_flags); break; } /* * Why test bit 0x80? The Linux driver: * * There is one exception: uCode sets bit 15 when it * originates the response/notification, i.e. when the * response/notification is not a direct response to a * command sent by the driver. For example, uCode issues * IWM_REPLY_RX when it sends a received frame to the driver; * it is not a direct response to any driver command. * * Ok, so since when is 7 == 15? Well, the Linux driver * uses a slightly different format for pkt->hdr, and "qid" * is actually the upper byte of a two-byte field. */ if (!(qid & (1 << 7))) iwm_cmd_done(sc, pkt); offset = nextoff; } if (stolen) m_freem(m); #undef HAVEROOM } /* * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt. * Basic structure from if_iwn */ static void iwm_notif_intr(struct iwm_softc *sc) { uint16_t hw; bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map, BUS_DMASYNC_POSTREAD); hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff; /* * Process responses */ while (sc->rxq.cur != hw) { struct iwm_rx_ring *ring = &sc->rxq; struct iwm_rx_data *data = &ring->data[ring->cur]; bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); IWM_DPRINTF(sc, IWM_DEBUG_INTR, "%s: hw = %d cur = %d\n", __func__, hw, ring->cur); iwm_handle_rxb(sc, data->m); ring->cur = (ring->cur + 1) % IWM_RX_RING_COUNT; } /* * Tell the firmware that it can reuse the ring entries that * we have just processed. * Seems like the hardware gets upset unless we align * the write by 8?? */ hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1; IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, rounddown2(hw, 8)); } static void iwm_intr(void *arg) { struct iwm_softc *sc = arg; int handled = 0; int r1, r2, rv = 0; int isperiodic = 0; IWM_LOCK(sc); IWM_WRITE(sc, IWM_CSR_INT_MASK, 0); if (sc->sc_flags & IWM_FLAG_USE_ICT) { uint32_t *ict = sc->ict_dma.vaddr; int tmp; tmp = htole32(ict[sc->ict_cur]); if (!tmp) goto out_ena; /* * ok, there was something. keep plowing until we have all. */ r1 = r2 = 0; while (tmp) { r1 |= tmp; ict[sc->ict_cur] = 0; sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT; tmp = htole32(ict[sc->ict_cur]); } /* this is where the fun begins. don't ask */ if (r1 == 0xffffffff) r1 = 0; /* i am not expected to understand this */ if (r1 & 0xc0000) r1 |= 0x8000; r1 = (0xff & r1) | ((0xff00 & r1) << 16); } else { r1 = IWM_READ(sc, IWM_CSR_INT); /* "hardware gone" (where, fishing?) */ if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0) goto out; r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS); } if (r1 == 0 && r2 == 0) { goto out_ena; } IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask); /* Safely ignore these bits for debug checks below */ r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD); if (r1 & IWM_CSR_INT_BIT_SW_ERR) { int i; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); #ifdef IWM_DEBUG iwm_nic_error(sc); #endif /* Dump driver status (TX and RX rings) while we're here. */ device_printf(sc->sc_dev, "driver status:\n"); for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) { struct iwm_tx_ring *ring = &sc->txq[i]; device_printf(sc->sc_dev, " tx ring %2d: qid=%-2d cur=%-3d " "queued=%-3d\n", i, ring->qid, ring->cur, ring->queued); } device_printf(sc->sc_dev, " rx ring: cur=%d\n", sc->rxq.cur); device_printf(sc->sc_dev, " 802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state); /* Reset our firmware state tracking. */ sc->sc_firmware_state = 0; /* Don't stop the device; just do a VAP restart */ IWM_UNLOCK(sc); if (vap == NULL) { printf("%s: null vap\n", __func__); return; } device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; " "restarting\n", __func__, vap->iv_state); ieee80211_restart_all(ic); return; } if (r1 & IWM_CSR_INT_BIT_HW_ERR) { handled |= IWM_CSR_INT_BIT_HW_ERR; device_printf(sc->sc_dev, "hardware error, stopping device\n"); iwm_stop(sc); rv = 1; goto out; } /* firmware chunk loaded */ if (r1 & IWM_CSR_INT_BIT_FH_TX) { IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK); handled |= IWM_CSR_INT_BIT_FH_TX; sc->sc_fw_chunk_done = 1; wakeup(&sc->sc_fw); } if (r1 & IWM_CSR_INT_BIT_RF_KILL) { handled |= IWM_CSR_INT_BIT_RF_KILL; if (iwm_check_rfkill(sc)) { device_printf(sc->sc_dev, "%s: rfkill switch, disabling interface\n", __func__); iwm_stop(sc); } } /* * The Linux driver uses periodic interrupts to avoid races. * We cargo-cult like it's going out of fashion. */ if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) { handled |= IWM_CSR_INT_BIT_RX_PERIODIC; IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC); if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0) IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS); isperiodic = 1; } if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) { handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX); IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK); iwm_notif_intr(sc); /* enable periodic interrupt, see above */ if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic) IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_ENA); } if (__predict_false(r1 & ~handled)) IWM_DPRINTF(sc, IWM_DEBUG_INTR, "%s: unhandled interrupts: %x\n", __func__, r1); rv = 1; out_ena: iwm_restore_interrupts(sc); out: IWM_UNLOCK(sc); return; } /* * Autoconf glue-sniffing */ #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_WL_3160_1 0x08b3 #define PCI_PRODUCT_INTEL_WL_3160_2 0x08b4 #define PCI_PRODUCT_INTEL_WL_3165_1 0x3165 #define PCI_PRODUCT_INTEL_WL_3165_2 0x3166 #define PCI_PRODUCT_INTEL_WL_3168_1 0x24fb #define PCI_PRODUCT_INTEL_WL_7260_1 0x08b1 #define PCI_PRODUCT_INTEL_WL_7260_2 0x08b2 #define PCI_PRODUCT_INTEL_WL_7265_1 0x095a #define PCI_PRODUCT_INTEL_WL_7265_2 0x095b #define PCI_PRODUCT_INTEL_WL_8260_1 0x24f3 #define PCI_PRODUCT_INTEL_WL_8260_2 0x24f4 #define PCI_PRODUCT_INTEL_WL_8265_1 0x24fd static const struct iwm_devices { uint16_t device; const struct iwm_cfg *cfg; } iwm_devices[] = { { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg }, { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg }, { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg }, { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg }, { PCI_PRODUCT_INTEL_WL_3168_1, &iwm3168_cfg }, { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg }, { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg }, { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg }, { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg }, { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg }, { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg }, { PCI_PRODUCT_INTEL_WL_8265_1, &iwm8265_cfg }, }; static int iwm_probe(device_t dev) { int i; for (i = 0; i < nitems(iwm_devices); i++) { if (pci_get_vendor(dev) == PCI_VENDOR_INTEL && pci_get_device(dev) == iwm_devices[i].device) { device_set_desc(dev, iwm_devices[i].cfg->name); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int iwm_dev_check(device_t dev) { struct iwm_softc *sc; uint16_t devid; int i; sc = device_get_softc(dev); devid = pci_get_device(dev); for (i = 0; i < nitems(iwm_devices); i++) { if (iwm_devices[i].device == devid) { sc->cfg = iwm_devices[i].cfg; return (0); } } device_printf(dev, "unknown adapter type\n"); return ENXIO; } /* PCI registers */ #define PCI_CFG_RETRY_TIMEOUT 0x041 static int iwm_pci_attach(device_t dev) { struct iwm_softc *sc; int count, error, rid; uint16_t reg; sc = device_get_softc(dev); /* We disable the RETRY_TIMEOUT register (0x41) to keep * PCI Tx retries from interfering with C3 CPU state */ pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1); /* Enable bus-mastering and hardware bug workaround. */ pci_enable_busmaster(dev); reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg)); /* if !MSI */ if (reg & PCIM_STATUS_INTxSTATE) { reg &= ~PCIM_STATUS_INTxSTATE; } pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg)); rid = PCIR_BAR(0); sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(sc->sc_dev, "can't map mem space\n"); return (ENXIO); } sc->sc_st = rman_get_bustag(sc->sc_mem); sc->sc_sh = rman_get_bushandle(sc->sc_mem); /* Install interrupt handler. */ count = 1; rid = 0; if (pci_alloc_msi(dev, &count) == 0) rid = 1; sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | (rid != 0 ? 0 : RF_SHAREABLE)); if (sc->sc_irq == NULL) { device_printf(dev, "can't map interrupt\n"); return (ENXIO); } error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, iwm_intr, sc, &sc->sc_ih); if (sc->sc_ih == NULL) { device_printf(dev, "can't establish interrupt"); return (ENXIO); } sc->sc_dmat = bus_get_dma_tag(sc->sc_dev); return (0); } static void iwm_pci_detach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); if (sc->sc_irq != NULL) { bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_irq), sc->sc_irq); pci_release_msi(dev); } if (sc->sc_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem), sc->sc_mem); } static int iwm_attach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); struct ieee80211com *ic = &sc->sc_ic; int error; int txq_i, i; sc->sc_dev = dev; sc->sc_attached = 1; IWM_LOCK_INIT(sc); mbufq_init(&sc->sc_snd, ifqmaxlen); callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0); callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0); TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc); sc->sc_notif_wait = iwm_notification_wait_init(sc); if (sc->sc_notif_wait == NULL) { device_printf(dev, "failed to init notification wait struct\n"); goto fail; } sc->sf_state = IWM_SF_UNINIT; /* Init phy db */ sc->sc_phy_db = iwm_phy_db_init(sc); if (!sc->sc_phy_db) { device_printf(dev, "Cannot init phy_db\n"); goto fail; } /* Set EBS as successful as long as not stated otherwise by the FW. */ sc->last_ebs_successful = TRUE; /* PCI attach */ error = iwm_pci_attach(dev); if (error != 0) goto fail; sc->sc_wantresp = -1; /* Match device id */ error = iwm_dev_check(dev); if (error != 0) goto fail; sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV); /* * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have * changed, and now the revision step also includes bit 0-1 (no more * "dash" value). To keep hw_rev backwards compatible - we'll store it * in the old format. */ if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) { int ret; uint32_t hw_step; sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) | (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2); if (iwm_prepare_card_hw(sc) != 0) { device_printf(dev, "could not initialize hardware\n"); goto fail; } /* * In order to recognize C step the driver should read the * chip version id located at the AUX bus MISC address. */ IWM_SETBITS(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE); DELAY(2); ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000); if (!ret) { device_printf(sc->sc_dev, "Failed to wake up the nic\n"); goto fail; } if (iwm_nic_lock(sc)) { hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG); hw_step |= IWM_ENABLE_WFPM; iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step); hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG); hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF; if (hw_step == 0x3) sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) | (IWM_SILICON_C_STEP << 2); iwm_nic_unlock(sc); } else { device_printf(sc->sc_dev, "Failed to lock the nic\n"); goto fail; } } /* special-case 7265D, it has the same PCI IDs. */ if (sc->cfg == &iwm7265_cfg && (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) { sc->cfg = &iwm7265d_cfg; } /* Allocate DMA memory for firmware transfers. */ if ((error = iwm_alloc_fwmem(sc)) != 0) { device_printf(dev, "could not allocate memory for firmware\n"); goto fail; } /* Allocate "Keep Warm" page. */ if ((error = iwm_alloc_kw(sc)) != 0) { device_printf(dev, "could not allocate keep warm page\n"); goto fail; } /* We use ICT interrupts */ if ((error = iwm_alloc_ict(sc)) != 0) { device_printf(dev, "could not allocate ICT table\n"); goto fail; } /* Allocate TX scheduler "rings". */ if ((error = iwm_alloc_sched(sc)) != 0) { device_printf(dev, "could not allocate TX scheduler rings\n"); goto fail; } /* Allocate TX rings */ for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) { if ((error = iwm_alloc_tx_ring(sc, &sc->txq[txq_i], txq_i)) != 0) { device_printf(dev, "could not allocate TX ring %d\n", txq_i); goto fail; } } /* Allocate RX ring. */ if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) { device_printf(dev, "could not allocate RX ring\n"); goto fail; } /* Clear pending interrupts. */ IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff); ic->ic_softc = sc; ic->ic_name = device_get_nameunit(sc->sc_dev); ic->ic_phytype = IEEE80211_T_OFDM; /* not only, but not used */ ic->ic_opmode = IEEE80211_M_STA; /* default to BSS mode */ /* Set device capabilities. */ ic->ic_caps = IEEE80211_C_STA | IEEE80211_C_WPA | /* WPA/RSN */ IEEE80211_C_WME | IEEE80211_C_PMGT | IEEE80211_C_SHSLOT | /* short slot time supported */ IEEE80211_C_SHPREAMBLE /* short preamble supported */ // IEEE80211_C_BGSCAN /* capable of bg scanning */ ; /* Advertise full-offload scanning */ ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD; for (i = 0; i < nitems(sc->sc_phyctxt); i++) { sc->sc_phyctxt[i].id = i; sc->sc_phyctxt[i].color = 0; sc->sc_phyctxt[i].ref = 0; sc->sc_phyctxt[i].channel = NULL; } /* Default noise floor */ sc->sc_noise = -96; /* Max RSSI */ sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM; #ifdef IWM_DEBUG SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLFLAG_RW, &sc->sc_debug, 0, "control debugging"); #endif error = iwm_read_firmware(sc); if (error) { goto fail; } else if (sc->sc_fw.fw_fp == NULL) { /* * XXX Add a solution for properly deferring firmware load * during bootup. */ goto fail; } else { sc->sc_preinit_hook.ich_func = iwm_preinit; sc->sc_preinit_hook.ich_arg = sc; if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) { device_printf(dev, "config_intrhook_establish failed\n"); goto fail; } } IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "<-%s\n", __func__); return 0; /* Free allocated memory if something failed during attachment. */ fail: iwm_detach_local(sc, 0); return ENXIO; } static int iwm_is_valid_ether_addr(uint8_t *addr) { char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr)) return (FALSE); return (TRUE); } static int iwm_wme_update(struct ieee80211com *ic) { #define IWM_EXP2(x) ((1 << (x)) - 1) /* CWmin = 2^ECWmin - 1 */ struct iwm_softc *sc = ic->ic_softc; struct chanAccParams chp; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_vap *ivp = IWM_VAP(vap); struct iwm_node *in; struct wmeParams tmp[WME_NUM_AC]; int aci, error; if (vap == NULL) return (0); ieee80211_wme_ic_getparams(ic, &chp); IEEE80211_LOCK(ic); for (aci = 0; aci < WME_NUM_AC; aci++) tmp[aci] = chp.cap_wmeParams[aci]; IEEE80211_UNLOCK(ic); IWM_LOCK(sc); for (aci = 0; aci < WME_NUM_AC; aci++) { const struct wmeParams *ac = &tmp[aci]; ivp->queue_params[aci].aifsn = ac->wmep_aifsn; ivp->queue_params[aci].cw_min = IWM_EXP2(ac->wmep_logcwmin); ivp->queue_params[aci].cw_max = IWM_EXP2(ac->wmep_logcwmax); ivp->queue_params[aci].edca_txop = IEEE80211_TXOP_TO_US(ac->wmep_txopLimit); } ivp->have_wme = TRUE; if (ivp->is_uploaded && vap->iv_bss != NULL) { in = IWM_NODE(vap->iv_bss); if (in->in_assoc) { if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: failed to update MAC\n", __func__); } } } IWM_UNLOCK(sc); return (0); #undef IWM_EXP2 } static void iwm_preinit(void *arg) { struct iwm_softc *sc = arg; device_t dev = sc->sc_dev; struct ieee80211com *ic = &sc->sc_ic; int error; IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s\n", __func__); IWM_LOCK(sc); if ((error = iwm_start_hw(sc)) != 0) { device_printf(dev, "could not initialize hardware\n"); IWM_UNLOCK(sc); goto fail; } error = iwm_run_init_mvm_ucode(sc, 1); iwm_stop_device(sc); if (error) { IWM_UNLOCK(sc); goto fail; } device_printf(dev, "hw rev 0x%x, fw ver %s, address %s\n", sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK, sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr)); /* not all hardware can do 5GHz band */ if (!sc->nvm_data->sku_cap_band_52GHz_enable) memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0, sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A])); IWM_UNLOCK(sc); iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels); /* * At this point we've committed - if we fail to do setup, * we now also have to tear down the net80211 state. */ ieee80211_ifattach(ic); ic->ic_vap_create = iwm_vap_create; ic->ic_vap_delete = iwm_vap_delete; ic->ic_raw_xmit = iwm_raw_xmit; ic->ic_node_alloc = iwm_node_alloc; ic->ic_scan_start = iwm_scan_start; ic->ic_scan_end = iwm_scan_end; ic->ic_update_mcast = iwm_update_mcast; ic->ic_getradiocaps = iwm_init_channel_map; ic->ic_set_channel = iwm_set_channel; ic->ic_scan_curchan = iwm_scan_curchan; ic->ic_scan_mindwell = iwm_scan_mindwell; ic->ic_wme.wme_update = iwm_wme_update; ic->ic_parent = iwm_parent; ic->ic_transmit = iwm_transmit; iwm_radiotap_attach(sc); if (bootverbose) ieee80211_announce(ic); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "<-%s\n", __func__); config_intrhook_disestablish(&sc->sc_preinit_hook); return; fail: config_intrhook_disestablish(&sc->sc_preinit_hook); iwm_detach_local(sc, 0); } /* * Attach the interface to 802.11 radiotap. */ static void iwm_radiotap_attach(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s begin\n", __func__); ieee80211_radiotap_attach(ic, &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap), IWM_TX_RADIOTAP_PRESENT, &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap), IWM_RX_RADIOTAP_PRESENT); IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE, "->%s end\n", __func__); } static struct ieee80211vap * iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN], const uint8_t mac[IEEE80211_ADDR_LEN]) { struct iwm_vap *ivp; struct ieee80211vap *vap; if (!TAILQ_EMPTY(&ic->ic_vaps)) /* only one at a time */ return NULL; ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO); vap = &ivp->iv_vap; ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid); vap->iv_bmissthreshold = 10; /* override default */ /* Override with driver methods. */ ivp->iv_newstate = vap->iv_newstate; vap->iv_newstate = iwm_newstate; ivp->id = IWM_DEFAULT_MACID; ivp->color = IWM_DEFAULT_COLOR; ivp->have_wme = FALSE; ivp->ps_disabled = FALSE; ieee80211_ratectl_init(vap); /* Complete setup. */ ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status, mac); ic->ic_opmode = opmode; return vap; } static void iwm_vap_delete(struct ieee80211vap *vap) { struct iwm_vap *ivp = IWM_VAP(vap); ieee80211_ratectl_deinit(vap); ieee80211_vap_detach(vap); free(ivp, M_80211_VAP); } static void iwm_xmit_queue_drain(struct iwm_softc *sc) { struct mbuf *m; struct ieee80211_node *ni; while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) { ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; ieee80211_free_node(ni); m_freem(m); } } static void iwm_scan_start(struct ieee80211com *ic) { struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_softc *sc = ic->ic_softc; int error; IWM_LOCK(sc); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { /* This should not be possible */ device_printf(sc->sc_dev, "%s: Previous scan not completed yet\n", __func__); } if (fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) error = iwm_mvm_umac_scan(sc); else error = iwm_mvm_lmac_scan(sc); if (error != 0) { device_printf(sc->sc_dev, "could not initiate scan\n"); IWM_UNLOCK(sc); ieee80211_cancel_scan(vap); } else { sc->sc_flags |= IWM_FLAG_SCAN_RUNNING; iwm_led_blink_start(sc); IWM_UNLOCK(sc); } } static void iwm_scan_end(struct ieee80211com *ic) { struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwm_softc *sc = ic->ic_softc; IWM_LOCK(sc); iwm_led_blink_stop(sc); if (vap->iv_state == IEEE80211_S_RUN) iwm_mvm_led_enable(sc); if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) { /* * Removing IWM_FLAG_SCAN_RUNNING now, is fine because * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq * taskqueue. */ sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING; iwm_mvm_scan_stop_wait(sc); } IWM_UNLOCK(sc); /* * Make sure we don't race, if sc_es_task is still enqueued here. * This is to make sure that it won't call ieee80211_scan_done * when we have already started the next scan. */ taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL); } static void iwm_update_mcast(struct ieee80211com *ic) { } static void iwm_set_channel(struct ieee80211com *ic) { } static void iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell) { } static void iwm_scan_mindwell(struct ieee80211_scan_state *ss) { return; } void iwm_init_task(void *arg1) { struct iwm_softc *sc = arg1; IWM_LOCK(sc); while (sc->sc_flags & IWM_FLAG_BUSY) msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0); sc->sc_flags |= IWM_FLAG_BUSY; iwm_stop(sc); if (sc->sc_ic.ic_nrunning > 0) iwm_init(sc); sc->sc_flags &= ~IWM_FLAG_BUSY; wakeup(&sc->sc_flags); IWM_UNLOCK(sc); } static int iwm_resume(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); int do_reinit = 0; /* * We disable the RETRY_TIMEOUT register (0x41) to keep * PCI Tx retries from interfering with C3 CPU state. */ pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1); if (!sc->sc_attached) return 0; iwm_init_task(device_get_softc(dev)); IWM_LOCK(sc); if (sc->sc_flags & IWM_FLAG_SCANNING) { sc->sc_flags &= ~IWM_FLAG_SCANNING; do_reinit = 1; } IWM_UNLOCK(sc); if (do_reinit) ieee80211_resume_all(&sc->sc_ic); return 0; } static int iwm_suspend(device_t dev) { int do_stop = 0; struct iwm_softc *sc = device_get_softc(dev); do_stop = !! (sc->sc_ic.ic_nrunning > 0); if (!sc->sc_attached) return (0); ieee80211_suspend_all(&sc->sc_ic); if (do_stop) { IWM_LOCK(sc); iwm_stop(sc); sc->sc_flags |= IWM_FLAG_SCANNING; IWM_UNLOCK(sc); } return (0); } static int iwm_detach_local(struct iwm_softc *sc, int do_net80211) { struct iwm_fw_info *fw = &sc->sc_fw; device_t dev = sc->sc_dev; int i; if (!sc->sc_attached) return 0; sc->sc_attached = 0; if (do_net80211) ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task); callout_drain(&sc->sc_led_blink_to); callout_drain(&sc->sc_watchdog_to); iwm_stop_device(sc); if (do_net80211) { IWM_LOCK(sc); iwm_xmit_queue_drain(sc); IWM_UNLOCK(sc); ieee80211_ifdetach(&sc->sc_ic); } iwm_phy_db_free(sc->sc_phy_db); sc->sc_phy_db = NULL; iwm_free_nvm_data(sc->nvm_data); /* Free descriptor rings */ iwm_free_rx_ring(sc, &sc->rxq); for (i = 0; i < nitems(sc->txq); i++) iwm_free_tx_ring(sc, &sc->txq[i]); /* Free firmware */ if (fw->fw_fp != NULL) iwm_fw_info_free(fw); /* Free scheduler */ iwm_dma_contig_free(&sc->sched_dma); iwm_dma_contig_free(&sc->ict_dma); iwm_dma_contig_free(&sc->kw_dma); iwm_dma_contig_free(&sc->fw_dma); iwm_free_fw_paging(sc); /* Finished with the hardware - detach things */ iwm_pci_detach(dev); if (sc->sc_notif_wait != NULL) { iwm_notification_wait_free(sc->sc_notif_wait); sc->sc_notif_wait = NULL; } IWM_LOCK_DESTROY(sc); return (0); } static int iwm_detach(device_t dev) { struct iwm_softc *sc = device_get_softc(dev); return (iwm_detach_local(sc, 1)); } static device_method_t iwm_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, iwm_probe), DEVMETHOD(device_attach, iwm_attach), DEVMETHOD(device_detach, iwm_detach), DEVMETHOD(device_suspend, iwm_suspend), DEVMETHOD(device_resume, iwm_resume), DEVMETHOD_END }; static driver_t iwm_pci_driver = { "iwm", iwm_pci_methods, sizeof (struct iwm_softc) }; static devclass_t iwm_devclass; DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL); MODULE_PNP_INFO("U16:device;P:#;T:vendor=0x8086", pci, iwm_pci_driver, iwm_devices, nitems(iwm_devices)); MODULE_DEPEND(iwm, firmware, 1, 1, 1); MODULE_DEPEND(iwm, pci, 1, 1, 1); MODULE_DEPEND(iwm, wlan, 1, 1, 1); Index: projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_7000.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_7000.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_7000.c (revision 345026) @@ -1,137 +1,138 @@ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * ****************************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved. * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright(c) 2015 Intel Deutschland GmbH * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * BSD LICENSE * * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved. * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright(c) 2015 Intel Deutschland GmbH * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * *****************************************************************************/ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwm.h" #include #include "if_iwm_config.h" #define IWM7260_FW "iwm7260fw" #define IWM3160_FW "iwm3160fw" #define IWM3168_FW "iwm3168fw" #define IWM7265_FW "iwm7265fw" #define IWM7265D_FW "iwm7265Dfw" #define IWM_NVM_HW_SECTION_NUM_FAMILY_7000 0 #define IWM_DEVICE_7000_COMMON \ .device_family = IWM_DEVICE_FAMILY_7000, \ .eeprom_size = IWM_OTP_LOW_IMAGE_SIZE_FAMILY_7000, \ .nvm_hw_section_num = IWM_NVM_HW_SECTION_NUM_FAMILY_7000, \ .apmg_wake_up_wa = 1 const struct iwm_cfg iwm7260_cfg = { .name = "Intel(R) Dual Band Wireless AC 7260", .fw_name = IWM7260_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 1, }; const struct iwm_cfg iwm3160_cfg = { .name = "Intel(R) Dual Band Wireless AC 3160", .fw_name = IWM3160_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 1, }; const struct iwm_cfg iwm3165_cfg = { .name = "Intel(R) Dual Band Wireless AC 3165", .fw_name = IWM7265D_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 0, }; const struct iwm_cfg iwm3168_cfg = { .name = "Intel(R) Dual Band Wireless AC 3168", .fw_name = IWM3168_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 0, + .nvm_type = IWM_NVM_SDP, }; const struct iwm_cfg iwm7265_cfg = { .name = "Intel(R) Dual Band Wireless AC 7265", .fw_name = IWM7265_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 0, }; const struct iwm_cfg iwm7265d_cfg = { .name = "Intel(R) Dual Band Wireless AC 7265", .fw_name = IWM7265D_FW, IWM_DEVICE_7000_COMMON, .host_interrupt_operation_mode = 0, }; Index: projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_config.h =================================================================== --- projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_config.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/iwm/if_iwm_config.h (revision 345026) @@ -1,139 +1,154 @@ /*- * Based on BSD-licensed source modules in the Linux iwlwifi driver, * which were used as the reference documentation for this implementation. * ****************************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved. * Copyright (C) 2016 Intel Deutschland GmbH * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * BSD LICENSE * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. * Copyright (C) 2016 Intel Deutschland GmbH * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * *****************************************************************************/ /* * $FreeBSD$ */ #ifndef __IWM_CONFIG_H__ #define __IWM_CONFIG_H__ enum iwm_device_family { IWM_DEVICE_FAMILY_UNDEFINED, IWM_DEVICE_FAMILY_7000, IWM_DEVICE_FAMILY_8000, }; #define IWM_DEFAULT_MAX_TX_POWER 22 /* Antenna presence definitions */ #define IWM_ANT_NONE 0x0 #define IWM_ANT_A (1 << 0) #define IWM_ANT_B (1 << 1) #define IWM_ANT_C (1 << 2) #define IWM_ANT_AB (IWM_ANT_A | IWM_ANT_B) #define IWM_ANT_AC (IWM_ANT_A | IWM_ANT_C) #define IWM_ANT_BC (IWM_ANT_B | IWM_ANT_C) #define IWM_ANT_ABC (IWM_ANT_A | IWM_ANT_B | IWM_ANT_C) static inline uint8_t num_of_ant(uint8_t mask) { return !!((mask) & IWM_ANT_A) + !!((mask) & IWM_ANT_B) + !!((mask) & IWM_ANT_C); } /* lower blocks contain EEPROM image and calibration data */ #define IWM_OTP_LOW_IMAGE_SIZE_FAMILY_7000 (16 * 512 * sizeof(uint16_t)) /* 16 KB */ #define IWM_OTP_LOW_IMAGE_SIZE_FAMILY_8000 (32 * 512 * sizeof(uint16_t)) /* 32 KB */ #define IWM_OTP_LOW_IMAGE_SIZE_FAMILY_9000 IWM_OTP_LOW_IMAGE_SIZE_FAMILY_8000 + /** + * enum iwl_nvm_type - nvm formats + * @IWM_NVM: the regular format + * @IWM_NVM_EXT: extended NVM format + * @IWM_NVM_SDP: NVM format used by 3168 series + */ +enum iwm_nvm_type { + IWM_NVM, + IWM_NVM_EXT, + IWM_NVM_SDP, +}; + +/** * struct iwm_cfg * @name: Official name of the device * @fw_name: Firmware filename. * @host_interrupt_operation_mode: device needs host interrupt operation * mode set * @nvm_hw_section_num: the ID of the HW NVM section * @apmg_wake_up_wa: should the MAC access REQ be asserted when a command * is in flight. This is due to a HW bug in 7260, 3160 and 7265. + * @nvm_type: see &enum iwl_nvm_type */ struct iwm_cfg { const char *name; const char *fw_name; uint16_t eeprom_size; enum iwm_device_family device_family; int host_interrupt_operation_mode; uint8_t nvm_hw_section_num; int apmg_wake_up_wa; + enum iwm_nvm_type nvm_type; }; /* * This list declares the config structures for all devices. */ extern const struct iwm_cfg iwm7260_cfg; extern const struct iwm_cfg iwm3160_cfg; extern const struct iwm_cfg iwm3165_cfg; extern const struct iwm_cfg iwm3168_cfg; extern const struct iwm_cfg iwm7265_cfg; extern const struct iwm_cfg iwm7265d_cfg; extern const struct iwm_cfg iwm8260_cfg; extern const struct iwm_cfg iwm8265_cfg; #endif /* __IWM_CONFIG_H__ */ Index: projects/import-googletest-1.8.1/sys/dev/iwm/if_iwmreg.h =================================================================== --- projects/import-googletest-1.8.1/sys/dev/iwm/if_iwmreg.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/iwm/if_iwmreg.h (revision 345026) @@ -1,6139 +1,6140 @@ /* $OpenBSD: if_iwmreg.h,v 1.4 2015/06/15 08:06:11 stsp Exp $ */ /* $FreeBSD$ */ /****************************************************************************** * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, * USA * * The full GNU General Public License is included in this distribution * in the file called COPYING. * * Contact Information: * Intel Linux Wireless * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 * * BSD LICENSE * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * *****************************************************************************/ #ifndef __IF_IWM_REG_H__ #define __IF_IWM_REG_H__ #define le16_to_cpup(_a_) (le16toh(*(const uint16_t *)(_a_))) #define le32_to_cpup(_a_) (le32toh(*(const uint32_t *)(_a_))) /* * BEGIN iwl-csr.h */ /* * CSR (control and status registers) * * CSR registers are mapped directly into PCI bus space, and are accessible * whenever platform supplies power to device, even when device is in * low power states due to driver-invoked device resets * (e.g. IWM_CSR_RESET_REG_FLAG_SW_RESET) or uCode-driven power-saving modes. * * Use iwl_write32() and iwl_read32() family to access these registers; * these provide simple PCI bus access, without waking up the MAC. * Do not use iwl_write_direct32() family for these registers; * no need to "grab nic access" via IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ. * The MAC (uCode processor, etc.) does not need to be powered up for accessing * the CSR registers. * * NOTE: Device does need to be awake in order to read this memory * via IWM_CSR_EEPROM and IWM_CSR_OTP registers */ #define IWM_CSR_HW_IF_CONFIG_REG (0x000) /* hardware interface config */ #define IWM_CSR_INT_COALESCING (0x004) /* accum ints, 32-usec units */ #define IWM_CSR_INT (0x008) /* host interrupt status/ack */ #define IWM_CSR_INT_MASK (0x00c) /* host interrupt enable */ #define IWM_CSR_FH_INT_STATUS (0x010) /* busmaster int status/ack*/ #define IWM_CSR_GPIO_IN (0x018) /* read external chip pins */ #define IWM_CSR_RESET (0x020) /* busmaster enable, NMI, etc*/ #define IWM_CSR_GP_CNTRL (0x024) /* 2nd byte of IWM_CSR_INT_COALESCING, not accessible via iwl_write32()! */ #define IWM_CSR_INT_PERIODIC_REG (0x005) /* * Hardware revision info * Bit fields: * 31-16: Reserved * 15-4: Type of device: see IWM_CSR_HW_REV_TYPE_xxx definitions * 3-2: Revision step: 0 = A, 1 = B, 2 = C, 3 = D * 1-0: "Dash" (-) value, as in A-1, etc. */ #define IWM_CSR_HW_REV (0x028) /* * EEPROM and OTP (one-time-programmable) memory reads * * NOTE: Device must be awake, initialized via apm_ops.init(), * in order to read. */ #define IWM_CSR_EEPROM_REG (0x02c) #define IWM_CSR_EEPROM_GP (0x030) #define IWM_CSR_OTP_GP_REG (0x034) #define IWM_CSR_GIO_REG (0x03C) #define IWM_CSR_GP_UCODE_REG (0x048) #define IWM_CSR_GP_DRIVER_REG (0x050) /* * UCODE-DRIVER GP (general purpose) mailbox registers. * SET/CLR registers set/clear bit(s) if "1" is written. */ #define IWM_CSR_UCODE_DRV_GP1 (0x054) #define IWM_CSR_UCODE_DRV_GP1_SET (0x058) #define IWM_CSR_UCODE_DRV_GP1_CLR (0x05c) #define IWM_CSR_UCODE_DRV_GP2 (0x060) #define IWM_CSR_MBOX_SET_REG (0x088) #define IWM_CSR_MBOX_SET_REG_OS_ALIVE 0x20 #define IWM_CSR_LED_REG (0x094) #define IWM_CSR_DRAM_INT_TBL_REG (0x0A0) #define IWM_CSR_MAC_SHADOW_REG_CTRL (0x0A8) /* 6000 and up */ /* GIO Chicken Bits (PCI Express bus link power management) */ #define IWM_CSR_GIO_CHICKEN_BITS (0x100) /* Analog phase-lock-loop configuration */ #define IWM_CSR_ANA_PLL_CFG (0x20c) /* * CSR Hardware Revision Workaround Register. Indicates hardware rev; * "step" determines CCK backoff for txpower calculation. Used for 4965 only. * See also IWM_CSR_HW_REV register. * Bit fields: * 3-2: 0 = A, 1 = B, 2 = C, 3 = D step * 1-0: "Dash" (-) value, as in C-1, etc. */ #define IWM_CSR_HW_REV_WA_REG (0x22C) #define IWM_CSR_DBG_HPET_MEM_REG (0x240) #define IWM_CSR_DBG_LINK_PWR_MGMT_REG (0x250) /* Bits for IWM_CSR_HW_IF_CONFIG_REG */ #define IWM_CSR_HW_IF_CONFIG_REG_MSK_MAC_DASH (0x00000003) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_MAC_STEP (0x0000000C) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_BOARD_VER (0x000000C0) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_MAC_SI (0x00000100) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_RADIO_SI (0x00000200) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_TYPE (0x00000C00) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_DASH (0x00003000) #define IWM_CSR_HW_IF_CONFIG_REG_MSK_PHY_STEP (0x0000C000) #define IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH (0) #define IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP (2) #define IWM_CSR_HW_IF_CONFIG_REG_POS_BOARD_VER (6) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE (10) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH (12) #define IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP (14) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A (0x00080000) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_EEPROM_OWN_SEM (0x00200000) #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY (0x00400000) /* PCI_OWN_SEM */ #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE (0x02000000) /* ME_OWN */ #define IWM_CSR_HW_IF_CONFIG_REG_PREPARE (0x08000000) /* WAKE_ME */ #define IWM_CSR_HW_IF_CONFIG_REG_ENABLE_PME (0x10000000) #define IWM_CSR_HW_IF_CONFIG_REG_PERSIST_MODE (0x40000000) /* PERSISTENCE */ #define IWM_CSR_INT_PERIODIC_DIS (0x00) /* disable periodic int*/ #define IWM_CSR_INT_PERIODIC_ENA (0xFF) /* 255*32 usec ~ 8 msec*/ /* interrupt flags in INTA, set by uCode or hardware (e.g. dma), * acknowledged (reset) by host writing "1" to flagged bits. */ #define IWM_CSR_INT_BIT_FH_RX (1 << 31) /* Rx DMA, cmd responses, FH_INT[17:16] */ #define IWM_CSR_INT_BIT_HW_ERR (1 << 29) /* DMA hardware error FH_INT[31] */ #define IWM_CSR_INT_BIT_RX_PERIODIC (1 << 28) /* Rx periodic */ #define IWM_CSR_INT_BIT_FH_TX (1 << 27) /* Tx DMA FH_INT[1:0] */ #define IWM_CSR_INT_BIT_SCD (1 << 26) /* TXQ pointer advanced */ #define IWM_CSR_INT_BIT_SW_ERR (1 << 25) /* uCode error */ #define IWM_CSR_INT_BIT_RF_KILL (1 << 7) /* HW RFKILL switch GP_CNTRL[27] toggled */ #define IWM_CSR_INT_BIT_CT_KILL (1 << 6) /* Critical temp (chip too hot) rfkill */ #define IWM_CSR_INT_BIT_SW_RX (1 << 3) /* Rx, command responses */ #define IWM_CSR_INT_BIT_WAKEUP (1 << 1) /* NIC controller waking up (pwr mgmt) */ #define IWM_CSR_INT_BIT_ALIVE (1 << 0) /* uCode interrupts once it initializes */ #define IWM_CSR_INI_SET_MASK (IWM_CSR_INT_BIT_FH_RX | \ IWM_CSR_INT_BIT_HW_ERR | \ IWM_CSR_INT_BIT_FH_TX | \ IWM_CSR_INT_BIT_SW_ERR | \ IWM_CSR_INT_BIT_RF_KILL | \ IWM_CSR_INT_BIT_SW_RX | \ IWM_CSR_INT_BIT_WAKEUP | \ IWM_CSR_INT_BIT_ALIVE | \ IWM_CSR_INT_BIT_RX_PERIODIC) /* interrupt flags in FH (flow handler) (PCI busmaster DMA) */ #define IWM_CSR_FH_INT_BIT_ERR (1 << 31) /* Error */ #define IWM_CSR_FH_INT_BIT_HI_PRIOR (1 << 30) /* High priority Rx, bypass coalescing */ #define IWM_CSR_FH_INT_BIT_RX_CHNL1 (1 << 17) /* Rx channel 1 */ #define IWM_CSR_FH_INT_BIT_RX_CHNL0 (1 << 16) /* Rx channel 0 */ #define IWM_CSR_FH_INT_BIT_TX_CHNL1 (1 << 1) /* Tx channel 1 */ #define IWM_CSR_FH_INT_BIT_TX_CHNL0 (1 << 0) /* Tx channel 0 */ #define IWM_CSR_FH_INT_RX_MASK (IWM_CSR_FH_INT_BIT_HI_PRIOR | \ IWM_CSR_FH_INT_BIT_RX_CHNL1 | \ IWM_CSR_FH_INT_BIT_RX_CHNL0) #define IWM_CSR_FH_INT_TX_MASK (IWM_CSR_FH_INT_BIT_TX_CHNL1 | \ IWM_CSR_FH_INT_BIT_TX_CHNL0) /* GPIO */ #define IWM_CSR_GPIO_IN_BIT_AUX_POWER (0x00000200) #define IWM_CSR_GPIO_IN_VAL_VAUX_PWR_SRC (0x00000000) #define IWM_CSR_GPIO_IN_VAL_VMAIN_PWR_SRC (0x00000200) /* RESET */ #define IWM_CSR_RESET_REG_FLAG_NEVO_RESET (0x00000001) #define IWM_CSR_RESET_REG_FLAG_FORCE_NMI (0x00000002) #define IWM_CSR_RESET_REG_FLAG_SW_RESET (0x00000080) #define IWM_CSR_RESET_REG_FLAG_MASTER_DISABLED (0x00000100) #define IWM_CSR_RESET_REG_FLAG_STOP_MASTER (0x00000200) #define IWM_CSR_RESET_LINK_PWR_MGMT_DISABLED (0x80000000) /* * GP (general purpose) CONTROL REGISTER * Bit fields: * 27: HW_RF_KILL_SW * Indicates state of (platform's) hardware RF-Kill switch * 26-24: POWER_SAVE_TYPE * Indicates current power-saving mode: * 000 -- No power saving * 001 -- MAC power-down * 010 -- PHY (radio) power-down * 011 -- Error * 9-6: SYS_CONFIG * Indicates current system configuration, reflecting pins on chip * as forced high/low by device circuit board. * 4: GOING_TO_SLEEP * Indicates MAC is entering a power-saving sleep power-down. * Not a good time to access device-internal resources. * 3: MAC_ACCESS_REQ * Host sets this to request and maintain MAC wakeup, to allow host * access to device-internal resources. Host must wait for * MAC_CLOCK_READY (and !GOING_TO_SLEEP) before accessing non-CSR * device registers. * 2: INIT_DONE * Host sets this to put device into fully operational D0 power mode. * Host resets this after SW_RESET to put device into low power mode. * 0: MAC_CLOCK_READY * Indicates MAC (ucode processor, etc.) is powered up and can run. * Internal resources are accessible. * NOTE: This does not indicate that the processor is actually running. * NOTE: This does not indicate that device has completed * init or post-power-down restore of internal SRAM memory. * Use IWM_CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP as indication that * SRAM is restored and uCode is in normal operation mode. * Later devices (5xxx/6xxx/1xxx) use non-volatile SRAM, and * do not need to save/restore it. * NOTE: After device reset, this bit remains "0" until host sets * INIT_DONE */ #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY (0x00000001) #define IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE (0x00000004) #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ (0x00000008) #define IWM_CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP (0x00000010) #define IWM_CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN (0x00000001) #define IWM_CSR_GP_CNTRL_REG_MSK_POWER_SAVE_TYPE (0x07000000) #define IWM_CSR_GP_CNTRL_REG_FLAG_MAC_POWER_SAVE (0x04000000) #define IWM_CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW (0x08000000) /* HW REV */ #define IWM_CSR_HW_REV_DASH(_val) (((_val) & 0x0000003) >> 0) #define IWM_CSR_HW_REV_STEP(_val) (((_val) & 0x000000C) >> 2) /** * hw_rev values */ enum { IWM_SILICON_A_STEP = 0, IWM_SILICON_B_STEP, IWM_SILICON_C_STEP, }; #define IWM_CSR_HW_REV_TYPE_MSK (0x000FFF0) #define IWM_CSR_HW_REV_TYPE_5300 (0x0000020) #define IWM_CSR_HW_REV_TYPE_5350 (0x0000030) #define IWM_CSR_HW_REV_TYPE_5100 (0x0000050) #define IWM_CSR_HW_REV_TYPE_5150 (0x0000040) #define IWM_CSR_HW_REV_TYPE_1000 (0x0000060) #define IWM_CSR_HW_REV_TYPE_6x00 (0x0000070) #define IWM_CSR_HW_REV_TYPE_6x50 (0x0000080) #define IWM_CSR_HW_REV_TYPE_6150 (0x0000084) #define IWM_CSR_HW_REV_TYPE_6x05 (0x00000B0) #define IWM_CSR_HW_REV_TYPE_6x30 IWM_CSR_HW_REV_TYPE_6x05 #define IWM_CSR_HW_REV_TYPE_6x35 IWM_CSR_HW_REV_TYPE_6x05 #define IWM_CSR_HW_REV_TYPE_2x30 (0x00000C0) #define IWM_CSR_HW_REV_TYPE_2x00 (0x0000100) #define IWM_CSR_HW_REV_TYPE_105 (0x0000110) #define IWM_CSR_HW_REV_TYPE_135 (0x0000120) #define IWM_CSR_HW_REV_TYPE_7265D (0x0000210) #define IWM_CSR_HW_REV_TYPE_NONE (0x00001F0) /* EEPROM REG */ #define IWM_CSR_EEPROM_REG_READ_VALID_MSK (0x00000001) #define IWM_CSR_EEPROM_REG_BIT_CMD (0x00000002) #define IWM_CSR_EEPROM_REG_MSK_ADDR (0x0000FFFC) #define IWM_CSR_EEPROM_REG_MSK_DATA (0xFFFF0000) /* EEPROM GP */ #define IWM_CSR_EEPROM_GP_VALID_MSK (0x00000007) /* signature */ #define IWM_CSR_EEPROM_GP_IF_OWNER_MSK (0x00000180) #define IWM_CSR_EEPROM_GP_BAD_SIGNATURE_BOTH_EEP_AND_OTP (0x00000000) #define IWM_CSR_EEPROM_GP_BAD_SIG_EEP_GOOD_SIG_OTP (0x00000001) #define IWM_CSR_EEPROM_GP_GOOD_SIG_EEP_LESS_THAN_4K (0x00000002) #define IWM_CSR_EEPROM_GP_GOOD_SIG_EEP_MORE_THAN_4K (0x00000004) /* One-time-programmable memory general purpose reg */ #define IWM_CSR_OTP_GP_REG_DEVICE_SELECT (0x00010000) /* 0 - EEPROM, 1 - OTP */ #define IWM_CSR_OTP_GP_REG_OTP_ACCESS_MODE (0x00020000) /* 0 - absolute, 1 - relative */ #define IWM_CSR_OTP_GP_REG_ECC_CORR_STATUS_MSK (0x00100000) /* bit 20 */ #define IWM_CSR_OTP_GP_REG_ECC_UNCORR_STATUS_MSK (0x00200000) /* bit 21 */ /* GP REG */ #define IWM_CSR_GP_REG_POWER_SAVE_STATUS_MSK (0x03000000) /* bit 24/25 */ #define IWM_CSR_GP_REG_NO_POWER_SAVE (0x00000000) #define IWM_CSR_GP_REG_MAC_POWER_SAVE (0x01000000) #define IWM_CSR_GP_REG_PHY_POWER_SAVE (0x02000000) #define IWM_CSR_GP_REG_POWER_SAVE_ERROR (0x03000000) /* CSR GIO */ #define IWM_CSR_GIO_REG_VAL_L0S_ENABLED (0x00000002) /* * UCODE-DRIVER GP (general purpose) mailbox register 1 * Host driver and uCode write and/or read this register to communicate with * each other. * Bit fields: * 4: UCODE_DISABLE * Host sets this to request permanent halt of uCode, same as * sending CARD_STATE command with "halt" bit set. * 3: CT_KILL_EXIT * Host sets this to request exit from CT_KILL state, i.e. host thinks * device temperature is low enough to continue normal operation. * 2: CMD_BLOCKED * Host sets this during RF KILL power-down sequence (HW, SW, CT KILL) * to release uCode to clear all Tx and command queues, enter * unassociated mode, and power down. * NOTE: Some devices also use HBUS_TARG_MBX_C register for this bit. * 1: SW_BIT_RFKILL * Host sets this when issuing CARD_STATE command to request * device sleep. * 0: MAC_SLEEP * uCode sets this when preparing a power-saving power-down. * uCode resets this when power-up is complete and SRAM is sane. * NOTE: device saves internal SRAM data to host when powering down, * and must restore this data after powering back up. * MAC_SLEEP is the best indication that restore is complete. * Later devices (5xxx/6xxx/1xxx) use non-volatile SRAM, and * do not need to save/restore it. */ #define IWM_CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP (0x00000001) #define IWM_CSR_UCODE_SW_BIT_RFKILL (0x00000002) #define IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED (0x00000004) #define IWM_CSR_UCODE_DRV_GP1_REG_BIT_CT_KILL_EXIT (0x00000008) #define IWM_CSR_UCODE_DRV_GP1_BIT_D3_CFG_COMPLETE (0x00000020) /* GP Driver */ #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_MSK (0x00000003) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_3x3_HYB (0x00000000) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_2x2_HYB (0x00000001) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_SKU_2x2_IPA (0x00000002) #define IWM_CSR_GP_DRIVER_REG_BIT_CALIB_VERSION6 (0x00000004) #define IWM_CSR_GP_DRIVER_REG_BIT_6050_1x2 (0x00000008) #define IWM_CSR_GP_DRIVER_REG_BIT_RADIO_IQ_INVER (0x00000080) /* GIO Chicken Bits (PCI Express bus link power management) */ #define IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX (0x00800000) #define IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER (0x20000000) /* LED */ #define IWM_CSR_LED_BSM_CTRL_MSK (0xFFFFFFDF) #define IWM_CSR_LED_REG_TURN_ON (0x60) #define IWM_CSR_LED_REG_TURN_OFF (0x20) /* ANA_PLL */ #define IWM_CSR50_ANA_PLL_CFG_VAL (0x00880300) /* HPET MEM debug */ #define IWM_CSR_DBG_HPET_MEM_REG_VAL (0xFFFF0000) /* DRAM INT TABLE */ #define IWM_CSR_DRAM_INT_TBL_ENABLE (1 << 31) #define IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER (1 << 28) #define IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK (1 << 27) /* SECURE boot registers */ #define IWM_CSR_SECURE_BOOT_CONFIG_ADDR (0x100) enum iwm_secure_boot_config_reg { IWM_CSR_SECURE_BOOT_CONFIG_INSPECTOR_BURNED_IN_OTP = 0x00000001, IWM_CSR_SECURE_BOOT_CONFIG_INSPECTOR_NOT_REQ = 0x00000002, }; #define IWM_CSR_SECURE_BOOT_CPU1_STATUS_ADDR (0x100) #define IWM_CSR_SECURE_BOOT_CPU2_STATUS_ADDR (0x100) enum iwm_secure_boot_status_reg { IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_STATUS = 0x00000003, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_COMPLETED = 0x00000002, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_SUCCESS = 0x00000004, IWM_CSR_SECURE_BOOT_CPU_STATUS_VERF_FAIL = 0x00000008, IWM_CSR_SECURE_BOOT_CPU_STATUS_SIGN_VERF_FAIL = 0x00000010, }; #define IWM_FH_UCODE_LOAD_STATUS 0x1af0 #define IWM_FH_MEM_TB_MAX_LENGTH 0x20000 #define IWM_LMPM_SECURE_UCODE_LOAD_CPU1_HDR_ADDR 0x1e78 #define IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR 0x1e7c #define IWM_LMPM_SECURE_CPU1_HDR_MEM_SPACE 0x420000 #define IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE 0x420400 #define IWM_CSR_SECURE_TIME_OUT (100) /* extended range in FW SRAM */ #define IWM_FW_MEM_EXTENDED_START 0x40000 #define IWM_FW_MEM_EXTENDED_END 0x57FFF /* FW chicken bits */ #define IWM_LMPM_CHICK 0xa01ff8 #define IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE 0x01 #define IWM_FH_TCSR_0_REG0 (0x1D00) /* * HBUS (Host-side Bus) * * HBUS registers are mapped directly into PCI bus space, but are used * to indirectly access device's internal memory or registers that * may be powered-down. * * Use iwl_write_direct32()/iwl_read_direct32() family for these registers; * host must "grab nic access" via CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ * to make sure the MAC (uCode processor, etc.) is powered up for accessing * internal resources. * * Do not use iwl_write32()/iwl_read32() family to access these registers; * these provide only simple PCI bus access, without waking up the MAC. */ #define IWM_HBUS_BASE (0x400) /* * Registers for accessing device's internal SRAM memory (e.g. SCD SRAM * structures, error log, event log, verifying uCode load). * First write to address register, then read from or write to data register * to complete the job. Once the address register is set up, accesses to * data registers auto-increment the address by one dword. * Bit usage for address registers (read or write): * 0-31: memory address within device */ #define IWM_HBUS_TARG_MEM_RADDR (IWM_HBUS_BASE+0x00c) #define IWM_HBUS_TARG_MEM_WADDR (IWM_HBUS_BASE+0x010) #define IWM_HBUS_TARG_MEM_WDAT (IWM_HBUS_BASE+0x018) #define IWM_HBUS_TARG_MEM_RDAT (IWM_HBUS_BASE+0x01c) /* Mailbox C, used as workaround alternative to CSR_UCODE_DRV_GP1 mailbox */ #define IWM_HBUS_TARG_MBX_C (IWM_HBUS_BASE+0x030) #define IWM_HBUS_TARG_MBX_C_REG_BIT_CMD_BLOCKED (0x00000004) /* * Registers for accessing device's internal peripheral registers * (e.g. SCD, BSM, etc.). First write to address register, * then read from or write to data register to complete the job. * Bit usage for address registers (read or write): * 0-15: register address (offset) within device * 24-25: (# bytes - 1) to read or write (e.g. 3 for dword) */ #define IWM_HBUS_TARG_PRPH_WADDR (IWM_HBUS_BASE+0x044) #define IWM_HBUS_TARG_PRPH_RADDR (IWM_HBUS_BASE+0x048) #define IWM_HBUS_TARG_PRPH_WDAT (IWM_HBUS_BASE+0x04c) #define IWM_HBUS_TARG_PRPH_RDAT (IWM_HBUS_BASE+0x050) /* enable the ID buf for read */ #define IWM_WFPM_PS_CTL_CLR 0xa0300c #define IWM_WFMP_MAC_ADDR_0 0xa03080 #define IWM_WFMP_MAC_ADDR_1 0xa03084 #define IWM_LMPM_PMG_EN 0xa01cec #define IWM_RADIO_REG_SYS_MANUAL_DFT_0 0xad4078 #define IWM_RFIC_REG_RD 0xad0470 #define IWM_WFPM_CTRL_REG 0xa03030 #define IWM_WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK 0x08000000 #define IWM_ENABLE_WFPM 0x80000000 #define IWM_AUX_MISC_REG 0xa200b0 #define IWM_HW_STEP_LOCATION_BITS 24 #define IWM_AUX_MISC_MASTER1_EN 0xa20818 #define IWM_AUX_MISC_MASTER1_EN_SBE_MSK 0x1 #define IWM_AUX_MISC_MASTER1_SMPHR_STATUS 0xa20800 #define IWM_RSA_ENABLE 0xa24b08 #define IWM_PREG_AUX_BUS_WPROT_0 0xa04cc0 #define IWM_SB_CFG_OVERRIDE_ADDR 0xa26c78 #define IWM_SB_CFG_OVERRIDE_ENABLE 0x8000 #define IWM_SB_CFG_BASE_OVERRIDE 0xa20000 #define IWM_SB_MODIFY_CFG_FLAG 0xa03088 #define IWM_SB_CPU_1_STATUS 0xa01e30 #define IWM_SB_CPU_2_STATUS 0Xa01e34 /* Used to enable DBGM */ #define IWM_HBUS_TARG_TEST_REG (IWM_HBUS_BASE+0x05c) /* * Per-Tx-queue write pointer (index, really!) * Indicates index to next TFD that driver will fill (1 past latest filled). * Bit usage: * 0-7: queue write index * 11-8: queue selector */ #define IWM_HBUS_TARG_WRPTR (IWM_HBUS_BASE+0x060) /********************************************************** * CSR values **********************************************************/ /* * host interrupt timeout value * used with setting interrupt coalescing timer * the CSR_INT_COALESCING is an 8 bit register in 32-usec unit * * default interrupt coalescing timer is 64 x 32 = 2048 usecs */ #define IWM_HOST_INT_TIMEOUT_MAX (0xFF) #define IWM_HOST_INT_TIMEOUT_DEF (0x40) #define IWM_HOST_INT_TIMEOUT_MIN (0x0) #define IWM_HOST_INT_OPER_MODE (1 << 31) /***************************************************************************** * 7000/3000 series SHR DTS addresses * *****************************************************************************/ /* Diode Results Register Structure: */ enum iwm_dtd_diode_reg { IWM_DTS_DIODE_REG_DIG_VAL = 0x000000FF, /* bits [7:0] */ IWM_DTS_DIODE_REG_VREF_LOW = 0x0000FF00, /* bits [15:8] */ IWM_DTS_DIODE_REG_VREF_HIGH = 0x00FF0000, /* bits [23:16] */ IWM_DTS_DIODE_REG_VREF_ID = 0x03000000, /* bits [25:24] */ IWM_DTS_DIODE_REG_PASS_ONCE = 0x80000000, /* bits [31:31] */ IWM_DTS_DIODE_REG_FLAGS_MSK = 0xFF000000, /* bits [31:24] */ /* Those are the masks INSIDE the flags bit-field: */ IWM_DTS_DIODE_REG_FLAGS_VREFS_ID_POS = 0, IWM_DTS_DIODE_REG_FLAGS_VREFS_ID = 0x00000003, /* bits [1:0] */ IWM_DTS_DIODE_REG_FLAGS_PASS_ONCE_POS = 7, IWM_DTS_DIODE_REG_FLAGS_PASS_ONCE = 0x00000080, /* bits [7:7] */ }; /* * END iwl-csr.h */ /* * BEGIN iwl-fw.h */ /** * enum iwm_ucode_tlv_flag - ucode API flags * @IWM_UCODE_TLV_FLAGS_PAN: This is PAN capable microcode; this previously * was a separate TLV but moved here to save space. * @IWM_UCODE_TLV_FLAGS_NEWSCAN: new uCode scan behaviour on hidden SSID, * treats good CRC threshold as a boolean * @IWM_UCODE_TLV_FLAGS_MFP: This uCode image supports MFP (802.11w). * @IWM_UCODE_TLV_FLAGS_UAPSD: This uCode image supports uAPSD * @IWM_UCODE_TLV_FLAGS_SHORT_BL: 16 entries of black list instead of 64 in scan * offload profile config command. * @IWM_UCODE_TLV_FLAGS_D3_6_IPV6_ADDRS: D3 image supports up to six * (rather than two) IPv6 addresses * @IWM_UCODE_TLV_FLAGS_NO_BASIC_SSID: not sending a probe with the SSID element * from the probe request template. * @IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL: new NS offload (small version) * @IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE: new NS offload (large version) * @IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT: General support for uAPSD * @IWM_UCODE_TLV_FLAGS_EBS_SUPPORT: this uCode image supports EBS. * @IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD: P2P client supports uAPSD power save * @IWM_UCODE_TLV_FLAGS_BCAST_FILTERING: uCode supports broadcast filtering. */ enum iwm_ucode_tlv_flag { IWM_UCODE_TLV_FLAGS_PAN = (1 << 0), IWM_UCODE_TLV_FLAGS_NEWSCAN = (1 << 1), IWM_UCODE_TLV_FLAGS_MFP = (1 << 2), IWM_UCODE_TLV_FLAGS_SHORT_BL = (1 << 7), IWM_UCODE_TLV_FLAGS_D3_6_IPV6_ADDRS = (1 << 10), IWM_UCODE_TLV_FLAGS_NO_BASIC_SSID = (1 << 12), IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL = (1 << 15), IWM_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE = (1 << 16), IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT = (1 << 24), IWM_UCODE_TLV_FLAGS_EBS_SUPPORT = (1 << 25), IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD = (1 << 26), IWM_UCODE_TLV_FLAGS_BCAST_FILTERING = (1 << 29), }; #define IWM_UCODE_TLV_FLAG_BITS \ "\020\1PAN\2NEWSCAN\3MFP\4P2P\5DW_BC_TABLE\6NEWBT_COEX\7PM_CMD\10SHORT_BL\11RX_ENERG \ Y\12TIME_EVENT_V2\13D3_6_IPV6\14BF_UPDATED\15NO_BASIC_SSID\17D3_CONTINUITY\20NEW_NSOFF \ L_S\21NEW_NSOFFL_L\22SCHED_SCAN\24STA_KEY_CMD\25DEVICE_PS_CMD\26P2P_PS\27P2P_PS_DCM\30 \ P2P_PS_SCM\31UAPSD_SUPPORT\32EBS\33P2P_PS_UAPSD\36BCAST_FILTERING\37GO_UAPSD\40LTE_COEX" /** * enum iwm_ucode_tlv_api - ucode api * @IWM_UCODE_TLV_API_FRAGMENTED_SCAN: This ucode supports active dwell time * longer than the passive one, which is essential for fragmented scan. * @IWM_UCODE_TLV_API_WIFI_MCC_UPDATE: ucode supports MCC updates with source. * @IWM_UCODE_TLV_API_LQ_SS_PARAMS: Configure STBC/BFER via LQ CMD ss_params * * @IWM_NUM_UCODE_TLV_API: number of bits used */ enum iwm_ucode_tlv_api { IWM_UCODE_TLV_API_FRAGMENTED_SCAN = 8, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE = 9, IWM_UCODE_TLV_API_LQ_SS_PARAMS = 18, IWM_NUM_UCODE_TLV_API = 32 }; #define IWM_UCODE_TLV_API_BITS \ "\020\10FRAGMENTED_SCAN\11WIFI_MCC_UPDATE\16WIDE_CMD_HDR\22LQ_SS_PARAMS\30EXT_SCAN_PRIO\33TX_POWER_CHAIN" /** * enum iwm_ucode_tlv_capa - ucode capabilities * @IWM_UCODE_TLV_CAPA_D0I3_SUPPORT: supports D0i3 * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT: supports Location Aware Regulatory * @IWM_UCODE_TLV_CAPA_UMAC_SCAN: supports UMAC scan. * @IWM_UCODE_TLV_CAPA_BEAMFORMER: supports Beamformer * @IWM_UCODE_TLV_CAPA_TOF_SUPPORT: supports Time of Flight (802.11mc FTM) * @IWM_UCODE_TLV_CAPA_TDLS_SUPPORT: support basic TDLS functionality * @IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT: supports insertion of current * tx power value into TPC Report action frame and Link Measurement Report * action frame * @IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT: supports updating current * channel in DS parameter set element in probe requests. * @IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT: supports adding TPC Report IE in * probe requests. * @IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT: supports Quiet Period requests * @IWM_UCODE_TLV_CAPA_DQA_SUPPORT: supports dynamic queue allocation (DQA), * which also implies support for the scheduler configuration command * @IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH: supports TDLS channel switching * @IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG: Consolidated D3-D0 image * @IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT: supports Hot Spot Command * @IWM_UCODE_TLV_CAPA_DC2DC_SUPPORT: supports DC2DC Command * @IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT: supports 2G coex Command * @IWM_UCODE_TLV_CAPA_CSUM_SUPPORT: supports TCP Checksum Offload * @IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS: support radio and beacon statistics * @IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD: support p2p standalone U-APSD * @IWM_UCODE_TLV_CAPA_BT_COEX_PLCR: enabled BT Coex packet level co-running * @IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC: ucode supports LAR updates with different * sources for the MCC. This TLV bit is a future replacement to * IWM_UCODE_TLV_API_WIFI_MCC_UPDATE. When either is set, multi-source LAR * is supported. * @IWM_UCODE_TLV_CAPA_BT_COEX_RRC: supports BT Coex RRC * @IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT: supports gscan * @IWM_UCODE_TLV_CAPA_NAN_SUPPORT: supports NAN * @IWM_UCODE_TLV_CAPA_UMAC_UPLOAD: supports upload mode in umac (1=supported, * 0=no support) * @IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE: extended DTS measurement * @IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS: supports short PM timeouts * @IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT: supports bt-coex Multi-priority LUT * @IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION: firmware will decide on what * antenna the beacon should be transmitted * @IWM_UCODE_TLV_CAPA_BEACON_STORING: firmware will store the latest beacon * from AP and will send it upon d0i3 exit. * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2: support LAR API V2 * @IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW: firmware responsible for CT-kill * @IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT: supports temperature * thresholds reporting * @IWM_UCODE_TLV_CAPA_CTDP_SUPPORT: supports cTDP command * @IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED: supports usniffer enabled in * regular image. * @IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG: support getting more shared * memory addresses from the firmware. * @IWM_UCODE_TLV_CAPA_LQM_SUPPORT: supports Link Quality Measurement * @IWM_UCODE_TLV_CAPA_TX_POWER_ACK: reduced TX power API has larger * command size (command version 4) that supports toggling ACK TX * power reduction. * * @IWM_NUM_UCODE_TLV_CAPA: number of bits used */ enum iwm_ucode_tlv_capa { IWM_UCODE_TLV_CAPA_D0I3_SUPPORT = 0, IWM_UCODE_TLV_CAPA_LAR_SUPPORT = 1, IWM_UCODE_TLV_CAPA_UMAC_SCAN = 2, IWM_UCODE_TLV_CAPA_BEAMFORMER = 3, IWM_UCODE_TLV_CAPA_TOF_SUPPORT = 5, IWM_UCODE_TLV_CAPA_TDLS_SUPPORT = 6, IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT = 8, IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT = 9, IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT = 10, IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT = 11, IWM_UCODE_TLV_CAPA_DQA_SUPPORT = 12, IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH = 13, IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG = 17, IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT = 18, IWM_UCODE_TLV_CAPA_DC2DC_CONFIG_SUPPORT = 19, IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT = 20, IWM_UCODE_TLV_CAPA_CSUM_SUPPORT = 21, IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS = 22, IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD = 26, IWM_UCODE_TLV_CAPA_BT_COEX_PLCR = 28, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC = 29, IWM_UCODE_TLV_CAPA_BT_COEX_RRC = 30, IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT = 31, IWM_UCODE_TLV_CAPA_NAN_SUPPORT = 34, IWM_UCODE_TLV_CAPA_UMAC_UPLOAD = 35, IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE = 64, IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS = 65, IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT = 67, IWM_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT = 68, IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION = 71, IWM_UCODE_TLV_CAPA_BEACON_STORING = 72, IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2 = 73, IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW = 74, IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT = 75, IWM_UCODE_TLV_CAPA_CTDP_SUPPORT = 76, IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED = 77, IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG = 80, IWM_UCODE_TLV_CAPA_LQM_SUPPORT = 81, IWM_UCODE_TLV_CAPA_TX_POWER_ACK = 84, IWM_NUM_UCODE_TLV_CAPA = 128 }; /* The default calibrate table size if not specified by firmware file */ #define IWM_DEFAULT_STANDARD_PHY_CALIBRATE_TBL_SIZE 18 #define IWM_MAX_STANDARD_PHY_CALIBRATE_TBL_SIZE 19 #define IWM_MAX_PHY_CALIBRATE_TBL_SIZE 253 /* The default max probe length if not specified by the firmware file */ #define IWM_DEFAULT_MAX_PROBE_LENGTH 200 /* * enumeration of ucode section. * This enumeration is used directly for older firmware (before 16.0). * For new firmware, there can be up to 4 sections (see below) but the * first one packaged into the firmware file is the DATA section and * some debugging code accesses that. */ enum iwm_ucode_sec { IWM_UCODE_SECTION_DATA, IWM_UCODE_SECTION_INST, }; /* * For 16.0 uCode and above, there is no differentiation between sections, * just an offset to the HW address. */ #define IWM_CPU1_CPU2_SEPARATOR_SECTION 0xFFFFCCCC #define IWM_PAGING_SEPARATOR_SECTION 0xAAAABBBB /* uCode version contains 4 values: Major/Minor/API/Serial */ #define IWM_UCODE_MAJOR(ver) (((ver) & 0xFF000000) >> 24) #define IWM_UCODE_MINOR(ver) (((ver) & 0x00FF0000) >> 16) #define IWM_UCODE_API(ver) (((ver) & 0x0000FF00) >> 8) #define IWM_UCODE_SERIAL(ver) ((ver) & 0x000000FF) /* * Calibration control struct. * Sent as part of the phy configuration command. * @flow_trigger: bitmap for which calibrations to perform according to * flow triggers. * @event_trigger: bitmap for which calibrations to perform according to * event triggers. */ struct iwm_tlv_calib_ctrl { uint32_t flow_trigger; uint32_t event_trigger; } __packed; enum iwm_fw_phy_cfg { IWM_FW_PHY_CFG_RADIO_TYPE_POS = 0, IWM_FW_PHY_CFG_RADIO_TYPE = 0x3 << IWM_FW_PHY_CFG_RADIO_TYPE_POS, IWM_FW_PHY_CFG_RADIO_STEP_POS = 2, IWM_FW_PHY_CFG_RADIO_STEP = 0x3 << IWM_FW_PHY_CFG_RADIO_STEP_POS, IWM_FW_PHY_CFG_RADIO_DASH_POS = 4, IWM_FW_PHY_CFG_RADIO_DASH = 0x3 << IWM_FW_PHY_CFG_RADIO_DASH_POS, IWM_FW_PHY_CFG_TX_CHAIN_POS = 16, IWM_FW_PHY_CFG_TX_CHAIN = 0xf << IWM_FW_PHY_CFG_TX_CHAIN_POS, IWM_FW_PHY_CFG_RX_CHAIN_POS = 20, IWM_FW_PHY_CFG_RX_CHAIN = 0xf << IWM_FW_PHY_CFG_RX_CHAIN_POS, }; #define IWM_UCODE_MAX_CS 1 /** * struct iwm_fw_cipher_scheme - a cipher scheme supported by FW. * @cipher: a cipher suite selector * @flags: cipher scheme flags (currently reserved for a future use) * @hdr_len: a size of MPDU security header * @pn_len: a size of PN * @pn_off: an offset of pn from the beginning of the security header * @key_idx_off: an offset of key index byte in the security header * @key_idx_mask: a bit mask of key_idx bits * @key_idx_shift: bit shift needed to get key_idx * @mic_len: mic length in bytes * @hw_cipher: a HW cipher index used in host commands */ struct iwm_fw_cipher_scheme { uint32_t cipher; uint8_t flags; uint8_t hdr_len; uint8_t pn_len; uint8_t pn_off; uint8_t key_idx_off; uint8_t key_idx_mask; uint8_t key_idx_shift; uint8_t mic_len; uint8_t hw_cipher; } __packed; /** * struct iwm_fw_cscheme_list - a cipher scheme list * @size: a number of entries * @cs: cipher scheme entries */ struct iwm_fw_cscheme_list { uint8_t size; struct iwm_fw_cipher_scheme cs[]; } __packed; /* * END iwl-fw.h */ /* * BEGIN iwl-fw-file.h */ /* v1/v2 uCode file layout */ struct iwm_ucode_header { uint32_t ver; /* major/minor/API/serial */ union { struct { uint32_t inst_size; /* bytes of runtime code */ uint32_t data_size; /* bytes of runtime data */ uint32_t init_size; /* bytes of init code */ uint32_t init_data_size; /* bytes of init data */ uint32_t boot_size; /* bytes of bootstrap code */ uint8_t data[0]; /* in same order as sizes */ } v1; struct { uint32_t build; /* build number */ uint32_t inst_size; /* bytes of runtime code */ uint32_t data_size; /* bytes of runtime data */ uint32_t init_size; /* bytes of init code */ uint32_t init_data_size; /* bytes of init data */ uint32_t boot_size; /* bytes of bootstrap code */ uint8_t data[0]; /* in same order as sizes */ } v2; } u; }; /* * new TLV uCode file layout * * The new TLV file format contains TLVs, that each specify * some piece of data. */ enum iwm_ucode_tlv_type { IWM_UCODE_TLV_INVALID = 0, /* unused */ IWM_UCODE_TLV_INST = 1, IWM_UCODE_TLV_DATA = 2, IWM_UCODE_TLV_INIT = 3, IWM_UCODE_TLV_INIT_DATA = 4, IWM_UCODE_TLV_BOOT = 5, IWM_UCODE_TLV_PROBE_MAX_LEN = 6, /* a uint32_t value */ IWM_UCODE_TLV_PAN = 7, IWM_UCODE_TLV_RUNT_EVTLOG_PTR = 8, IWM_UCODE_TLV_RUNT_EVTLOG_SIZE = 9, IWM_UCODE_TLV_RUNT_ERRLOG_PTR = 10, IWM_UCODE_TLV_INIT_EVTLOG_PTR = 11, IWM_UCODE_TLV_INIT_EVTLOG_SIZE = 12, IWM_UCODE_TLV_INIT_ERRLOG_PTR = 13, IWM_UCODE_TLV_ENHANCE_SENS_TBL = 14, IWM_UCODE_TLV_PHY_CALIBRATION_SIZE = 15, IWM_UCODE_TLV_WOWLAN_INST = 16, IWM_UCODE_TLV_WOWLAN_DATA = 17, IWM_UCODE_TLV_FLAGS = 18, IWM_UCODE_TLV_SEC_RT = 19, IWM_UCODE_TLV_SEC_INIT = 20, IWM_UCODE_TLV_SEC_WOWLAN = 21, IWM_UCODE_TLV_DEF_CALIB = 22, IWM_UCODE_TLV_PHY_SKU = 23, IWM_UCODE_TLV_SECURE_SEC_RT = 24, IWM_UCODE_TLV_SECURE_SEC_INIT = 25, IWM_UCODE_TLV_SECURE_SEC_WOWLAN = 26, IWM_UCODE_TLV_NUM_OF_CPU = 27, IWM_UCODE_TLV_CSCHEME = 28, /* * Following two are not in our base tag, but allow * handling ucode version 9. */ IWM_UCODE_TLV_API_CHANGES_SET = 29, IWM_UCODE_TLV_ENABLED_CAPABILITIES = 30, IWM_UCODE_TLV_N_SCAN_CHANNELS = 31, IWM_UCODE_TLV_PAGING = 32, IWM_UCODE_TLV_SEC_RT_USNIFFER = 34, IWM_UCODE_TLV_SDIO_ADMA_ADDR = 35, IWM_UCODE_TLV_FW_VERSION = 36, IWM_UCODE_TLV_FW_DBG_DEST = 38, IWM_UCODE_TLV_FW_DBG_CONF = 39, IWM_UCODE_TLV_FW_DBG_TRIGGER = 40, IWM_UCODE_TLV_FW_GSCAN_CAPA = 50, IWM_UCODE_TLV_FW_MEM_SEG = 51, }; struct iwm_ucode_tlv { uint32_t type; /* see above */ uint32_t length; /* not including type/length fields */ uint8_t data[0]; }; struct iwm_ucode_api { uint32_t api_index; uint32_t api_flags; } __packed; struct iwm_ucode_capa { uint32_t api_index; uint32_t api_capa; } __packed; #define IWM_TLV_UCODE_MAGIC 0x0a4c5749 struct iwm_tlv_ucode_header { /* * The TLV style ucode header is distinguished from * the v1/v2 style header by first four bytes being * zero, as such is an invalid combination of * major/minor/API/serial versions. */ uint32_t zero; uint32_t magic; uint8_t human_readable[64]; uint32_t ver; /* major/minor/API/serial */ uint32_t build; uint64_t ignore; /* * The data contained herein has a TLV layout, * see above for the TLV header and types. * Note that each TLV is padded to a length * that is a multiple of 4 for alignment. */ uint8_t data[0]; }; /* * END iwl-fw-file.h */ /* * BEGIN iwl-prph.h */ /* * Registers in this file are internal, not PCI bus memory mapped. * Driver accesses these via IWM_HBUS_TARG_PRPH_* registers. */ #define IWM_PRPH_BASE (0x00000) #define IWM_PRPH_END (0xFFFFF) /* APMG (power management) constants */ #define IWM_APMG_BASE (IWM_PRPH_BASE + 0x3000) #define IWM_APMG_CLK_CTRL_REG (IWM_APMG_BASE + 0x0000) #define IWM_APMG_CLK_EN_REG (IWM_APMG_BASE + 0x0004) #define IWM_APMG_CLK_DIS_REG (IWM_APMG_BASE + 0x0008) #define IWM_APMG_PS_CTRL_REG (IWM_APMG_BASE + 0x000c) #define IWM_APMG_PCIDEV_STT_REG (IWM_APMG_BASE + 0x0010) #define IWM_APMG_RFKILL_REG (IWM_APMG_BASE + 0x0014) #define IWM_APMG_RTC_INT_STT_REG (IWM_APMG_BASE + 0x001c) #define IWM_APMG_RTC_INT_MSK_REG (IWM_APMG_BASE + 0x0020) #define IWM_APMG_DIGITAL_SVR_REG (IWM_APMG_BASE + 0x0058) #define IWM_APMG_ANALOG_SVR_REG (IWM_APMG_BASE + 0x006C) #define IWM_APMS_CLK_VAL_MRB_FUNC_MODE (0x00000001) #define IWM_APMG_CLK_VAL_DMA_CLK_RQT (0x00000200) #define IWM_APMG_CLK_VAL_BSM_CLK_RQT (0x00000800) #define IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS (0x00400000) #define IWM_APMG_PS_CTRL_VAL_RESET_REQ (0x04000000) #define IWM_APMG_PS_CTRL_MSK_PWR_SRC (0x03000000) #define IWM_APMG_PS_CTRL_VAL_PWR_SRC_VMAIN (0x00000000) #define IWM_APMG_PS_CTRL_VAL_PWR_SRC_VAUX (0x02000000) #define IWM_APMG_SVR_VOLTAGE_CONFIG_BIT_MSK (0x000001E0) /* bit 8:5 */ #define IWM_APMG_SVR_DIGITAL_VOLTAGE_1_32 (0x00000060) #define IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS (0x00000800) #define IWM_APMG_RTC_INT_STT_RFKILL (0x10000000) /* Device system time */ #define IWM_DEVICE_SYSTEM_TIME_REG 0xA0206C /* Device NMI register */ #define IWM_DEVICE_SET_NMI_REG 0x00a01c30 #define IWM_DEVICE_SET_NMI_VAL_HW 0x01 #define IWM_DEVICE_SET_NMI_VAL_DRV 0x80 #define IWM_DEVICE_SET_NMI_8000_REG 0x00a01c24 #define IWM_DEVICE_SET_NMI_8000_VAL 0x1000000 /* * Device reset for family 8000 * write to bit 24 in order to reset the CPU */ #define IWM_RELEASE_CPU_RESET 0x300c #define IWM_RELEASE_CPU_RESET_BIT 0x1000000 /***************************************************************************** * 7000/3000 series SHR DTS addresses * *****************************************************************************/ #define IWM_SHR_MISC_WFM_DTS_EN (0x00a10024) #define IWM_DTSC_CFG_MODE (0x00a10604) #define IWM_DTSC_VREF_AVG (0x00a10648) #define IWM_DTSC_VREF5_AVG (0x00a1064c) #define IWM_DTSC_CFG_MODE_PERIODIC (0x2) #define IWM_DTSC_PTAT_AVG (0x00a10650) /** * Tx Scheduler * * The Tx Scheduler selects the next frame to be transmitted, choosing TFDs * (Transmit Frame Descriptors) from up to 16 circular Tx queues resident in * host DRAM. It steers each frame's Tx command (which contains the frame * data) into one of up to 7 prioritized Tx DMA FIFO channels within the * device. A queue maps to only one (selectable by driver) Tx DMA channel, * but one DMA channel may take input from several queues. * * Tx DMA FIFOs have dedicated purposes. * * For 5000 series and up, they are used differently * (cf. iwl5000_default_queue_to_tx_fifo in iwl-5000.c): * * 0 -- EDCA BK (background) frames, lowest priority * 1 -- EDCA BE (best effort) frames, normal priority * 2 -- EDCA VI (video) frames, higher priority * 3 -- EDCA VO (voice) and management frames, highest priority * 4 -- unused * 5 -- unused * 6 -- unused * 7 -- Commands * * Driver should normally map queues 0-6 to Tx DMA/FIFO channels 0-6. * In addition, driver can map the remaining queues to Tx DMA/FIFO * channels 0-3 to support 11n aggregation via EDCA DMA channels. * * The driver sets up each queue to work in one of two modes: * * 1) Scheduler-Ack, in which the scheduler automatically supports a * block-ack (BA) window of up to 64 TFDs. In this mode, each queue * contains TFDs for a unique combination of Recipient Address (RA) * and Traffic Identifier (TID), that is, traffic of a given * Quality-Of-Service (QOS) priority, destined for a single station. * * In scheduler-ack mode, the scheduler keeps track of the Tx status of * each frame within the BA window, including whether it's been transmitted, * and whether it's been acknowledged by the receiving station. The device * automatically processes block-acks received from the receiving STA, * and reschedules un-acked frames to be retransmitted (successful * Tx completion may end up being out-of-order). * * The driver must maintain the queue's Byte Count table in host DRAM * for this mode. * This mode does not support fragmentation. * * 2) FIFO (a.k.a. non-Scheduler-ACK), in which each TFD is processed in order. * The device may automatically retry Tx, but will retry only one frame * at a time, until receiving ACK from receiving station, or reaching * retry limit and giving up. * * The command queue (#4/#9) must use this mode! * This mode does not require use of the Byte Count table in host DRAM. * * Driver controls scheduler operation via 3 means: * 1) Scheduler registers * 2) Shared scheduler data base in internal SRAM * 3) Shared data in host DRAM * * Initialization: * * When loading, driver should allocate memory for: * 1) 16 TFD circular buffers, each with space for (typically) 256 TFDs. * 2) 16 Byte Count circular buffers in 16 KBytes contiguous memory * (1024 bytes for each queue). * * After receiving "Alive" response from uCode, driver must initialize * the scheduler (especially for queue #4/#9, the command queue, otherwise * the driver can't issue commands!): */ #define IWM_SCD_MEM_LOWER_BOUND (0x0000) /** * Max Tx window size is the max number of contiguous TFDs that the scheduler * can keep track of at one time when creating block-ack chains of frames. * Note that "64" matches the number of ack bits in a block-ack packet. */ #define IWM_SCD_WIN_SIZE 64 #define IWM_SCD_FRAME_LIMIT 64 #define IWM_SCD_TXFIFO_POS_TID (0) #define IWM_SCD_TXFIFO_POS_RA (4) #define IWM_SCD_QUEUE_RA_TID_MAP_RATID_MSK (0x01FF) /* agn SCD */ #define IWM_SCD_QUEUE_STTS_REG_POS_TXF (0) #define IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE (3) #define IWM_SCD_QUEUE_STTS_REG_POS_WSL (4) #define IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN (19) #define IWM_SCD_QUEUE_STTS_REG_MSK (0x017F0000) #define IWM_SCD_QUEUE_CTX_REG1_CREDIT_POS (8) #define IWM_SCD_QUEUE_CTX_REG1_CREDIT_MSK (0x00FFFF00) #define IWM_SCD_QUEUE_CTX_REG1_SUPER_CREDIT_POS (24) #define IWM_SCD_QUEUE_CTX_REG1_SUPER_CREDIT_MSK (0xFF000000) #define IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS (0) #define IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK (0x0000007F) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS (16) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK (0x007F0000) #define IWM_SCD_GP_CTRL_ENABLE_31_QUEUES (1 << 0) #define IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE (1 << 18) /* Context Data */ #define IWM_SCD_CONTEXT_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x600) #define IWM_SCD_CONTEXT_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x6A0) /* Tx status */ #define IWM_SCD_TX_STTS_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x6A0) #define IWM_SCD_TX_STTS_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x7E0) /* Translation Data */ #define IWM_SCD_TRANS_TBL_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x7E0) #define IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x808) #define IWM_SCD_CONTEXT_QUEUE_OFFSET(x)\ (IWM_SCD_CONTEXT_MEM_LOWER_BOUND + ((x) * 8)) #define IWM_SCD_TX_STTS_QUEUE_OFFSET(x)\ (IWM_SCD_TX_STTS_MEM_LOWER_BOUND + ((x) * 16)) #define IWM_SCD_TRANS_TBL_OFFSET_QUEUE(x) \ ((IWM_SCD_TRANS_TBL_MEM_LOWER_BOUND + ((x) * 2)) & 0xfffc) #define IWM_SCD_BASE (IWM_PRPH_BASE + 0xa02c00) #define IWM_SCD_SRAM_BASE_ADDR (IWM_SCD_BASE + 0x0) #define IWM_SCD_DRAM_BASE_ADDR (IWM_SCD_BASE + 0x8) #define IWM_SCD_AIT (IWM_SCD_BASE + 0x0c) #define IWM_SCD_TXFACT (IWM_SCD_BASE + 0x10) #define IWM_SCD_ACTIVE (IWM_SCD_BASE + 0x14) #define IWM_SCD_QUEUECHAIN_SEL (IWM_SCD_BASE + 0xe8) #define IWM_SCD_CHAINEXT_EN (IWM_SCD_BASE + 0x244) #define IWM_SCD_AGGR_SEL (IWM_SCD_BASE + 0x248) #define IWM_SCD_INTERRUPT_MASK (IWM_SCD_BASE + 0x108) #define IWM_SCD_GP_CTRL (IWM_SCD_BASE + 0x1a8) #define IWM_SCD_EN_CTRL (IWM_SCD_BASE + 0x254) static inline unsigned int IWM_SCD_QUEUE_WRPTR(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x18 + chnl * 4; return IWM_SCD_BASE + 0x284 + (chnl - 20) * 4; } static inline unsigned int IWM_SCD_QUEUE_RDPTR(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x68 + chnl * 4; return IWM_SCD_BASE + 0x2B4 + (chnl - 20) * 4; } static inline unsigned int IWM_SCD_QUEUE_STATUS_BITS(unsigned int chnl) { if (chnl < 20) return IWM_SCD_BASE + 0x10c + chnl * 4; return IWM_SCD_BASE + 0x384 + (chnl - 20) * 4; } /*********************** END TX SCHEDULER *************************************/ /* Oscillator clock */ #define IWM_OSC_CLK (0xa04068) #define IWM_OSC_CLK_FORCE_CONTROL (0x8) /* * END iwl-prph.h */ /* * BEGIN iwl-fh.h */ /****************************/ /* Flow Handler Definitions */ /****************************/ /** * This I/O area is directly read/writable by driver (e.g. Linux uses writel()) * Addresses are offsets from device's PCI hardware base address. */ #define IWM_FH_MEM_LOWER_BOUND (0x1000) #define IWM_FH_MEM_UPPER_BOUND (0x2000) /** * Keep-Warm (KW) buffer base address. * * Driver must allocate a 4KByte buffer that is for keeping the * host DRAM powered on (via dummy accesses to DRAM) to maintain low-latency * DRAM access when doing Txing or Rxing. The dummy accesses prevent host * from going into a power-savings mode that would cause higher DRAM latency, * and possible data over/under-runs, before all Tx/Rx is complete. * * Driver loads IWM_FH_KW_MEM_ADDR_REG with the physical address (bits 35:4) * of the buffer, which must be 4K aligned. Once this is set up, the device * automatically invokes keep-warm accesses when normal accesses might not * be sufficient to maintain fast DRAM response. * * Bit fields: * 31-0: Keep-warm buffer physical base address [35:4], must be 4K aligned */ #define IWM_FH_KW_MEM_ADDR_REG (IWM_FH_MEM_LOWER_BOUND + 0x97C) /** * TFD Circular Buffers Base (CBBC) addresses * * Device has 16 base pointer registers, one for each of 16 host-DRAM-resident * circular buffers (CBs/queues) containing Transmit Frame Descriptors (TFDs) * (see struct iwm_tfd_frame). These 16 pointer registers are offset by 0x04 * bytes from one another. Each TFD circular buffer in DRAM must be 256-byte * aligned (address bits 0-7 must be 0). * Later devices have 20 (5000 series) or 30 (higher) queues, but the registers * for them are in different places. * * Bit fields in each pointer register: * 27-0: TFD CB physical base address [35:8], must be 256-byte aligned */ #define IWM_FH_MEM_CBBC_0_15_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9D0) #define IWM_FH_MEM_CBBC_0_15_UPPER_BOUN (IWM_FH_MEM_LOWER_BOUND + 0xA10) #define IWM_FH_MEM_CBBC_16_19_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xBF0) #define IWM_FH_MEM_CBBC_16_19_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_CBBC_20_31_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xB20) #define IWM_FH_MEM_CBBC_20_31_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xB80) /* Find TFD CB base pointer for given queue */ static inline unsigned int IWM_FH_MEM_CBBC_QUEUE(unsigned int chnl) { if (chnl < 16) return IWM_FH_MEM_CBBC_0_15_LOWER_BOUND + 4 * chnl; if (chnl < 20) return IWM_FH_MEM_CBBC_16_19_LOWER_BOUND + 4 * (chnl - 16); return IWM_FH_MEM_CBBC_20_31_LOWER_BOUND + 4 * (chnl - 20); } /** * Rx SRAM Control and Status Registers (RSCSR) * * These registers provide handshake between driver and device for the Rx queue * (this queue handles *all* command responses, notifications, Rx data, etc. * sent from uCode to host driver). Unlike Tx, there is only one Rx * queue, and only one Rx DMA/FIFO channel. Also unlike Tx, which can * concatenate up to 20 DRAM buffers to form a Tx frame, each Receive Buffer * Descriptor (RBD) points to only one Rx Buffer (RB); there is a 1:1 * mapping between RBDs and RBs. * * Driver must allocate host DRAM memory for the following, and set the * physical address of each into device registers: * * 1) Receive Buffer Descriptor (RBD) circular buffer (CB), typically with 256 * entries (although any power of 2, up to 4096, is selectable by driver). * Each entry (1 dword) points to a receive buffer (RB) of consistent size * (typically 4K, although 8K or 16K are also selectable by driver). * Driver sets up RB size and number of RBDs in the CB via Rx config * register IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG. * * Bit fields within one RBD: * 27-0: Receive Buffer physical address bits [35:8], 256-byte aligned * * Driver sets physical address [35:8] of base of RBD circular buffer * into IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG [27:0]. * * 2) Rx status buffer, 8 bytes, in which uCode indicates which Rx Buffers * (RBs) have been filled, via a "write pointer", actually the index of * the RB's corresponding RBD within the circular buffer. Driver sets * physical address [35:4] into IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG [31:0]. * * Bit fields in lower dword of Rx status buffer (upper dword not used * by driver: * 31-12: Not used by driver * 11- 0: Index of last filled Rx buffer descriptor * (device writes, driver reads this value) * * As the driver prepares Receive Buffers (RBs) for device to fill, driver must * enter pointers to these RBs into contiguous RBD circular buffer entries, * and update the device's "write" index register, * IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG. * * This "write" index corresponds to the *next* RBD that the driver will make * available, i.e. one RBD past the tail of the ready-to-fill RBDs within * the circular buffer. This value should initially be 0 (before preparing any * RBs), should be 8 after preparing the first 8 RBs (for example), and must * wrap back to 0 at the end of the circular buffer (but don't wrap before * "read" index has advanced past 1! See below). * NOTE: DEVICE EXPECTS THE WRITE INDEX TO BE INCREMENTED IN MULTIPLES OF 8. * * As the device fills RBs (referenced from contiguous RBDs within the circular * buffer), it updates the Rx status buffer in host DRAM, 2) described above, * to tell the driver the index of the latest filled RBD. The driver must * read this "read" index from DRAM after receiving an Rx interrupt from device * * The driver must also internally keep track of a third index, which is the * next RBD to process. When receiving an Rx interrupt, driver should process * all filled but unprocessed RBs up to, but not including, the RB * corresponding to the "read" index. For example, if "read" index becomes "1", * driver may process the RB pointed to by RBD 0. Depending on volume of * traffic, there may be many RBs to process. * * If read index == write index, device thinks there is no room to put new data. * Due to this, the maximum number of filled RBs is 255, instead of 256. To * be safe, make sure that there is a gap of at least 2 RBDs between "write" * and "read" indexes; that is, make sure that there are no more than 254 * buffers waiting to be filled. */ #define IWM_FH_MEM_RSCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xBC0) #define IWM_FH_MEM_RSCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_RSCSR_CHNL0 (IWM_FH_MEM_RSCSR_LOWER_BOUND) /** * Physical base address of 8-byte Rx Status buffer. * Bit fields: * 31-0: Rx status buffer physical base address [35:4], must 16-byte aligned. */ #define IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG (IWM_FH_MEM_RSCSR_CHNL0) /** * Physical base address of Rx Buffer Descriptor Circular Buffer. * Bit fields: * 27-0: RBD CD physical base address [35:8], must be 256-byte aligned. */ #define IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x004) /** * Rx write pointer (index, really!). * Bit fields: * 11-0: Index of driver's most recent prepared-to-be-filled RBD, + 1. * NOTE: For 256-entry circular buffer, use only bits [7:0]. */ #define IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x008) #define IWM_FH_RSCSR_CHNL0_WPTR (IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG) #define IWM_FW_RSCSR_CHNL0_RXDCB_RDPTR_REG (IWM_FH_MEM_RSCSR_CHNL0 + 0x00c) #define IWM_FH_RSCSR_CHNL0_RDPTR IWM_FW_RSCSR_CHNL0_RXDCB_RDPTR_REG /** * Rx Config/Status Registers (RCSR) * Rx Config Reg for channel 0 (only channel used) * * Driver must initialize IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG as follows for * normal operation (see bit fields). * * Clearing IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG to 0 turns off Rx DMA. * Driver should poll IWM_FH_MEM_RSSR_RX_STATUS_REG for * IWM_FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE (bit 24) before continuing. * * Bit fields: * 31-30: Rx DMA channel enable: '00' off/pause, '01' pause at end of frame, * '10' operate normally * 29-24: reserved * 23-20: # RBDs in circular buffer = 2^value; use "8" for 256 RBDs (normal), * min "5" for 32 RBDs, max "12" for 4096 RBDs. * 19-18: reserved * 17-16: size of each receive buffer; '00' 4K (normal), '01' 8K, * '10' 12K, '11' 16K. * 15-14: reserved * 13-12: IRQ destination; '00' none, '01' host driver (normal operation) * 11- 4: timeout for closing Rx buffer and interrupting host (units 32 usec) * typical value 0x10 (about 1/2 msec) * 3- 0: reserved */ #define IWM_FH_MEM_RCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC00) #define IWM_FH_MEM_RCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xCC0) #define IWM_FH_MEM_RCSR_CHNL0 (IWM_FH_MEM_RCSR_LOWER_BOUND) #define IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG (IWM_FH_MEM_RCSR_CHNL0) #define IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR (IWM_FH_MEM_RCSR_CHNL0 + 0x8) #define IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ (IWM_FH_MEM_RCSR_CHNL0 + 0x10) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RB_TIMEOUT_MSK (0x00000FF0) /* bits 4-11 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_MSK (0x00001000) /* bits 12 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK (0x00008000) /* bit 15 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RB_SIZE_MSK (0x00030000) /* bits 16-17 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_RBDBC_SIZE_MSK (0x00F00000) /* bits 20-23 */ #define IWM_FH_RCSR_CHNL0_RX_CONFIG_DMA_CHNL_EN_MSK (0xC0000000) /* bits 30-31*/ #define IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS (20) #define IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS (4) #define IWM_RX_RB_TIMEOUT (0x11) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_PAUSE_VAL (0x00000000) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_PAUSE_EOF_VAL (0x40000000) #define IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL (0x80000000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K (0x00000000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K (0x00010000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_12K (0x00020000) #define IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_16K (0x00030000) #define IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY (0x00000004) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_NO_INT_VAL (0x00000000) #define IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL (0x00001000) /** * Rx Shared Status Registers (RSSR) * * After stopping Rx DMA channel (writing 0 to * IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG), driver must poll * IWM_FH_MEM_RSSR_RX_STATUS_REG until Rx channel is idle. * * Bit fields: * 24: 1 = Channel 0 is idle * * IWM_FH_MEM_RSSR_SHARED_CTRL_REG and IWM_FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV * contain default values that should not be altered by the driver. */ #define IWM_FH_MEM_RSSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xC40) #define IWM_FH_MEM_RSSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xD00) #define IWM_FH_MEM_RSSR_SHARED_CTRL_REG (IWM_FH_MEM_RSSR_LOWER_BOUND) #define IWM_FH_MEM_RSSR_RX_STATUS_REG (IWM_FH_MEM_RSSR_LOWER_BOUND + 0x004) #define IWM_FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV\ (IWM_FH_MEM_RSSR_LOWER_BOUND + 0x008) #define IWM_FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE (0x01000000) #define IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT 28 /* TFDB Area - TFDs buffer table */ #define IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK (0xFFFFFFFF) #define IWM_FH_TFDIB_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x900) #define IWM_FH_TFDIB_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x958) #define IWM_FH_TFDIB_CTRL0_REG(_chnl) (IWM_FH_TFDIB_LOWER_BOUND + 0x8 * (_chnl)) #define IWM_FH_TFDIB_CTRL1_REG(_chnl) (IWM_FH_TFDIB_LOWER_BOUND + 0x8 * (_chnl) + 0x4) /** * Transmit DMA Channel Control/Status Registers (TCSR) * * Device has one configuration register for each of 8 Tx DMA/FIFO channels * supported in hardware (don't confuse these with the 16 Tx queues in DRAM, * which feed the DMA/FIFO channels); config regs are separated by 0x20 bytes. * * To use a Tx DMA channel, driver must initialize its * IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl) with: * * IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | * IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE_VAL * * All other bits should be 0. * * Bit fields: * 31-30: Tx DMA channel enable: '00' off/pause, '01' pause at end of frame, * '10' operate normally * 29- 4: Reserved, set to "0" * 3: Enable internal DMA requests (1, normal operation), disable (0) * 2- 0: Reserved, set to "0" */ #define IWM_FH_TCSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xD00) #define IWM_FH_TCSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xE60) /* Find Control/Status reg for given Tx DMA/FIFO channel */ #define IWM_FH_TCSR_CHNL_NUM (8) /* TCSR: tx_config register values */ #define IWM_FH_TCSR_CHNL_TX_CONFIG_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl)) #define IWM_FH_TCSR_CHNL_TX_CREDIT_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl) + 0x4) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(_chnl) \ (IWM_FH_TCSR_LOWER_BOUND + 0x20 * (_chnl) + 0x8) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_MSG_MODE_TXF (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_MSG_MODE_DRV (0x00000001) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE (0x00000008) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_NOINT (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD (0x00100000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_IFTFD (0x00200000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_NOINT (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_ENDTFD (0x00400000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_RTC_IFTFD (0x00800000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE (0x00000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE_EOF (0x40000000) #define IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE (0x80000000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_EMPTY (0x00000000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_WAIT (0x00002000) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID (0x00000003) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM (20) #define IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX (12) /** * Tx Shared Status Registers (TSSR) * * After stopping Tx DMA channel (writing 0 to * IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl)), driver must poll * IWM_FH_TSSR_TX_STATUS_REG until selected Tx channel is idle * (channel's buffers empty | no pending requests). * * Bit fields: * 31-24: 1 = Channel buffers empty (channel 7:0) * 23-16: 1 = No pending requests (channel 7:0) */ #define IWM_FH_TSSR_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xEA0) #define IWM_FH_TSSR_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0xEC0) #define IWM_FH_TSSR_TX_STATUS_REG (IWM_FH_TSSR_LOWER_BOUND + 0x010) /** * Bit fields for TSSR(Tx Shared Status & Control) error status register: * 31: Indicates an address error when accessed to internal memory * uCode/driver must write "1" in order to clear this flag * 30: Indicates that Host did not send the expected number of dwords to FH * uCode/driver must write "1" in order to clear this flag * 16-9:Each status bit is for one channel. Indicates that an (Error) ActDMA * command was received from the scheduler while the TRB was already full * with previous command * uCode/driver must write "1" in order to clear this flag * 7-0: Each status bit indicates a channel's TxCredit error. When an error * bit is set, it indicates that the FH has received a full indication * from the RTC TxFIFO and the current value of the TxCredit counter was * not equal to zero. This mean that the credit mechanism was not * synchronized to the TxFIFO status * uCode/driver must write "1" in order to clear this flag */ #define IWM_FH_TSSR_TX_ERROR_REG (IWM_FH_TSSR_LOWER_BOUND + 0x018) #define IWM_FH_TSSR_TX_MSG_CONFIG_REG (IWM_FH_TSSR_LOWER_BOUND + 0x008) #define IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(_chnl) ((1 << (_chnl)) << 16) /* Tx service channels */ #define IWM_FH_SRVC_CHNL (9) #define IWM_FH_SRVC_LOWER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9C8) #define IWM_FH_SRVC_UPPER_BOUND (IWM_FH_MEM_LOWER_BOUND + 0x9D0) #define IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(_chnl) \ (IWM_FH_SRVC_LOWER_BOUND + ((_chnl) - 9) * 0x4) #define IWM_FH_TX_CHICKEN_BITS_REG (IWM_FH_MEM_LOWER_BOUND + 0xE98) #define IWM_FH_TX_TRB_REG(_chan) (IWM_FH_MEM_LOWER_BOUND + 0x958 + \ (_chan) * 4) /* Instruct FH to increment the retry count of a packet when * it is brought from the memory to TX-FIFO */ #define IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN (0x00000002) #define IWM_RX_QUEUE_SIZE 256 #define IWM_RX_QUEUE_MASK 255 #define IWM_RX_QUEUE_SIZE_LOG 8 /* * RX related structures and functions */ #define IWM_RX_FREE_BUFFERS 64 #define IWM_RX_LOW_WATERMARK 8 /** * struct iwm_rb_status - reseve buffer status * host memory mapped FH registers * @closed_rb_num [0:11] - Indicates the index of the RB which was closed * @closed_fr_num [0:11] - Indicates the index of the RX Frame which was closed * @finished_rb_num [0:11] - Indicates the index of the current RB * in which the last frame was written to * @finished_fr_num [0:11] - Indicates the index of the RX Frame * which was transferred */ struct iwm_rb_status { uint16_t closed_rb_num; uint16_t closed_fr_num; uint16_t finished_rb_num; uint16_t finished_fr_nam; uint32_t unused; } __packed; #define IWM_TFD_QUEUE_SIZE_MAX (256) #define IWM_TFD_QUEUE_SIZE_BC_DUP (64) #define IWM_TFD_QUEUE_BC_SIZE (IWM_TFD_QUEUE_SIZE_MAX + \ IWM_TFD_QUEUE_SIZE_BC_DUP) #define IWM_TX_DMA_MASK DMA_BIT_MASK(36) #define IWM_NUM_OF_TBS 20 static inline uint8_t iwm_get_dma_hi_addr(bus_addr_t addr) { return (sizeof(addr) > sizeof(uint32_t) ? (addr >> 16) >> 16 : 0) & 0xF; } /** * struct iwm_tfd_tb transmit buffer descriptor within transmit frame descriptor * * This structure contains dma address and length of transmission address * * @lo: low [31:0] portion of the dma address of TX buffer * every even is unaligned on 16 bit boundary * @hi_n_len 0-3 [35:32] portion of dma * 4-15 length of the tx buffer */ struct iwm_tfd_tb { uint32_t lo; uint16_t hi_n_len; } __packed; /** * struct iwm_tfd * * Transmit Frame Descriptor (TFD) * * @ __reserved1[3] reserved * @ num_tbs 0-4 number of active tbs * 5 reserved * 6-7 padding (not used) * @ tbs[20] transmit frame buffer descriptors * @ __pad padding * * Each Tx queue uses a circular buffer of 256 TFDs stored in host DRAM. * Both driver and device share these circular buffers, each of which must be * contiguous 256 TFDs x 128 bytes-per-TFD = 32 KBytes * * Driver must indicate the physical address of the base of each * circular buffer via the IWM_FH_MEM_CBBC_QUEUE registers. * * Each TFD contains pointer/size information for up to 20 data buffers * in host DRAM. These buffers collectively contain the (one) frame described * by the TFD. Each buffer must be a single contiguous block of memory within * itself, but buffers may be scattered in host DRAM. Each buffer has max size * of (4K - 4). The concatenates all of a TFD's buffers into a single * Tx frame, up to 8 KBytes in size. * * A maximum of 255 (not 256!) TFDs may be on a queue waiting for Tx. */ struct iwm_tfd { uint8_t __reserved1[3]; uint8_t num_tbs; struct iwm_tfd_tb tbs[IWM_NUM_OF_TBS]; uint32_t __pad; } __packed; /* Keep Warm Size */ #define IWM_KW_SIZE 0x1000 /* 4k */ /* Fixed (non-configurable) rx data from phy */ /** * struct iwm_agn_schedq_bc_tbl scheduler byte count table * base physical address provided by IWM_SCD_DRAM_BASE_ADDR * @tfd_offset 0-12 - tx command byte count * 12-16 - station index */ struct iwm_agn_scd_bc_tbl { uint16_t tfd_offset[IWM_TFD_QUEUE_BC_SIZE]; } __packed; /* * END iwl-fh.h */ /* * BEGIN mvm/fw-api.h */ /* Maximum number of Tx queues. */ #define IWM_MVM_MAX_QUEUES 31 /* Tx queue numbers */ enum { IWM_MVM_OFFCHANNEL_QUEUE = 8, IWM_MVM_CMD_QUEUE = 9, IWM_MVM_AUX_QUEUE = 15, }; enum iwm_mvm_tx_fifo { IWM_MVM_TX_FIFO_BK = 0, IWM_MVM_TX_FIFO_BE, IWM_MVM_TX_FIFO_VI, IWM_MVM_TX_FIFO_VO, IWM_MVM_TX_FIFO_MCAST = 5, IWM_MVM_TX_FIFO_CMD = 7, }; #define IWM_MVM_STATION_COUNT 16 /* commands */ enum { IWM_MVM_ALIVE = 0x1, IWM_REPLY_ERROR = 0x2, IWM_INIT_COMPLETE_NOTIF = 0x4, /* PHY context commands */ IWM_PHY_CONTEXT_CMD = 0x8, IWM_DBG_CFG = 0x9, /* UMAC scan commands */ IWM_SCAN_ITERATION_COMPLETE_UMAC = 0xb5, IWM_SCAN_CFG_CMD = 0xc, IWM_SCAN_REQ_UMAC = 0xd, IWM_SCAN_ABORT_UMAC = 0xe, IWM_SCAN_COMPLETE_UMAC = 0xf, /* station table */ IWM_ADD_STA_KEY = 0x17, IWM_ADD_STA = 0x18, IWM_REMOVE_STA = 0x19, /* TX */ IWM_TX_CMD = 0x1c, IWM_TXPATH_FLUSH = 0x1e, IWM_MGMT_MCAST_KEY = 0x1f, /* scheduler config */ IWM_SCD_QUEUE_CFG = 0x1d, /* global key */ IWM_WEP_KEY = 0x20, /* MAC and Binding commands */ IWM_MAC_CONTEXT_CMD = 0x28, IWM_TIME_EVENT_CMD = 0x29, /* both CMD and response */ IWM_TIME_EVENT_NOTIFICATION = 0x2a, IWM_BINDING_CONTEXT_CMD = 0x2b, IWM_TIME_QUOTA_CMD = 0x2c, IWM_NON_QOS_TX_COUNTER_CMD = 0x2d, IWM_LQ_CMD = 0x4e, /* paging block to FW cpu2 */ IWM_FW_PAGING_BLOCK_CMD = 0x4f, /* Scan offload */ IWM_SCAN_OFFLOAD_REQUEST_CMD = 0x51, IWM_SCAN_OFFLOAD_ABORT_CMD = 0x52, IWM_HOT_SPOT_CMD = 0x53, IWM_SCAN_OFFLOAD_COMPLETE = 0x6d, IWM_SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6e, IWM_SCAN_OFFLOAD_CONFIG_CMD = 0x6f, IWM_MATCH_FOUND_NOTIFICATION = 0xd9, IWM_SCAN_ITERATION_COMPLETE = 0xe7, /* Phy */ IWM_PHY_CONFIGURATION_CMD = 0x6a, IWM_CALIB_RES_NOTIF_PHY_DB = 0x6b, IWM_PHY_DB_CMD = 0x6c, /* Power - legacy power table command */ IWM_POWER_TABLE_CMD = 0x77, IWM_PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78, IWM_LTR_CONFIG = 0xee, /* Thermal Throttling*/ IWM_REPLY_THERMAL_MNG_BACKOFF = 0x7e, /* NVM */ IWM_NVM_ACCESS_CMD = 0x88, IWM_SET_CALIB_DEFAULT_CMD = 0x8e, IWM_BEACON_NOTIFICATION = 0x90, IWM_BEACON_TEMPLATE_CMD = 0x91, IWM_TX_ANT_CONFIGURATION_CMD = 0x98, IWM_BT_CONFIG = 0x9b, IWM_STATISTICS_NOTIFICATION = 0x9d, IWM_REDUCE_TX_POWER_CMD = 0x9f, /* RF-KILL commands and notifications */ IWM_CARD_STATE_CMD = 0xa0, IWM_CARD_STATE_NOTIFICATION = 0xa1, IWM_MISSED_BEACONS_NOTIFICATION = 0xa2, IWM_MFUART_LOAD_NOTIFICATION = 0xb1, /* Power - new power table command */ IWM_MAC_PM_POWER_TABLE = 0xa9, IWM_REPLY_RX_PHY_CMD = 0xc0, IWM_REPLY_RX_MPDU_CMD = 0xc1, IWM_BA_NOTIF = 0xc5, /* Location Aware Regulatory */ IWM_MCC_UPDATE_CMD = 0xc8, IWM_MCC_CHUB_UPDATE_CMD = 0xc9, /* BT Coex */ IWM_BT_COEX_PRIO_TABLE = 0xcc, IWM_BT_COEX_PROT_ENV = 0xcd, IWM_BT_PROFILE_NOTIFICATION = 0xce, IWM_BT_COEX_CI = 0x5d, IWM_REPLY_SF_CFG_CMD = 0xd1, IWM_REPLY_BEACON_FILTERING_CMD = 0xd2, /* DTS measurements */ IWM_CMD_DTS_MEASUREMENT_TRIGGER = 0xdc, IWM_DTS_MEASUREMENT_NOTIFICATION = 0xdd, IWM_REPLY_DEBUG_CMD = 0xf0, IWM_DEBUG_LOG_MSG = 0xf7, IWM_MCAST_FILTER_CMD = 0xd0, /* D3 commands/notifications */ IWM_D3_CONFIG_CMD = 0xd3, IWM_PROT_OFFLOAD_CONFIG_CMD = 0xd4, IWM_OFFLOADS_QUERY_CMD = 0xd5, IWM_REMOTE_WAKE_CONFIG_CMD = 0xd6, /* for WoWLAN in particular */ IWM_WOWLAN_PATTERNS = 0xe0, IWM_WOWLAN_CONFIGURATION = 0xe1, IWM_WOWLAN_TSC_RSC_PARAM = 0xe2, IWM_WOWLAN_TKIP_PARAM = 0xe3, IWM_WOWLAN_KEK_KCK_MATERIAL = 0xe4, IWM_WOWLAN_GET_STATUSES = 0xe5, IWM_WOWLAN_TX_POWER_PER_DB = 0xe6, /* and for NetDetect */ IWM_NET_DETECT_CONFIG_CMD = 0x54, IWM_NET_DETECT_PROFILES_QUERY_CMD = 0x56, IWM_NET_DETECT_PROFILES_CMD = 0x57, IWM_NET_DETECT_HOTSPOTS_CMD = 0x58, IWM_NET_DETECT_HOTSPOTS_QUERY_CMD = 0x59, }; enum iwm_phy_ops_subcmd_ids { IWM_CMD_DTS_MEASUREMENT_TRIGGER_WIDE = 0x0, IWM_CTDP_CONFIG_CMD = 0x03, IWM_TEMP_REPORTING_THRESHOLDS_CMD = 0x04, IWM_CT_KILL_NOTIFICATION = 0xFE, IWM_DTS_MEASUREMENT_NOTIF_WIDE = 0xFF, }; /* command groups */ enum { IWM_LEGACY_GROUP = 0x0, IWM_LONG_GROUP = 0x1, IWM_SYSTEM_GROUP = 0x2, IWM_MAC_CONF_GROUP = 0x3, IWM_PHY_OPS_GROUP = 0x4, IWM_DATA_PATH_GROUP = 0x5, IWM_PROT_OFFLOAD_GROUP = 0xb, }; /** * struct iwm_cmd_response - generic response struct for most commands * @status: status of the command asked, changes for each one */ struct iwm_cmd_response { uint32_t status; }; /* * struct iwm_tx_ant_cfg_cmd * @valid: valid antenna configuration */ struct iwm_tx_ant_cfg_cmd { uint32_t valid; } __packed; /** * struct iwm_reduce_tx_power_cmd - TX power reduction command * IWM_REDUCE_TX_POWER_CMD = 0x9f * @flags: (reserved for future implementation) * @mac_context_id: id of the mac ctx for which we are reducing TX power. * @pwr_restriction: TX power restriction in dBms. */ struct iwm_reduce_tx_power_cmd { uint8_t flags; uint8_t mac_context_id; uint16_t pwr_restriction; } __packed; /* IWM_TX_REDUCED_POWER_API_S_VER_1 */ enum iwm_dev_tx_power_cmd_mode { IWM_TX_POWER_MODE_SET_MAC = 0, IWM_TX_POWER_MODE_SET_DEVICE = 1, IWM_TX_POWER_MODE_SET_CHAINS = 2, IWM_TX_POWER_MODE_SET_ACK = 3, }; /* TX_POWER_REDUCED_FLAGS_TYPE_API_E_VER_4 */; #define IWM_NUM_CHAIN_LIMITS 2 #define IWM_NUM_SUB_BANDS 5 /** * struct iwm_dev_tx_power_cmd - TX power reduction command * @set_mode: see &enum iwl_dev_tx_power_cmd_mode * @mac_context_id: id of the mac ctx for which we are reducing TX power. * @pwr_restriction: TX power restriction in 1/8 dBms. * @dev_24: device TX power restriction in 1/8 dBms * @dev_52_low: device TX power restriction upper band - low * @dev_52_high: device TX power restriction upper band - high * @per_chain_restriction: per chain restrictions */ struct iwm_dev_tx_power_cmd_v3 { uint32_t set_mode; uint32_t mac_context_id; uint16_t pwr_restriction; uint16_t dev_24; uint16_t dev_52_low; uint16_t dev_52_high; uint16_t per_chain_restriction[IWM_NUM_CHAIN_LIMITS][IWM_NUM_SUB_BANDS]; } __packed; /* TX_REDUCED_POWER_API_S_VER_3 */ #define IWM_DEV_MAX_TX_POWER 0x7FFF /** * struct iwm_dev_tx_power_cmd - TX power reduction command * @v3: version 3 of the command, embedded here for easier software handling * @enable_ack_reduction: enable or disable close range ack TX power * reduction. */ struct iwm_dev_tx_power_cmd { /* v4 is just an extension of v3 - keep this here */ struct iwm_dev_tx_power_cmd_v3 v3; uint8_t enable_ack_reduction; uint8_t reserved[3]; } __packed; /* TX_REDUCED_POWER_API_S_VER_4 */ /* * Calibration control struct. * Sent as part of the phy configuration command. * @flow_trigger: bitmap for which calibrations to perform according to * flow triggers. * @event_trigger: bitmap for which calibrations to perform according to * event triggers. */ struct iwm_calib_ctrl { uint32_t flow_trigger; uint32_t event_trigger; } __packed; /* This enum defines the bitmap of various calibrations to enable in both * init ucode and runtime ucode through IWM_CALIBRATION_CFG_CMD. */ enum iwm_calib_cfg { IWM_CALIB_CFG_XTAL_IDX = (1 << 0), IWM_CALIB_CFG_TEMPERATURE_IDX = (1 << 1), IWM_CALIB_CFG_VOLTAGE_READ_IDX = (1 << 2), IWM_CALIB_CFG_PAPD_IDX = (1 << 3), IWM_CALIB_CFG_TX_PWR_IDX = (1 << 4), IWM_CALIB_CFG_DC_IDX = (1 << 5), IWM_CALIB_CFG_BB_FILTER_IDX = (1 << 6), IWM_CALIB_CFG_LO_LEAKAGE_IDX = (1 << 7), IWM_CALIB_CFG_TX_IQ_IDX = (1 << 8), IWM_CALIB_CFG_TX_IQ_SKEW_IDX = (1 << 9), IWM_CALIB_CFG_RX_IQ_IDX = (1 << 10), IWM_CALIB_CFG_RX_IQ_SKEW_IDX = (1 << 11), IWM_CALIB_CFG_SENSITIVITY_IDX = (1 << 12), IWM_CALIB_CFG_CHAIN_NOISE_IDX = (1 << 13), IWM_CALIB_CFG_DISCONNECTED_ANT_IDX = (1 << 14), IWM_CALIB_CFG_ANT_COUPLING_IDX = (1 << 15), IWM_CALIB_CFG_DAC_IDX = (1 << 16), IWM_CALIB_CFG_ABS_IDX = (1 << 17), IWM_CALIB_CFG_AGC_IDX = (1 << 18), }; /* * Phy configuration command. */ struct iwm_phy_cfg_cmd { uint32_t phy_cfg; struct iwm_calib_ctrl calib_control; } __packed; #define IWM_PHY_CFG_RADIO_TYPE ((1 << 0) | (1 << 1)) #define IWM_PHY_CFG_RADIO_STEP ((1 << 2) | (1 << 3)) #define IWM_PHY_CFG_RADIO_DASH ((1 << 4) | (1 << 5)) #define IWM_PHY_CFG_PRODUCT_NUMBER ((1 << 6) | (1 << 7)) #define IWM_PHY_CFG_TX_CHAIN_A (1 << 8) #define IWM_PHY_CFG_TX_CHAIN_B (1 << 9) #define IWM_PHY_CFG_TX_CHAIN_C (1 << 10) #define IWM_PHY_CFG_RX_CHAIN_A (1 << 12) #define IWM_PHY_CFG_RX_CHAIN_B (1 << 13) #define IWM_PHY_CFG_RX_CHAIN_C (1 << 14) /* Target of the IWM_NVM_ACCESS_CMD */ enum { IWM_NVM_ACCESS_TARGET_CACHE = 0, IWM_NVM_ACCESS_TARGET_OTP = 1, IWM_NVM_ACCESS_TARGET_EEPROM = 2, }; /* Section types for IWM_NVM_ACCESS_CMD */ enum { IWM_NVM_SECTION_TYPE_SW = 1, IWM_NVM_SECTION_TYPE_REGULATORY = 3, IWM_NVM_SECTION_TYPE_CALIBRATION = 4, IWM_NVM_SECTION_TYPE_PRODUCTION = 5, + IWM_NVM_SECTION_TYPE_REGULATORY_SDP = 8, IWM_NVM_SECTION_TYPE_MAC_OVERRIDE = 11, IWM_NVM_SECTION_TYPE_PHY_SKU = 12, IWM_NVM_MAX_NUM_SECTIONS = 13, }; /** * struct iwm_nvm_access_cmd_ver2 - Request the device to send an NVM section * @op_code: 0 - read, 1 - write * @target: IWM_NVM_ACCESS_TARGET_* * @type: IWM_NVM_SECTION_TYPE_* * @offset: offset in bytes into the section * @length: in bytes, to read/write * @data: if write operation, the data to write. On read its empty */ struct iwm_nvm_access_cmd { uint8_t op_code; uint8_t target; uint16_t type; uint16_t offset; uint16_t length; uint8_t data[]; } __packed; /* IWM_NVM_ACCESS_CMD_API_S_VER_2 */ #define IWM_NUM_OF_FW_PAGING_BLOCKS 33 /* 32 for data and 1 block for CSS */ /* * struct iwm_fw_paging_cmd - paging layout * * (IWM_FW_PAGING_BLOCK_CMD = 0x4f) * * Send to FW the paging layout in the driver. * * @flags: various flags for the command * @block_size: the block size in powers of 2 * @block_num: number of blocks specified in the command. * @device_phy_addr: virtual addresses from device side */ struct iwm_fw_paging_cmd { uint32_t flags; uint32_t block_size; uint32_t block_num; uint32_t device_phy_addr[IWM_NUM_OF_FW_PAGING_BLOCKS]; } __packed; /* IWM_FW_PAGING_BLOCK_CMD_API_S_VER_1 */ /* * Fw items ID's * * @IWM_FW_ITEM_ID_PAGING: Address of the pages that the FW will upload * download */ enum iwm_fw_item_id { IWM_FW_ITEM_ID_PAGING = 3, }; /* * struct iwm_fw_get_item_cmd - get an item from the fw */ struct iwm_fw_get_item_cmd { uint32_t item_id; } __packed; /* IWM_FW_GET_ITEM_CMD_API_S_VER_1 */ /** * struct iwm_nvm_access_resp_ver2 - response to IWM_NVM_ACCESS_CMD * @offset: offset in bytes into the section * @length: in bytes, either how much was written or read * @type: IWM_NVM_SECTION_TYPE_* * @status: 0 for success, fail otherwise * @data: if read operation, the data returned. Empty on write. */ struct iwm_nvm_access_resp { uint16_t offset; uint16_t length; uint16_t type; uint16_t status; uint8_t data[]; } __packed; /* IWM_NVM_ACCESS_CMD_RESP_API_S_VER_2 */ /* IWM_MVM_ALIVE 0x1 */ /* alive response is_valid values */ #define IWM_ALIVE_RESP_UCODE_OK (1 << 0) #define IWM_ALIVE_RESP_RFKILL (1 << 1) /* alive response ver_type values */ enum { IWM_FW_TYPE_HW = 0, IWM_FW_TYPE_PROT = 1, IWM_FW_TYPE_AP = 2, IWM_FW_TYPE_WOWLAN = 3, IWM_FW_TYPE_TIMING = 4, IWM_FW_TYPE_WIPAN = 5 }; /* alive response ver_subtype values */ enum { IWM_FW_SUBTYPE_FULL_FEATURE = 0, IWM_FW_SUBTYPE_BOOTSRAP = 1, /* Not valid */ IWM_FW_SUBTYPE_REDUCED = 2, IWM_FW_SUBTYPE_ALIVE_ONLY = 3, IWM_FW_SUBTYPE_WOWLAN = 4, IWM_FW_SUBTYPE_AP_SUBTYPE = 5, IWM_FW_SUBTYPE_WIPAN = 6, IWM_FW_SUBTYPE_INITIALIZE = 9 }; #define IWM_ALIVE_STATUS_ERR 0xDEAD #define IWM_ALIVE_STATUS_OK 0xCAFE #define IWM_ALIVE_FLG_RFKILL (1 << 0) struct iwm_lmac_alive { uint32_t ucode_major; uint32_t ucode_minor; uint8_t ver_subtype; uint8_t ver_type; uint8_t mac; uint8_t opt; uint32_t timestamp; uint32_t error_event_table_ptr; /* SRAM address for error log */ uint32_t log_event_table_ptr; /* SRAM address for LMAC event log */ uint32_t cpu_register_ptr; uint32_t dbgm_config_ptr; uint32_t alive_counter_ptr; uint32_t scd_base_ptr; /* SRAM address for SCD */ uint32_t st_fwrd_addr; /* pointer to Store and forward */ uint32_t st_fwrd_size; } __packed; /* UCODE_ALIVE_NTFY_API_S_VER_3 */ struct iwm_umac_alive { uint32_t umac_major; /* UMAC version: major */ uint32_t umac_minor; /* UMAC version: minor */ uint32_t error_info_addr; /* SRAM address for UMAC error log */ uint32_t dbg_print_buff_addr; } __packed; /* UMAC_ALIVE_DATA_API_S_VER_2 */ struct iwm_mvm_alive_resp_v3 { uint16_t status; uint16_t flags; struct iwm_lmac_alive lmac_data; struct iwm_umac_alive umac_data; } __packed; /* ALIVE_RES_API_S_VER_3 */ struct iwm_mvm_alive_resp { uint16_t status; uint16_t flags; struct iwm_lmac_alive lmac_data[2]; struct iwm_umac_alive umac_data; } __packed; /* ALIVE_RES_API_S_VER_4 */ /* Error response/notification */ enum { IWM_FW_ERR_UNKNOWN_CMD = 0x0, IWM_FW_ERR_INVALID_CMD_PARAM = 0x1, IWM_FW_ERR_SERVICE = 0x2, IWM_FW_ERR_ARC_MEMORY = 0x3, IWM_FW_ERR_ARC_CODE = 0x4, IWM_FW_ERR_WATCH_DOG = 0x5, IWM_FW_ERR_WEP_GRP_KEY_INDX = 0x10, IWM_FW_ERR_WEP_KEY_SIZE = 0x11, IWM_FW_ERR_OBSOLETE_FUNC = 0x12, IWM_FW_ERR_UNEXPECTED = 0xFE, IWM_FW_ERR_FATAL = 0xFF }; /** * struct iwm_error_resp - FW error indication * ( IWM_REPLY_ERROR = 0x2 ) * @error_type: one of IWM_FW_ERR_* * @cmd_id: the command ID for which the error occurred * @bad_cmd_seq_num: sequence number of the erroneous command * @error_service: which service created the error, applicable only if * error_type = 2, otherwise 0 * @timestamp: TSF in usecs. */ struct iwm_error_resp { uint32_t error_type; uint8_t cmd_id; uint8_t reserved1; uint16_t bad_cmd_seq_num; uint32_t error_service; uint64_t timestamp; } __packed; /* Common PHY, MAC and Bindings definitions */ #define IWM_MAX_MACS_IN_BINDING (3) #define IWM_MAX_BINDINGS (4) #define IWM_AUX_BINDING_INDEX (3) #define IWM_MAX_PHYS (4) /* Used to extract ID and color from the context dword */ #define IWM_FW_CTXT_ID_POS (0) #define IWM_FW_CTXT_ID_MSK (0xff << IWM_FW_CTXT_ID_POS) #define IWM_FW_CTXT_COLOR_POS (8) #define IWM_FW_CTXT_COLOR_MSK (0xff << IWM_FW_CTXT_COLOR_POS) #define IWM_FW_CTXT_INVALID (0xffffffff) #define IWM_FW_CMD_ID_AND_COLOR(_id, _color) ((_id << IWM_FW_CTXT_ID_POS) |\ (_color << IWM_FW_CTXT_COLOR_POS)) /* Possible actions on PHYs, MACs and Bindings */ enum { IWM_FW_CTXT_ACTION_STUB = 0, IWM_FW_CTXT_ACTION_ADD, IWM_FW_CTXT_ACTION_MODIFY, IWM_FW_CTXT_ACTION_REMOVE, IWM_FW_CTXT_ACTION_NUM }; /* COMMON_CONTEXT_ACTION_API_E_VER_1 */ /* Time Events */ /* Time Event types, according to MAC type */ enum iwm_time_event_type { /* BSS Station Events */ IWM_TE_BSS_STA_AGGRESSIVE_ASSOC, IWM_TE_BSS_STA_ASSOC, IWM_TE_BSS_EAP_DHCP_PROT, IWM_TE_BSS_QUIET_PERIOD, /* P2P Device Events */ IWM_TE_P2P_DEVICE_DISCOVERABLE, IWM_TE_P2P_DEVICE_LISTEN, IWM_TE_P2P_DEVICE_ACTION_SCAN, IWM_TE_P2P_DEVICE_FULL_SCAN, /* P2P Client Events */ IWM_TE_P2P_CLIENT_AGGRESSIVE_ASSOC, IWM_TE_P2P_CLIENT_ASSOC, IWM_TE_P2P_CLIENT_QUIET_PERIOD, /* P2P GO Events */ IWM_TE_P2P_GO_ASSOC_PROT, IWM_TE_P2P_GO_REPETITIVE_NOA, IWM_TE_P2P_GO_CT_WINDOW, /* WiDi Sync Events */ IWM_TE_WIDI_TX_SYNC, IWM_TE_MAX }; /* IWM_MAC_EVENT_TYPE_API_E_VER_1 */ /* Time event - defines for command API v1 */ /* * @IWM_TE_V1_FRAG_NONE: fragmentation of the time event is NOT allowed. * @IWM_TE_V1_FRAG_SINGLE: fragmentation of the time event is allowed, but only * the first fragment is scheduled. * @IWM_TE_V1_FRAG_DUAL: fragmentation of the time event is allowed, but only * the first 2 fragments are scheduled. * @IWM_TE_V1_FRAG_ENDLESS: fragmentation of the time event is allowed, and any * number of fragments are valid. * * Other than the constant defined above, specifying a fragmentation value 'x' * means that the event can be fragmented but only the first 'x' will be * scheduled. */ enum { IWM_TE_V1_FRAG_NONE = 0, IWM_TE_V1_FRAG_SINGLE = 1, IWM_TE_V1_FRAG_DUAL = 2, IWM_TE_V1_FRAG_ENDLESS = 0xffffffff }; /* If a Time Event can be fragmented, this is the max number of fragments */ #define IWM_TE_V1_FRAG_MAX_MSK 0x0fffffff /* Repeat the time event endlessly (until removed) */ #define IWM_TE_V1_REPEAT_ENDLESS 0xffffffff /* If a Time Event has bounded repetitions, this is the maximal value */ #define IWM_TE_V1_REPEAT_MAX_MSK_V1 0x0fffffff /* Time Event dependencies: none, on another TE, or in a specific time */ enum { IWM_TE_V1_INDEPENDENT = 0, IWM_TE_V1_DEP_OTHER = (1 << 0), IWM_TE_V1_DEP_TSF = (1 << 1), IWM_TE_V1_EVENT_SOCIOPATHIC = (1 << 2), }; /* IWM_MAC_EVENT_DEPENDENCY_POLICY_API_E_VER_2 */ /* * @IWM_TE_V1_NOTIF_NONE: no notifications * @IWM_TE_V1_NOTIF_HOST_EVENT_START: request/receive notification on event start * @IWM_TE_V1_NOTIF_HOST_EVENT_END:request/receive notification on event end * @IWM_TE_V1_NOTIF_INTERNAL_EVENT_START: internal FW use * @IWM_TE_V1_NOTIF_INTERNAL_EVENT_END: internal FW use. * @IWM_TE_V1_NOTIF_HOST_FRAG_START: request/receive notification on frag start * @IWM_TE_V1_NOTIF_HOST_FRAG_END:request/receive notification on frag end * @IWM_TE_V1_NOTIF_INTERNAL_FRAG_START: internal FW use. * @IWM_TE_V1_NOTIF_INTERNAL_FRAG_END: internal FW use. * * Supported Time event notifications configuration. * A notification (both event and fragment) includes a status indicating weather * the FW was able to schedule the event or not. For fragment start/end * notification the status is always success. There is no start/end fragment * notification for monolithic events. */ enum { IWM_TE_V1_NOTIF_NONE = 0, IWM_TE_V1_NOTIF_HOST_EVENT_START = (1 << 0), IWM_TE_V1_NOTIF_HOST_EVENT_END = (1 << 1), IWM_TE_V1_NOTIF_INTERNAL_EVENT_START = (1 << 2), IWM_TE_V1_NOTIF_INTERNAL_EVENT_END = (1 << 3), IWM_TE_V1_NOTIF_HOST_FRAG_START = (1 << 4), IWM_TE_V1_NOTIF_HOST_FRAG_END = (1 << 5), IWM_TE_V1_NOTIF_INTERNAL_FRAG_START = (1 << 6), IWM_TE_V1_NOTIF_INTERNAL_FRAG_END = (1 << 7), IWM_T2_V2_START_IMMEDIATELY = (1 << 11), }; /* IWM_MAC_EVENT_ACTION_API_E_VER_2 */ /* Time event - defines for command API */ /* * @IWM_TE_V2_FRAG_NONE: fragmentation of the time event is NOT allowed. * @IWM_TE_V2_FRAG_SINGLE: fragmentation of the time event is allowed, but only * the first fragment is scheduled. * @IWM_TE_V2_FRAG_DUAL: fragmentation of the time event is allowed, but only * the first 2 fragments are scheduled. * @IWM_TE_V2_FRAG_ENDLESS: fragmentation of the time event is allowed, and any * number of fragments are valid. * * Other than the constant defined above, specifying a fragmentation value 'x' * means that the event can be fragmented but only the first 'x' will be * scheduled. */ enum { IWM_TE_V2_FRAG_NONE = 0, IWM_TE_V2_FRAG_SINGLE = 1, IWM_TE_V2_FRAG_DUAL = 2, IWM_TE_V2_FRAG_MAX = 0xfe, IWM_TE_V2_FRAG_ENDLESS = 0xff }; /* Repeat the time event endlessly (until removed) */ #define IWM_TE_V2_REPEAT_ENDLESS 0xff /* If a Time Event has bounded repetitions, this is the maximal value */ #define IWM_TE_V2_REPEAT_MAX 0xfe #define IWM_TE_V2_PLACEMENT_POS 12 #define IWM_TE_V2_ABSENCE_POS 15 /* Time event policy values * A notification (both event and fragment) includes a status indicating weather * the FW was able to schedule the event or not. For fragment start/end * notification the status is always success. There is no start/end fragment * notification for monolithic events. * * @IWM_TE_V2_DEFAULT_POLICY: independent, social, present, unoticable * @IWM_TE_V2_NOTIF_HOST_EVENT_START: request/receive notification on event start * @IWM_TE_V2_NOTIF_HOST_EVENT_END:request/receive notification on event end * @IWM_TE_V2_NOTIF_INTERNAL_EVENT_START: internal FW use * @IWM_TE_V2_NOTIF_INTERNAL_EVENT_END: internal FW use. * @IWM_TE_V2_NOTIF_HOST_FRAG_START: request/receive notification on frag start * @IWM_TE_V2_NOTIF_HOST_FRAG_END:request/receive notification on frag end * @IWM_TE_V2_NOTIF_INTERNAL_FRAG_START: internal FW use. * @IWM_TE_V2_NOTIF_INTERNAL_FRAG_END: internal FW use. * @IWM_TE_V2_DEP_OTHER: depends on another time event * @IWM_TE_V2_DEP_TSF: depends on a specific time * @IWM_TE_V2_EVENT_SOCIOPATHIC: can't co-exist with other events of tha same MAC * @IWM_TE_V2_ABSENCE: are we present or absent during the Time Event. */ enum { IWM_TE_V2_DEFAULT_POLICY = 0x0, /* notifications (event start/stop, fragment start/stop) */ IWM_TE_V2_NOTIF_HOST_EVENT_START = (1 << 0), IWM_TE_V2_NOTIF_HOST_EVENT_END = (1 << 1), IWM_TE_V2_NOTIF_INTERNAL_EVENT_START = (1 << 2), IWM_TE_V2_NOTIF_INTERNAL_EVENT_END = (1 << 3), IWM_TE_V2_NOTIF_HOST_FRAG_START = (1 << 4), IWM_TE_V2_NOTIF_HOST_FRAG_END = (1 << 5), IWM_TE_V2_NOTIF_INTERNAL_FRAG_START = (1 << 6), IWM_TE_V2_NOTIF_INTERNAL_FRAG_END = (1 << 7), IWM_TE_V2_NOTIF_MSK = 0xff, /* placement characteristics */ IWM_TE_V2_DEP_OTHER = (1 << IWM_TE_V2_PLACEMENT_POS), IWM_TE_V2_DEP_TSF = (1 << (IWM_TE_V2_PLACEMENT_POS + 1)), IWM_TE_V2_EVENT_SOCIOPATHIC = (1 << (IWM_TE_V2_PLACEMENT_POS + 2)), /* are we present or absent during the Time Event. */ IWM_TE_V2_ABSENCE = (1 << IWM_TE_V2_ABSENCE_POS), }; /** * struct iwm_time_event_cmd_api - configuring Time Events * with struct IWM_MAC_TIME_EVENT_DATA_API_S_VER_2 (see also * with version 1. determined by IWM_UCODE_TLV_FLAGS) * ( IWM_TIME_EVENT_CMD = 0x29 ) * @id_and_color: ID and color of the relevant MAC * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @id: this field has two meanings, depending on the action: * If the action is ADD, then it means the type of event to add. * For all other actions it is the unique event ID assigned when the * event was added by the FW. * @apply_time: When to start the Time Event (in GP2) * @max_delay: maximum delay to event's start (apply time), in TU * @depends_on: the unique ID of the event we depend on (if any) * @interval: interval between repetitions, in TU * @duration: duration of event in TU * @repeat: how many repetitions to do, can be IWM_TE_REPEAT_ENDLESS * @max_frags: maximal number of fragments the Time Event can be divided to * @policy: defines whether uCode shall notify the host or other uCode modules * on event and/or fragment start and/or end * using one of IWM_TE_INDEPENDENT, IWM_TE_DEP_OTHER, IWM_TE_DEP_TSF * IWM_TE_EVENT_SOCIOPATHIC * using IWM_TE_ABSENCE and using IWM_TE_NOTIF_* */ struct iwm_time_event_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; uint32_t id; /* IWM_MAC_TIME_EVENT_DATA_API_S_VER_2 */ uint32_t apply_time; uint32_t max_delay; uint32_t depends_on; uint32_t interval; uint32_t duration; uint8_t repeat; uint8_t max_frags; uint16_t policy; } __packed; /* IWM_MAC_TIME_EVENT_CMD_API_S_VER_2 */ /** * struct iwm_time_event_resp - response structure to iwm_time_event_cmd * @status: bit 0 indicates success, all others specify errors * @id: the Time Event type * @unique_id: the unique ID assigned (in ADD) or given (others) to the TE * @id_and_color: ID and color of the relevant MAC */ struct iwm_time_event_resp { uint32_t status; uint32_t id; uint32_t unique_id; uint32_t id_and_color; } __packed; /* IWM_MAC_TIME_EVENT_RSP_API_S_VER_1 */ /** * struct iwm_time_event_notif - notifications of time event start/stop * ( IWM_TIME_EVENT_NOTIFICATION = 0x2a ) * @timestamp: action timestamp in GP2 * @session_id: session's unique id * @unique_id: unique id of the Time Event itself * @id_and_color: ID and color of the relevant MAC * @action: one of IWM_TE_NOTIF_START or IWM_TE_NOTIF_END * @status: true if scheduled, false otherwise (not executed) */ struct iwm_time_event_notif { uint32_t timestamp; uint32_t session_id; uint32_t unique_id; uint32_t id_and_color; uint32_t action; uint32_t status; } __packed; /* IWM_MAC_TIME_EVENT_NTFY_API_S_VER_1 */ /* Bindings and Time Quota */ /** * struct iwm_binding_cmd - configuring bindings * ( IWM_BINDING_CONTEXT_CMD = 0x2b ) * @id_and_color: ID and color of the relevant Binding * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @macs: array of MAC id and colors which belong to the binding * @phy: PHY id and color which belongs to the binding */ struct iwm_binding_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_BINDING_DATA_API_S_VER_1 */ uint32_t macs[IWM_MAX_MACS_IN_BINDING]; uint32_t phy; } __packed; /* IWM_BINDING_CMD_API_S_VER_1 */ /* The maximal number of fragments in the FW's schedule session */ #define IWM_MVM_MAX_QUOTA 128 /** * struct iwm_time_quota_data - configuration of time quota per binding * @id_and_color: ID and color of the relevant Binding * @quota: absolute time quota in TU. The scheduler will try to divide the * remainig quota (after Time Events) according to this quota. * @max_duration: max uninterrupted context duration in TU */ struct iwm_time_quota_data { uint32_t id_and_color; uint32_t quota; uint32_t max_duration; } __packed; /* IWM_TIME_QUOTA_DATA_API_S_VER_1 */ /** * struct iwm_time_quota_cmd - configuration of time quota between bindings * ( IWM_TIME_QUOTA_CMD = 0x2c ) * @quotas: allocations per binding */ struct iwm_time_quota_cmd { struct iwm_time_quota_data quotas[IWM_MAX_BINDINGS]; } __packed; /* IWM_TIME_QUOTA_ALLOCATION_CMD_API_S_VER_1 */ /* PHY context */ /* Supported bands */ #define IWM_PHY_BAND_5 (0) #define IWM_PHY_BAND_24 (1) /* Supported channel width, vary if there is VHT support */ #define IWM_PHY_VHT_CHANNEL_MODE20 (0x0) #define IWM_PHY_VHT_CHANNEL_MODE40 (0x1) #define IWM_PHY_VHT_CHANNEL_MODE80 (0x2) #define IWM_PHY_VHT_CHANNEL_MODE160 (0x3) /* * Control channel position: * For legacy set bit means upper channel, otherwise lower. * For VHT - bit-2 marks if the control is lower/upper relative to center-freq * bits-1:0 mark the distance from the center freq. for 20Mhz, offset is 0. * center_freq * | * 40Mhz |_______|_______| * 80Mhz |_______|_______|_______|_______| * 160Mhz |_______|_______|_______|_______|_______|_______|_______|_______| * code 011 010 001 000 | 100 101 110 111 */ #define IWM_PHY_VHT_CTRL_POS_1_BELOW (0x0) #define IWM_PHY_VHT_CTRL_POS_2_BELOW (0x1) #define IWM_PHY_VHT_CTRL_POS_3_BELOW (0x2) #define IWM_PHY_VHT_CTRL_POS_4_BELOW (0x3) #define IWM_PHY_VHT_CTRL_POS_1_ABOVE (0x4) #define IWM_PHY_VHT_CTRL_POS_2_ABOVE (0x5) #define IWM_PHY_VHT_CTRL_POS_3_ABOVE (0x6) #define IWM_PHY_VHT_CTRL_POS_4_ABOVE (0x7) /* * @band: IWM_PHY_BAND_* * @channel: channel number * @width: PHY_[VHT|LEGACY]_CHANNEL_* * @ctrl channel: PHY_[VHT|LEGACY]_CTRL_* */ struct iwm_fw_channel_info { uint8_t band; uint8_t channel; uint8_t width; uint8_t ctrl_pos; } __packed; #define IWM_PHY_RX_CHAIN_DRIVER_FORCE_POS (0) #define IWM_PHY_RX_CHAIN_DRIVER_FORCE_MSK \ (0x1 << IWM_PHY_RX_CHAIN_DRIVER_FORCE_POS) #define IWM_PHY_RX_CHAIN_VALID_POS (1) #define IWM_PHY_RX_CHAIN_VALID_MSK \ (0x7 << IWM_PHY_RX_CHAIN_VALID_POS) #define IWM_PHY_RX_CHAIN_FORCE_SEL_POS (4) #define IWM_PHY_RX_CHAIN_FORCE_SEL_MSK \ (0x7 << IWM_PHY_RX_CHAIN_FORCE_SEL_POS) #define IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS (7) #define IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_MSK \ (0x7 << IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS) #define IWM_PHY_RX_CHAIN_CNT_POS (10) #define IWM_PHY_RX_CHAIN_CNT_MSK \ (0x3 << IWM_PHY_RX_CHAIN_CNT_POS) #define IWM_PHY_RX_CHAIN_MIMO_CNT_POS (12) #define IWM_PHY_RX_CHAIN_MIMO_CNT_MSK \ (0x3 << IWM_PHY_RX_CHAIN_MIMO_CNT_POS) #define IWM_PHY_RX_CHAIN_MIMO_FORCE_POS (14) #define IWM_PHY_RX_CHAIN_MIMO_FORCE_MSK \ (0x1 << IWM_PHY_RX_CHAIN_MIMO_FORCE_POS) /* TODO: fix the value, make it depend on firmware at runtime? */ #define IWM_NUM_PHY_CTX 3 /* TODO: complete missing documentation */ /** * struct iwm_phy_context_cmd - config of the PHY context * ( IWM_PHY_CONTEXT_CMD = 0x8 ) * @id_and_color: ID and color of the relevant Binding * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @apply_time: 0 means immediate apply and context switch. * other value means apply new params after X usecs * @tx_param_color: ??? * @channel_info: * @txchain_info: ??? * @rxchain_info: ??? * @acquisition_data: ??? * @dsp_cfg_flags: set to 0 */ struct iwm_phy_context_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_PHY_CONTEXT_DATA_API_S_VER_1 */ uint32_t apply_time; uint32_t tx_param_color; struct iwm_fw_channel_info ci; uint32_t txchain_info; uint32_t rxchain_info; uint32_t acquisition_data; uint32_t dsp_cfg_flags; } __packed; /* IWM_PHY_CONTEXT_CMD_API_VER_1 */ #define IWM_RX_INFO_PHY_CNT 8 #define IWM_RX_INFO_ENERGY_ANT_ABC_IDX 1 #define IWM_RX_INFO_ENERGY_ANT_A_MSK 0x000000ff #define IWM_RX_INFO_ENERGY_ANT_B_MSK 0x0000ff00 #define IWM_RX_INFO_ENERGY_ANT_C_MSK 0x00ff0000 #define IWM_RX_INFO_ENERGY_ANT_A_POS 0 #define IWM_RX_INFO_ENERGY_ANT_B_POS 8 #define IWM_RX_INFO_ENERGY_ANT_C_POS 16 #define IWM_RX_INFO_AGC_IDX 1 #define IWM_RX_INFO_RSSI_AB_IDX 2 #define IWM_OFDM_AGC_A_MSK 0x0000007f #define IWM_OFDM_AGC_A_POS 0 #define IWM_OFDM_AGC_B_MSK 0x00003f80 #define IWM_OFDM_AGC_B_POS 7 #define IWM_OFDM_AGC_CODE_MSK 0x3fe00000 #define IWM_OFDM_AGC_CODE_POS 20 #define IWM_OFDM_RSSI_INBAND_A_MSK 0x00ff #define IWM_OFDM_RSSI_A_POS 0 #define IWM_OFDM_RSSI_ALLBAND_A_MSK 0xff00 #define IWM_OFDM_RSSI_ALLBAND_A_POS 8 #define IWM_OFDM_RSSI_INBAND_B_MSK 0xff0000 #define IWM_OFDM_RSSI_B_POS 16 #define IWM_OFDM_RSSI_ALLBAND_B_MSK 0xff000000 #define IWM_OFDM_RSSI_ALLBAND_B_POS 24 /** * struct iwm_rx_phy_info - phy info * (IWM_REPLY_RX_PHY_CMD = 0xc0) * @non_cfg_phy_cnt: non configurable DSP phy data byte count * @cfg_phy_cnt: configurable DSP phy data byte count * @stat_id: configurable DSP phy data set ID * @reserved1: * @system_timestamp: GP2 at on air rise * @timestamp: TSF at on air rise * @beacon_time_stamp: beacon at on-air rise * @phy_flags: general phy flags: band, modulation, ... * @channel: channel number * @non_cfg_phy_buf: for various implementations of non_cfg_phy * @rate_n_flags: IWM_RATE_MCS_* * @byte_count: frame's byte-count * @frame_time: frame's time on the air, based on byte count and frame rate * calculation * @mac_active_msk: what MACs were active when the frame was received * * Before each Rx, the device sends this data. It contains PHY information * about the reception of the packet. */ struct iwm_rx_phy_info { uint8_t non_cfg_phy_cnt; uint8_t cfg_phy_cnt; uint8_t stat_id; uint8_t reserved1; uint32_t system_timestamp; uint64_t timestamp; uint32_t beacon_time_stamp; uint16_t phy_flags; #define IWM_PHY_INFO_FLAG_SHPREAMBLE (1 << 2) uint16_t channel; uint32_t non_cfg_phy[IWM_RX_INFO_PHY_CNT]; uint8_t rate; uint8_t rflags; uint16_t xrflags; uint32_t byte_count; uint16_t mac_active_msk; uint16_t frame_time; } __packed; struct iwm_rx_mpdu_res_start { uint16_t byte_count; uint16_t reserved; } __packed; /** * enum iwm_rx_phy_flags - to parse %iwm_rx_phy_info phy_flags * @IWM_RX_RES_PHY_FLAGS_BAND_24: true if the packet was received on 2.4 band * @IWM_RX_RES_PHY_FLAGS_MOD_CCK: * @IWM_RX_RES_PHY_FLAGS_SHORT_PREAMBLE: true if packet's preamble was short * @IWM_RX_RES_PHY_FLAGS_NARROW_BAND: * @IWM_RX_RES_PHY_FLAGS_ANTENNA: antenna on which the packet was received * @IWM_RX_RES_PHY_FLAGS_AGG: set if the packet was part of an A-MPDU * @IWM_RX_RES_PHY_FLAGS_OFDM_HT: The frame was an HT frame * @IWM_RX_RES_PHY_FLAGS_OFDM_GF: The frame used GF preamble * @IWM_RX_RES_PHY_FLAGS_OFDM_VHT: The frame was a VHT frame */ enum iwm_rx_phy_flags { IWM_RX_RES_PHY_FLAGS_BAND_24 = (1 << 0), IWM_RX_RES_PHY_FLAGS_MOD_CCK = (1 << 1), IWM_RX_RES_PHY_FLAGS_SHORT_PREAMBLE = (1 << 2), IWM_RX_RES_PHY_FLAGS_NARROW_BAND = (1 << 3), IWM_RX_RES_PHY_FLAGS_ANTENNA = (0x7 << 4), IWM_RX_RES_PHY_FLAGS_ANTENNA_POS = 4, IWM_RX_RES_PHY_FLAGS_AGG = (1 << 7), IWM_RX_RES_PHY_FLAGS_OFDM_HT = (1 << 8), IWM_RX_RES_PHY_FLAGS_OFDM_GF = (1 << 9), IWM_RX_RES_PHY_FLAGS_OFDM_VHT = (1 << 10), }; /** * enum iwm_mvm_rx_status - written by fw for each Rx packet * @IWM_RX_MPDU_RES_STATUS_CRC_OK: CRC is fine * @IWM_RX_MPDU_RES_STATUS_OVERRUN_OK: there was no RXE overflow * @IWM_RX_MPDU_RES_STATUS_SRC_STA_FOUND: * @IWM_RX_MPDU_RES_STATUS_KEY_VALID: * @IWM_RX_MPDU_RES_STATUS_KEY_PARAM_OK: * @IWM_RX_MPDU_RES_STATUS_ICV_OK: ICV is fine, if not, the packet is destroyed * @IWM_RX_MPDU_RES_STATUS_MIC_OK: used for CCM alg only. TKIP MIC is checked * in the driver. * @IWM_RX_MPDU_RES_STATUS_TTAK_OK: TTAK is fine * @IWM_RX_MPDU_RES_STATUS_MNG_FRAME_REPLAY_ERR: valid for alg = CCM_CMAC or * alg = CCM only. Checks replay attack for 11w frames. Relevant only if * %IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME is set. * @IWM_RX_MPDU_RES_STATUS_SEC_NO_ENC: this frame is not encrypted * @IWM_RX_MPDU_RES_STATUS_SEC_WEP_ENC: this frame is encrypted using WEP * @IWM_RX_MPDU_RES_STATUS_SEC_CCM_ENC: this frame is encrypted using CCM * @IWM_RX_MPDU_RES_STATUS_SEC_TKIP_ENC: this frame is encrypted using TKIP * @IWM_RX_MPDU_RES_STATUS_SEC_CCM_CMAC_ENC: this frame is encrypted using CCM_CMAC * @IWM_RX_MPDU_RES_STATUS_SEC_ENC_ERR: this frame couldn't be decrypted * @IWM_RX_MPDU_RES_STATUS_SEC_ENC_MSK: bitmask of the encryption algorithm * @IWM_RX_MPDU_RES_STATUS_DEC_DONE: this frame has been successfully decrypted * @IWM_RX_MPDU_RES_STATUS_PROTECT_FRAME_BIT_CMP: * @IWM_RX_MPDU_RES_STATUS_EXT_IV_BIT_CMP: * @IWM_RX_MPDU_RES_STATUS_KEY_ID_CMP_BIT: * @IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME: this frame is an 11w management frame * @IWM_RX_MPDU_RES_STATUS_HASH_INDEX_MSK: * @IWM_RX_MPDU_RES_STATUS_STA_ID_MSK: * @IWM_RX_MPDU_RES_STATUS_RRF_KILL: * @IWM_RX_MPDU_RES_STATUS_FILTERING_MSK: * @IWM_RX_MPDU_RES_STATUS2_FILTERING_MSK: */ enum iwm_mvm_rx_status { IWM_RX_MPDU_RES_STATUS_CRC_OK = (1 << 0), IWM_RX_MPDU_RES_STATUS_OVERRUN_OK = (1 << 1), IWM_RX_MPDU_RES_STATUS_SRC_STA_FOUND = (1 << 2), IWM_RX_MPDU_RES_STATUS_KEY_VALID = (1 << 3), IWM_RX_MPDU_RES_STATUS_KEY_PARAM_OK = (1 << 4), IWM_RX_MPDU_RES_STATUS_ICV_OK = (1 << 5), IWM_RX_MPDU_RES_STATUS_MIC_OK = (1 << 6), IWM_RX_MPDU_RES_STATUS_TTAK_OK = (1 << 7), IWM_RX_MPDU_RES_STATUS_MNG_FRAME_REPLAY_ERR = (1 << 7), IWM_RX_MPDU_RES_STATUS_SEC_NO_ENC = (0 << 8), IWM_RX_MPDU_RES_STATUS_SEC_WEP_ENC = (1 << 8), IWM_RX_MPDU_RES_STATUS_SEC_CCM_ENC = (2 << 8), IWM_RX_MPDU_RES_STATUS_SEC_TKIP_ENC = (3 << 8), IWM_RX_MPDU_RES_STATUS_SEC_EXT_ENC = (4 << 8), IWM_RX_MPDU_RES_STATUS_SEC_CCM_CMAC_ENC = (6 << 8), IWM_RX_MPDU_RES_STATUS_SEC_ENC_ERR = (7 << 8), IWM_RX_MPDU_RES_STATUS_SEC_ENC_MSK = (7 << 8), IWM_RX_MPDU_RES_STATUS_DEC_DONE = (1 << 11), IWM_RX_MPDU_RES_STATUS_PROTECT_FRAME_BIT_CMP = (1 << 12), IWM_RX_MPDU_RES_STATUS_EXT_IV_BIT_CMP = (1 << 13), IWM_RX_MPDU_RES_STATUS_KEY_ID_CMP_BIT = (1 << 14), IWM_RX_MPDU_RES_STATUS_ROBUST_MNG_FRAME = (1 << 15), IWM_RX_MPDU_RES_STATUS_HASH_INDEX_MSK = (0x3F0000), IWM_RX_MPDU_RES_STATUS_STA_ID_MSK = (0x1f000000), IWM_RX_MPDU_RES_STATUS_RRF_KILL = (1 << 29), IWM_RX_MPDU_RES_STATUS_FILTERING_MSK = (0xc00000), IWM_RX_MPDU_RES_STATUS2_FILTERING_MSK = (0xc0000000), }; /** * struct iwm_radio_version_notif - information on the radio version * ( IWM_RADIO_VERSION_NOTIFICATION = 0x68 ) * @radio_flavor: * @radio_step: * @radio_dash: */ struct iwm_radio_version_notif { uint32_t radio_flavor; uint32_t radio_step; uint32_t radio_dash; } __packed; /* IWM_RADIO_VERSION_NOTOFICATION_S_VER_1 */ enum iwm_card_state_flags { IWM_CARD_ENABLED = 0x00, IWM_HW_CARD_DISABLED = 0x01, IWM_SW_CARD_DISABLED = 0x02, IWM_CT_KILL_CARD_DISABLED = 0x04, IWM_HALT_CARD_DISABLED = 0x08, IWM_CARD_DISABLED_MSK = 0x0f, IWM_CARD_IS_RX_ON = 0x10, }; /** * struct iwm_radio_version_notif - information on the radio version * (IWM_CARD_STATE_NOTIFICATION = 0xa1 ) * @flags: %iwm_card_state_flags */ struct iwm_card_state_notif { uint32_t flags; } __packed; /* CARD_STATE_NTFY_API_S_VER_1 */ /** * struct iwm_missed_beacons_notif - information on missed beacons * ( IWM_MISSED_BEACONS_NOTIFICATION = 0xa2 ) * @mac_id: interface ID * @consec_missed_beacons_since_last_rx: number of consecutive missed * beacons since last RX. * @consec_missed_beacons: number of consecutive missed beacons * @num_expected_beacons: * @num_recvd_beacons: */ struct iwm_missed_beacons_notif { uint32_t mac_id; uint32_t consec_missed_beacons_since_last_rx; uint32_t consec_missed_beacons; uint32_t num_expected_beacons; uint32_t num_recvd_beacons; } __packed; /* IWM_MISSED_BEACON_NTFY_API_S_VER_3 */ /** * struct iwm_mfuart_load_notif - mfuart image version & status * ( IWM_MFUART_LOAD_NOTIFICATION = 0xb1 ) * @installed_ver: installed image version * @external_ver: external image version * @status: MFUART loading status * @duration: MFUART loading time */ struct iwm_mfuart_load_notif { uint32_t installed_ver; uint32_t external_ver; uint32_t status; uint32_t duration; } __packed; /*MFU_LOADER_NTFY_API_S_VER_1*/ /** * struct iwm_set_calib_default_cmd - set default value for calibration. * ( IWM_SET_CALIB_DEFAULT_CMD = 0x8e ) * @calib_index: the calibration to set value for * @length: of data * @data: the value to set for the calibration result */ struct iwm_set_calib_default_cmd { uint16_t calib_index; uint16_t length; uint8_t data[0]; } __packed; /* IWM_PHY_CALIB_OVERRIDE_VALUES_S */ #define IWM_MAX_PORT_ID_NUM 2 #define IWM_MAX_MCAST_FILTERING_ADDRESSES 256 /** * struct iwm_mcast_filter_cmd - configure multicast filter. * @filter_own: Set 1 to filter out multicast packets sent by station itself * @port_id: Multicast MAC addresses array specifier. This is a strange way * to identify network interface adopted in host-device IF. * It is used by FW as index in array of addresses. This array has * IWM_MAX_PORT_ID_NUM members. * @count: Number of MAC addresses in the array * @pass_all: Set 1 to pass all multicast packets. * @bssid: current association BSSID. * @addr_list: Place holder for array of MAC addresses. * IMPORTANT: add padding if necessary to ensure DWORD alignment. */ struct iwm_mcast_filter_cmd { uint8_t filter_own; uint8_t port_id; uint8_t count; uint8_t pass_all; uint8_t bssid[6]; uint8_t reserved[2]; uint8_t addr_list[0]; } __packed; /* IWM_MCAST_FILTERING_CMD_API_S_VER_1 */ /* * The first MAC indices (starting from 0) * are available to the driver, AUX follows */ #define IWM_MAC_INDEX_AUX 4 #define IWM_MAC_INDEX_MIN_DRIVER 0 #define IWM_NUM_MAC_INDEX_DRIVER IWM_MAC_INDEX_AUX #define IWM_NUM_MAC_INDEX (IWM_MAC_INDEX_AUX + 1) /*********************************** * Statistics API ***********************************/ struct iwm_mvm_statistics_dbg { uint32_t burst_check; uint32_t burst_count; uint32_t wait_for_silence_timeout_cnt; uint32_t reserved[3]; } __packed; /* IWM_STATISTICS_DEBUG_API_S_VER_2 */ struct iwm_mvm_statistics_div { uint32_t tx_on_a; uint32_t tx_on_b; uint32_t exec_time; uint32_t probe_time; uint32_t rssi_ant; uint32_t reserved2; } __packed; /* IWM_STATISTICS_SLOW_DIV_API_S_VER_2 */ struct iwm_mvm_statistics_rx_non_phy { uint32_t bogus_cts; /* CTS received when not expecting CTS */ uint32_t bogus_ack; /* ACK received when not expecting ACK */ uint32_t non_bssid_frames; /* number of frames with BSSID that * doesn't belong to the STA BSSID */ uint32_t filtered_frames; /* count frames that were dumped in the * filtering process */ uint32_t non_channel_beacons; /* beacons with our bss id but not on * our serving channel */ uint32_t channel_beacons; /* beacons with our bss id and in our * serving channel */ uint32_t num_missed_bcon; /* number of missed beacons */ uint32_t adc_rx_saturation_time; /* count in 0.8us units the time the * ADC was in saturation */ uint32_t ina_detection_search_time;/* total time (in 0.8us) searched * for INA */ uint32_t beacon_silence_rssi[3];/* RSSI silence after beacon frame */ uint32_t interference_data_flag; /* flag for interference data * availability. 1 when data is * available. */ uint32_t channel_load; /* counts RX Enable time in uSec */ uint32_t dsp_false_alarms; /* DSP false alarm (both OFDM * and CCK) counter */ uint32_t beacon_rssi_a; uint32_t beacon_rssi_b; uint32_t beacon_rssi_c; uint32_t beacon_energy_a; uint32_t beacon_energy_b; uint32_t beacon_energy_c; uint32_t num_bt_kills; uint32_t mac_id; uint32_t directed_data_mpdu; } __packed; /* IWM_STATISTICS_RX_NON_PHY_API_S_VER_3 */ struct iwm_mvm_statistics_rx_phy { uint32_t ina_cnt; uint32_t fina_cnt; uint32_t plcp_err; uint32_t crc32_err; uint32_t overrun_err; uint32_t early_overrun_err; uint32_t crc32_good; uint32_t false_alarm_cnt; uint32_t fina_sync_err_cnt; uint32_t sfd_timeout; uint32_t fina_timeout; uint32_t unresponded_rts; uint32_t rxe_frame_limit_overrun; uint32_t sent_ack_cnt; uint32_t sent_cts_cnt; uint32_t sent_ba_rsp_cnt; uint32_t dsp_self_kill; uint32_t mh_format_err; uint32_t re_acq_main_rssi_sum; uint32_t reserved; } __packed; /* IWM_STATISTICS_RX_PHY_API_S_VER_2 */ struct iwm_mvm_statistics_rx_ht_phy { uint32_t plcp_err; uint32_t overrun_err; uint32_t early_overrun_err; uint32_t crc32_good; uint32_t crc32_err; uint32_t mh_format_err; uint32_t agg_crc32_good; uint32_t agg_mpdu_cnt; uint32_t agg_cnt; uint32_t unsupport_mcs; } __packed; /* IWM_STATISTICS_HT_RX_PHY_API_S_VER_1 */ struct iwm_mvm_statistics_tx_non_phy { uint32_t preamble_cnt; uint32_t rx_detected_cnt; uint32_t bt_prio_defer_cnt; uint32_t bt_prio_kill_cnt; uint32_t few_bytes_cnt; uint32_t cts_timeout; uint32_t ack_timeout; uint32_t expected_ack_cnt; uint32_t actual_ack_cnt; uint32_t dump_msdu_cnt; uint32_t burst_abort_next_frame_mismatch_cnt; uint32_t burst_abort_missing_next_frame_cnt; uint32_t cts_timeout_collision; uint32_t ack_or_ba_timeout_collision; } __packed; /* IWM_STATISTICS_TX_NON_PHY_API_S_VER_3 */ #define IWM_MAX_CHAINS 3 struct iwm_mvm_statistics_tx_non_phy_agg { uint32_t ba_timeout; uint32_t ba_reschedule_frames; uint32_t scd_query_agg_frame_cnt; uint32_t scd_query_no_agg; uint32_t scd_query_agg; uint32_t scd_query_mismatch; uint32_t frame_not_ready; uint32_t underrun; uint32_t bt_prio_kill; uint32_t rx_ba_rsp_cnt; int8_t txpower[IWM_MAX_CHAINS]; int8_t reserved; uint32_t reserved2; } __packed; /* IWM_STATISTICS_TX_NON_PHY_AGG_API_S_VER_1 */ struct iwm_mvm_statistics_tx_channel_width { uint32_t ext_cca_narrow_ch20[1]; uint32_t ext_cca_narrow_ch40[2]; uint32_t ext_cca_narrow_ch80[3]; uint32_t ext_cca_narrow_ch160[4]; uint32_t last_tx_ch_width_indx; uint32_t rx_detected_per_ch_width[4]; uint32_t success_per_ch_width[4]; uint32_t fail_per_ch_width[4]; }; /* IWM_STATISTICS_TX_CHANNEL_WIDTH_API_S_VER_1 */ struct iwm_mvm_statistics_tx { struct iwm_mvm_statistics_tx_non_phy general; struct iwm_mvm_statistics_tx_non_phy_agg agg; struct iwm_mvm_statistics_tx_channel_width channel_width; } __packed; /* IWM_STATISTICS_TX_API_S_VER_4 */ struct iwm_mvm_statistics_bt_activity { uint32_t hi_priority_tx_req_cnt; uint32_t hi_priority_tx_denied_cnt; uint32_t lo_priority_tx_req_cnt; uint32_t lo_priority_tx_denied_cnt; uint32_t hi_priority_rx_req_cnt; uint32_t hi_priority_rx_denied_cnt; uint32_t lo_priority_rx_req_cnt; uint32_t lo_priority_rx_denied_cnt; } __packed; /* IWM_STATISTICS_BT_ACTIVITY_API_S_VER_1 */ struct iwm_mvm_statistics_general_v8 { uint32_t radio_temperature; uint32_t radio_voltage; struct iwm_mvm_statistics_dbg dbg; uint32_t sleep_time; uint32_t slots_out; uint32_t slots_idle; uint32_t ttl_timestamp; struct iwm_mvm_statistics_div slow_div; uint32_t rx_enable_counter; /* * num_of_sos_states: * count the number of times we have to re-tune * in order to get out of bad PHY status */ uint32_t num_of_sos_states; uint32_t beacon_filtered; uint32_t missed_beacons; uint8_t beacon_filter_average_energy; uint8_t beacon_filter_reason; uint8_t beacon_filter_current_energy; uint8_t beacon_filter_reserved; uint32_t beacon_filter_delta_time; struct iwm_mvm_statistics_bt_activity bt_activity; uint64_t rx_time; uint64_t on_time_rf; uint64_t on_time_scan; uint64_t tx_time; uint32_t beacon_counter[IWM_NUM_MAC_INDEX]; uint8_t beacon_average_energy[IWM_NUM_MAC_INDEX]; uint8_t reserved[4 - (IWM_NUM_MAC_INDEX % 4)]; } __packed; /* IWM_STATISTICS_GENERAL_API_S_VER_8 */ struct iwm_mvm_statistics_rx { struct iwm_mvm_statistics_rx_phy ofdm; struct iwm_mvm_statistics_rx_phy cck; struct iwm_mvm_statistics_rx_non_phy general; struct iwm_mvm_statistics_rx_ht_phy ofdm_ht; } __packed; /* IWM_STATISTICS_RX_API_S_VER_3 */ /* * IWM_STATISTICS_NOTIFICATION = 0x9d (notification only, not a command) * * By default, uCode issues this notification after receiving a beacon * while associated. To disable this behavior, set DISABLE_NOTIF flag in the * IWM_STATISTICS_CMD (0x9c), below. */ struct iwm_notif_statistics_v10 { uint32_t flag; struct iwm_mvm_statistics_rx rx; struct iwm_mvm_statistics_tx tx; struct iwm_mvm_statistics_general_v8 general; } __packed; /* IWM_STATISTICS_NTFY_API_S_VER_10 */ #define IWM_STATISTICS_FLG_CLEAR 0x1 #define IWM_STATISTICS_FLG_DISABLE_NOTIF 0x2 struct iwm_statistics_cmd { uint32_t flags; } __packed; /* IWM_STATISTICS_CMD_API_S_VER_1 */ /*********************************** * Smart Fifo API ***********************************/ /* Smart Fifo state */ enum iwm_sf_state { IWM_SF_LONG_DELAY_ON = 0, /* should never be called by driver */ IWM_SF_FULL_ON, IWM_SF_UNINIT, IWM_SF_INIT_OFF, IWM_SF_HW_NUM_STATES }; /* Smart Fifo possible scenario */ enum iwm_sf_scenario { IWM_SF_SCENARIO_SINGLE_UNICAST, IWM_SF_SCENARIO_AGG_UNICAST, IWM_SF_SCENARIO_MULTICAST, IWM_SF_SCENARIO_BA_RESP, IWM_SF_SCENARIO_TX_RESP, IWM_SF_NUM_SCENARIO }; #define IWM_SF_TRANSIENT_STATES_NUMBER 2 /* IWM_SF_LONG_DELAY_ON and IWM_SF_FULL_ON */ #define IWM_SF_NUM_TIMEOUT_TYPES 2 /* Aging timer and Idle timer */ /* smart FIFO default values */ #define IWM_SF_W_MARK_SISO 4096 #define IWM_SF_W_MARK_MIMO2 8192 #define IWM_SF_W_MARK_MIMO3 6144 #define IWM_SF_W_MARK_LEGACY 4096 #define IWM_SF_W_MARK_SCAN 4096 /* SF Scenarios timers for default configuration (aligned to 32 uSec) */ #define IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_AGG_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_MCAST_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_MCAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_BA_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_BA_AGING_TIMER_DEF 400 /* 0.4 mSec */ #define IWM_SF_TX_RE_IDLE_TIMER_DEF 160 /* 150 uSec */ #define IWM_SF_TX_RE_AGING_TIMER_DEF 400 /* 0.4 mSec */ /* SF Scenarios timers for FULL_ON state (aligned to 32 uSec) */ #define IWM_SF_SINGLE_UNICAST_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_SINGLE_UNICAST_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_AGG_UNICAST_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_AGG_UNICAST_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_MCAST_IDLE_TIMER 2016 /* 2 mSec */ #define IWM_SF_MCAST_AGING_TIMER 10016 /* 10 mSec */ #define IWM_SF_BA_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_BA_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_TX_RE_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_TX_RE_AGING_TIMER 2016 /* 2 mSec */ #define IWM_SF_LONG_DELAY_AGING_TIMER 1000000 /* 1 Sec */ #define IWM_SF_CFG_DUMMY_NOTIF_OFF (1 << 16) /** * Smart Fifo configuration command. * @state: smart fifo state, types listed in iwm_sf_state. * @watermark: Minimum allowed available free space in RXF for transient state. * @long_delay_timeouts: aging and idle timer values for each scenario * in long delay state. * @full_on_timeouts: timer values for each scenario in full on state. */ struct iwm_sf_cfg_cmd { uint32_t state; uint32_t watermark[IWM_SF_TRANSIENT_STATES_NUMBER]; uint32_t long_delay_timeouts[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES]; uint32_t full_on_timeouts[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES]; } __packed; /* IWM_SF_CFG_API_S_VER_2 */ /* * END mvm/fw-api.h */ /* * BEGIN mvm/fw-api-mac.h */ enum iwm_ac { IWM_AC_BK, IWM_AC_BE, IWM_AC_VI, IWM_AC_VO, IWM_AC_NUM, }; /** * enum iwm_mac_protection_flags - MAC context flags * @IWM_MAC_PROT_FLG_TGG_PROTECT: 11g protection when transmitting OFDM frames, * this will require CCK RTS/CTS2self. * RTS/CTS will protect full burst time. * @IWM_MAC_PROT_FLG_HT_PROT: enable HT protection * @IWM_MAC_PROT_FLG_FAT_PROT: protect 40 MHz transmissions * @IWM_MAC_PROT_FLG_SELF_CTS_EN: allow CTS2self */ enum iwm_mac_protection_flags { IWM_MAC_PROT_FLG_TGG_PROTECT = (1 << 3), IWM_MAC_PROT_FLG_HT_PROT = (1 << 23), IWM_MAC_PROT_FLG_FAT_PROT = (1 << 24), IWM_MAC_PROT_FLG_SELF_CTS_EN = (1 << 30), }; #define IWM_MAC_FLG_SHORT_SLOT (1 << 4) #define IWM_MAC_FLG_SHORT_PREAMBLE (1 << 5) /** * enum iwm_mac_types - Supported MAC types * @IWM_FW_MAC_TYPE_FIRST: lowest supported MAC type * @IWM_FW_MAC_TYPE_AUX: Auxiliary MAC (internal) * @IWM_FW_MAC_TYPE_LISTENER: monitor MAC type (?) * @IWM_FW_MAC_TYPE_PIBSS: Pseudo-IBSS * @IWM_FW_MAC_TYPE_IBSS: IBSS * @IWM_FW_MAC_TYPE_BSS_STA: BSS (managed) station * @IWM_FW_MAC_TYPE_P2P_DEVICE: P2P Device * @IWM_FW_MAC_TYPE_P2P_STA: P2P client * @IWM_FW_MAC_TYPE_GO: P2P GO * @IWM_FW_MAC_TYPE_TEST: ? * @IWM_FW_MAC_TYPE_MAX: highest support MAC type */ enum iwm_mac_types { IWM_FW_MAC_TYPE_FIRST = 1, IWM_FW_MAC_TYPE_AUX = IWM_FW_MAC_TYPE_FIRST, IWM_FW_MAC_TYPE_LISTENER, IWM_FW_MAC_TYPE_PIBSS, IWM_FW_MAC_TYPE_IBSS, IWM_FW_MAC_TYPE_BSS_STA, IWM_FW_MAC_TYPE_P2P_DEVICE, IWM_FW_MAC_TYPE_P2P_STA, IWM_FW_MAC_TYPE_GO, IWM_FW_MAC_TYPE_TEST, IWM_FW_MAC_TYPE_MAX = IWM_FW_MAC_TYPE_TEST }; /* IWM_MAC_CONTEXT_TYPE_API_E_VER_1 */ /** * enum iwm_tsf_id - TSF hw timer ID * @IWM_TSF_ID_A: use TSF A * @IWM_TSF_ID_B: use TSF B * @IWM_TSF_ID_C: use TSF C * @IWM_TSF_ID_D: use TSF D * @IWM_NUM_TSF_IDS: number of TSF timers available */ enum iwm_tsf_id { IWM_TSF_ID_A = 0, IWM_TSF_ID_B = 1, IWM_TSF_ID_C = 2, IWM_TSF_ID_D = 3, IWM_NUM_TSF_IDS = 4, }; /* IWM_TSF_ID_API_E_VER_1 */ /** * struct iwm_mac_data_ap - configuration data for AP MAC context * @beacon_time: beacon transmit time in system time * @beacon_tsf: beacon transmit time in TSF * @bi: beacon interval in TU * @bi_reciprocal: 2^32 / bi * @dtim_interval: dtim transmit time in TU * @dtim_reciprocal: 2^32 / dtim_interval * @mcast_qid: queue ID for multicast traffic * @beacon_template: beacon template ID */ struct iwm_mac_data_ap { uint32_t beacon_time; uint64_t beacon_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t dtim_interval; uint32_t dtim_reciprocal; uint32_t mcast_qid; uint32_t beacon_template; } __packed; /* AP_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_ibss - configuration data for IBSS MAC context * @beacon_time: beacon transmit time in system time * @beacon_tsf: beacon transmit time in TSF * @bi: beacon interval in TU * @bi_reciprocal: 2^32 / bi * @beacon_template: beacon template ID */ struct iwm_mac_data_ibss { uint32_t beacon_time; uint64_t beacon_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t beacon_template; } __packed; /* IBSS_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_sta - configuration data for station MAC context * @is_assoc: 1 for associated state, 0 otherwise * @dtim_time: DTIM arrival time in system time * @dtim_tsf: DTIM arrival time in TSF * @bi: beacon interval in TU, applicable only when associated * @bi_reciprocal: 2^32 / bi , applicable only when associated * @dtim_interval: DTIM interval in TU, applicable only when associated * @dtim_reciprocal: 2^32 / dtim_interval , applicable only when associated * @listen_interval: in beacon intervals, applicable only when associated * @assoc_id: unique ID assigned by the AP during association */ struct iwm_mac_data_sta { uint32_t is_assoc; uint32_t dtim_time; uint64_t dtim_tsf; uint32_t bi; uint32_t bi_reciprocal; uint32_t dtim_interval; uint32_t dtim_reciprocal; uint32_t listen_interval; uint32_t assoc_id; uint32_t assoc_beacon_arrive_time; } __packed; /* IWM_STA_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_go - configuration data for P2P GO MAC context * @ap: iwm_mac_data_ap struct with most config data * @ctwin: client traffic window in TU (period after TBTT when GO is present). * 0 indicates that there is no CT window. * @opp_ps_enabled: indicate that opportunistic PS allowed */ struct iwm_mac_data_go { struct iwm_mac_data_ap ap; uint32_t ctwin; uint32_t opp_ps_enabled; } __packed; /* GO_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_p2p_sta - configuration data for P2P client MAC context * @sta: iwm_mac_data_sta struct with most config data * @ctwin: client traffic window in TU (period after TBTT when GO is present). * 0 indicates that there is no CT window. */ struct iwm_mac_data_p2p_sta { struct iwm_mac_data_sta sta; uint32_t ctwin; } __packed; /* P2P_STA_MAC_DATA_API_S_VER_1 */ /** * struct iwm_mac_data_pibss - Pseudo IBSS config data * @stats_interval: interval in TU between statistics notifications to host. */ struct iwm_mac_data_pibss { uint32_t stats_interval; } __packed; /* PIBSS_MAC_DATA_API_S_VER_1 */ /* * struct iwm_mac_data_p2p_dev - configuration data for the P2P Device MAC * context. * @is_disc_extended: if set to true, P2P Device discoverability is enabled on * other channels as well. This should be to true only in case that the * device is discoverable and there is an active GO. Note that setting this * field when not needed, will increase the number of interrupts and have * effect on the platform power, as this setting opens the Rx filters on * all macs. */ struct iwm_mac_data_p2p_dev { uint32_t is_disc_extended; } __packed; /* _P2P_DEV_MAC_DATA_API_S_VER_1 */ /** * enum iwm_mac_filter_flags - MAC context filter flags * @IWM_MAC_FILTER_IN_PROMISC: accept all data frames * @IWM_MAC_FILTER_IN_CONTROL_AND_MGMT: pass all mangement and * control frames to the host * @IWM_MAC_FILTER_ACCEPT_GRP: accept multicast frames * @IWM_MAC_FILTER_DIS_DECRYPT: don't decrypt unicast frames * @IWM_MAC_FILTER_DIS_GRP_DECRYPT: don't decrypt multicast frames * @IWM_MAC_FILTER_IN_BEACON: transfer foreign BSS's beacons to host * (in station mode when associated) * @IWM_MAC_FILTER_OUT_BCAST: filter out all broadcast frames * @IWM_MAC_FILTER_IN_CRC32: extract FCS and append it to frames * @IWM_MAC_FILTER_IN_PROBE_REQUEST: pass probe requests to host */ enum iwm_mac_filter_flags { IWM_MAC_FILTER_IN_PROMISC = (1 << 0), IWM_MAC_FILTER_IN_CONTROL_AND_MGMT = (1 << 1), IWM_MAC_FILTER_ACCEPT_GRP = (1 << 2), IWM_MAC_FILTER_DIS_DECRYPT = (1 << 3), IWM_MAC_FILTER_DIS_GRP_DECRYPT = (1 << 4), IWM_MAC_FILTER_IN_BEACON = (1 << 6), IWM_MAC_FILTER_OUT_BCAST = (1 << 8), IWM_MAC_FILTER_IN_CRC32 = (1 << 11), IWM_MAC_FILTER_IN_PROBE_REQUEST = (1 << 12), }; /** * enum iwm_mac_qos_flags - QoS flags * @IWM_MAC_QOS_FLG_UPDATE_EDCA: ? * @IWM_MAC_QOS_FLG_TGN: HT is enabled * @IWM_MAC_QOS_FLG_TXOP_TYPE: ? * */ enum iwm_mac_qos_flags { IWM_MAC_QOS_FLG_UPDATE_EDCA = (1 << 0), IWM_MAC_QOS_FLG_TGN = (1 << 1), IWM_MAC_QOS_FLG_TXOP_TYPE = (1 << 4), }; /** * struct iwm_ac_qos - QOS timing params for IWM_MAC_CONTEXT_CMD * @cw_min: Contention window, start value in numbers of slots. * Should be a power-of-2, minus 1. Device's default is 0x0f. * @cw_max: Contention window, max value in numbers of slots. * Should be a power-of-2, minus 1. Device's default is 0x3f. * @aifsn: Number of slots in Arbitration Interframe Space (before * performing random backoff timing prior to Tx). Device default 1. * @fifos_mask: FIFOs used by this MAC for this AC * @edca_txop: Length of Tx opportunity, in uSecs. Device default is 0. * * One instance of this config struct for each of 4 EDCA access categories * in struct iwm_qosparam_cmd. * * Device will automatically increase contention window by (2*CW) + 1 for each * transmission retry. Device uses cw_max as a bit mask, ANDed with new CW * value, to cap the CW value. */ struct iwm_ac_qos { uint16_t cw_min; uint16_t cw_max; uint8_t aifsn; uint8_t fifos_mask; uint16_t edca_txop; } __packed; /* IWM_AC_QOS_API_S_VER_2 */ /** * struct iwm_mac_ctx_cmd - command structure to configure MAC contexts * ( IWM_MAC_CONTEXT_CMD = 0x28 ) * @id_and_color: ID and color of the MAC * @action: action to perform, one of IWM_FW_CTXT_ACTION_* * @mac_type: one of IWM_FW_MAC_TYPE_* * @tsd_id: TSF HW timer, one of IWM_TSF_ID_* * @node_addr: MAC address * @bssid_addr: BSSID * @cck_rates: basic rates available for CCK * @ofdm_rates: basic rates available for OFDM * @protection_flags: combination of IWM_MAC_PROT_FLG_FLAG_* * @cck_short_preamble: 0x20 for enabling short preamble, 0 otherwise * @short_slot: 0x10 for enabling short slots, 0 otherwise * @filter_flags: combination of IWM_MAC_FILTER_* * @qos_flags: from IWM_MAC_QOS_FLG_* * @ac: one iwm_mac_qos configuration for each AC * @mac_specific: one of struct iwm_mac_data_*, according to mac_type */ struct iwm_mac_ctx_cmd { /* COMMON_INDEX_HDR_API_S_VER_1 */ uint32_t id_and_color; uint32_t action; /* IWM_MAC_CONTEXT_COMMON_DATA_API_S_VER_1 */ uint32_t mac_type; uint32_t tsf_id; uint8_t node_addr[6]; uint16_t reserved_for_node_addr; uint8_t bssid_addr[6]; uint16_t reserved_for_bssid_addr; uint32_t cck_rates; uint32_t ofdm_rates; uint32_t protection_flags; uint32_t cck_short_preamble; uint32_t short_slot; uint32_t filter_flags; /* IWM_MAC_QOS_PARAM_API_S_VER_1 */ uint32_t qos_flags; struct iwm_ac_qos ac[IWM_AC_NUM+1]; /* IWM_MAC_CONTEXT_COMMON_DATA_API_S */ union { struct iwm_mac_data_ap ap; struct iwm_mac_data_go go; struct iwm_mac_data_sta sta; struct iwm_mac_data_p2p_sta p2p_sta; struct iwm_mac_data_p2p_dev p2p_dev; struct iwm_mac_data_pibss pibss; struct iwm_mac_data_ibss ibss; }; } __packed; /* IWM_MAC_CONTEXT_CMD_API_S_VER_1 */ static inline uint32_t iwm_mvm_reciprocal(uint32_t v) { if (!v) return 0; return 0xFFFFFFFF / v; } #define IWM_NONQOS_SEQ_GET 0x1 #define IWM_NONQOS_SEQ_SET 0x2 struct iwm_nonqos_seq_query_cmd { uint32_t get_set_flag; uint32_t mac_id_n_color; uint16_t value; uint16_t reserved; } __packed; /* IWM_NON_QOS_TX_COUNTER_GET_SET_API_S_VER_1 */ /* * END mvm/fw-api-mac.h */ /* * BEGIN mvm/fw-api-power.h */ /* Power Management Commands, Responses, Notifications */ /** * enum iwm_ltr_config_flags - masks for LTR config command flags * @IWM_LTR_CFG_FLAG_FEATURE_ENABLE: Feature operational status * @IWM_LTR_CFG_FLAG_HW_DIS_ON_SHADOW_REG_ACCESS: allow LTR change on shadow * memory access * @IWM_LTR_CFG_FLAG_HW_EN_SHRT_WR_THROUGH: allow LTR msg send on ANY LTR * reg change * @IWM_LTR_CFG_FLAG_HW_DIS_ON_D0_2_D3: allow LTR msg send on transition from * D0 to D3 * @IWM_LTR_CFG_FLAG_SW_SET_SHORT: fixed static short LTR register * @IWM_LTR_CFG_FLAG_SW_SET_LONG: fixed static short LONG register * @IWM_LTR_CFG_FLAG_DENIE_C10_ON_PD: allow going into C10 on PD */ enum iwm_ltr_config_flags { IWM_LTR_CFG_FLAG_FEATURE_ENABLE = (1 << 0), IWM_LTR_CFG_FLAG_HW_DIS_ON_SHADOW_REG_ACCESS = (1 << 1), IWM_LTR_CFG_FLAG_HW_EN_SHRT_WR_THROUGH = (1 << 2), IWM_LTR_CFG_FLAG_HW_DIS_ON_D0_2_D3 = (1 << 3), IWM_LTR_CFG_FLAG_SW_SET_SHORT = (1 << 4), IWM_LTR_CFG_FLAG_SW_SET_LONG = (1 << 5), IWM_LTR_CFG_FLAG_DENIE_C10_ON_PD = (1 << 6), }; /** * struct iwm_ltr_config_cmd_v1 - configures the LTR * @flags: See %enum iwm_ltr_config_flags */ struct iwm_ltr_config_cmd_v1 { uint32_t flags; uint32_t static_long; uint32_t static_short; } __packed; /* LTR_CAPABLE_API_S_VER_1 */ #define IWM_LTR_VALID_STATES_NUM 4 /** * struct iwm_ltr_config_cmd - configures the LTR * @flags: See %enum iwm_ltr_config_flags * @static_long: * @static_short: * @ltr_cfg_values: * @ltr_short_idle_timeout: */ struct iwm_ltr_config_cmd { uint32_t flags; uint32_t static_long; uint32_t static_short; uint32_t ltr_cfg_values[IWM_LTR_VALID_STATES_NUM]; uint32_t ltr_short_idle_timeout; } __packed; /* LTR_CAPABLE_API_S_VER_2 */ /* Radio LP RX Energy Threshold measured in dBm */ #define IWM_POWER_LPRX_RSSI_THRESHOLD 75 #define IWM_POWER_LPRX_RSSI_THRESHOLD_MAX 94 #define IWM_POWER_LPRX_RSSI_THRESHOLD_MIN 30 /** * enum iwm_scan_flags - masks for power table command flags * @IWM_POWER_FLAGS_POWER_SAVE_ENA_MSK: '1' Allow to save power by turning off * receiver and transmitter. '0' - does not allow. * @IWM_POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK: '0' Driver disables power management, * '1' Driver enables PM (use rest of parameters) * @IWM_POWER_FLAGS_SKIP_OVER_DTIM_MSK: '0' PM have to walk up every DTIM, * '1' PM could sleep over DTIM till listen Interval. * @IWM_POWER_FLAGS_SNOOZE_ENA_MSK: Enable snoozing only if uAPSD is enabled and all * access categories are both delivery and trigger enabled. * @IWM_POWER_FLAGS_BT_SCO_ENA: Enable BT SCO coex only if uAPSD and * PBW Snoozing enabled * @IWM_POWER_FLAGS_ADVANCE_PM_ENA_MSK: Advanced PM (uAPSD) enable mask * @IWM_POWER_FLAGS_LPRX_ENA_MSK: Low Power RX enable. * @IWM_POWER_FLAGS_AP_UAPSD_MISBEHAVING_ENA_MSK: AP/GO's uAPSD misbehaving * detection enablement */ enum iwm_power_flags { IWM_POWER_FLAGS_POWER_SAVE_ENA_MSK = (1 << 0), IWM_POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK = (1 << 1), IWM_POWER_FLAGS_SKIP_OVER_DTIM_MSK = (1 << 2), IWM_POWER_FLAGS_SNOOZE_ENA_MSK = (1 << 5), IWM_POWER_FLAGS_BT_SCO_ENA = (1 << 8), IWM_POWER_FLAGS_ADVANCE_PM_ENA_MSK = (1 << 9), IWM_POWER_FLAGS_LPRX_ENA_MSK = (1 << 11), IWM_POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK = (1 << 12), }; #define IWM_POWER_VEC_SIZE 5 /** * struct iwm_powertable_cmd - legacy power command. Beside old API support this * is used also with a new power API for device wide power settings. * IWM_POWER_TABLE_CMD = 0x77 (command, has simple generic response) * * @flags: Power table command flags from IWM_POWER_FLAGS_* * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. * Minimum allowed:- 3 * DTIM. Keep alive period must be * set regardless of power scheme or current power state. * FW use this value also when PM is disabled. * @rx_data_timeout: Minimum time (usec) from last Rx packet for AM to * PSM transition - legacy PM * @tx_data_timeout: Minimum time (usec) from last Tx packet for AM to * PSM transition - legacy PM * @sleep_interval: not in use * @skip_dtim_periods: Number of DTIM periods to skip if Skip over DTIM flag * is set. For example, if it is required to skip over * one DTIM, this value need to be set to 2 (DTIM periods). * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled. * Default: 80dbm */ struct iwm_powertable_cmd { /* PM_POWER_TABLE_CMD_API_S_VER_6 */ uint16_t flags; uint8_t keep_alive_seconds; uint8_t debug_flags; uint32_t rx_data_timeout; uint32_t tx_data_timeout; uint32_t sleep_interval[IWM_POWER_VEC_SIZE]; uint32_t skip_dtim_periods; uint32_t lprx_rssi_threshold; } __packed; /** * enum iwm_device_power_flags - masks for device power command flags * @IWM_DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK: '1' Allow to save power by turning off * receiver and transmitter. '0' - does not allow. */ enum iwm_device_power_flags { IWM_DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK = (1 << 0), }; /** * struct iwm_device_power_cmd - device wide power command. * IWM_DEVICE_POWER_CMD = 0x77 (command, has simple generic response) * * @flags: Power table command flags from IWM_DEVICE_POWER_FLAGS_* */ struct iwm_device_power_cmd { /* PM_POWER_TABLE_CMD_API_S_VER_6 */ uint16_t flags; uint16_t reserved; } __packed; /** * struct iwm_mac_power_cmd - New power command containing uAPSD support * IWM_MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response) * @id_and_color: MAC contex identifier * @flags: Power table command flags from POWER_FLAGS_* * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. * Minimum allowed:- 3 * DTIM. Keep alive period must be * set regardless of power scheme or current power state. * FW use this value also when PM is disabled. * @rx_data_timeout: Minimum time (usec) from last Rx packet for AM to * PSM transition - legacy PM * @tx_data_timeout: Minimum time (usec) from last Tx packet for AM to * PSM transition - legacy PM * @sleep_interval: not in use * @skip_dtim_periods: Number of DTIM periods to skip if Skip over DTIM flag * is set. For example, if it is required to skip over * one DTIM, this value need to be set to 2 (DTIM periods). * @rx_data_timeout_uapsd: Minimum time (usec) from last Rx packet for AM to * PSM transition - uAPSD * @tx_data_timeout_uapsd: Minimum time (usec) from last Tx packet for AM to * PSM transition - uAPSD * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled. * Default: 80dbm * @num_skip_dtim: Number of DTIMs to skip if Skip over DTIM flag is set * @snooze_interval: Maximum time between attempts to retrieve buffered data * from the AP [msec] * @snooze_window: A window of time in which PBW snoozing insures that all * packets received. It is also the minimum time from last * received unicast RX packet, before client stops snoozing * for data. [msec] * @snooze_step: TBD * @qndp_tid: TID client shall use for uAPSD QNDP triggers * @uapsd_ac_flags: Set trigger-enabled and delivery-enabled indication for * each corresponding AC. * Use IEEE80211_WMM_IE_STA_QOSINFO_AC* for correct values. * @uapsd_max_sp: Use IEEE80211_WMM_IE_STA_QOSINFO_SP_* for correct * values. * @heavy_tx_thld_packets: TX threshold measured in number of packets * @heavy_rx_thld_packets: RX threshold measured in number of packets * @heavy_tx_thld_percentage: TX threshold measured in load's percentage * @heavy_rx_thld_percentage: RX threshold measured in load's percentage * @limited_ps_threshold: */ struct iwm_mac_power_cmd { /* CONTEXT_DESC_API_T_VER_1 */ uint32_t id_and_color; /* CLIENT_PM_POWER_TABLE_S_VER_1 */ uint16_t flags; uint16_t keep_alive_seconds; uint32_t rx_data_timeout; uint32_t tx_data_timeout; uint32_t rx_data_timeout_uapsd; uint32_t tx_data_timeout_uapsd; uint8_t lprx_rssi_threshold; uint8_t skip_dtim_periods; uint16_t snooze_interval; uint16_t snooze_window; uint8_t snooze_step; uint8_t qndp_tid; uint8_t uapsd_ac_flags; uint8_t uapsd_max_sp; uint8_t heavy_tx_thld_packets; uint8_t heavy_rx_thld_packets; uint8_t heavy_tx_thld_percentage; uint8_t heavy_rx_thld_percentage; uint8_t limited_ps_threshold; uint8_t reserved; } __packed; /* * struct iwm_uapsd_misbehaving_ap_notif - FW sends this notification when * associated AP is identified as improperly implementing uAPSD protocol. * IWM_PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78 * @sta_id: index of station in uCode's station table - associated AP ID in * this context. */ struct iwm_uapsd_misbehaving_ap_notif { uint32_t sta_id; uint8_t mac_id; uint8_t reserved[3]; } __packed; /** * struct iwm_beacon_filter_cmd * IWM_REPLY_BEACON_FILTERING_CMD = 0xd2 (command) * @id_and_color: MAC contex identifier * @bf_energy_delta: Used for RSSI filtering, if in 'normal' state. Send beacon * to driver if delta in Energy values calculated for this and last * passed beacon is greater than this threshold. Zero value means that * the Energy change is ignored for beacon filtering, and beacon will * not be forced to be sent to driver regardless of this delta. Typical * energy delta 5dB. * @bf_roaming_energy_delta: Used for RSSI filtering, if in 'roaming' state. * Send beacon to driver if delta in Energy values calculated for this * and last passed beacon is greater than this threshold. Zero value * means that the Energy change is ignored for beacon filtering while in * Roaming state, typical energy delta 1dB. * @bf_roaming_state: Used for RSSI filtering. If absolute Energy values * calculated for current beacon is less than the threshold, use * Roaming Energy Delta Threshold, otherwise use normal Energy Delta * Threshold. Typical energy threshold is -72dBm. * @bf_temp_threshold: This threshold determines the type of temperature * filtering (Slow or Fast) that is selected (Units are in Celsuis): * If the current temperature is above this threshold - Fast filter * will be used, If the current temperature is below this threshold - * Slow filter will be used. * @bf_temp_fast_filter: Send Beacon to driver if delta in temperature values * calculated for this and the last passed beacon is greater than this * threshold. Zero value means that the temperature change is ignored for * beacon filtering; beacons will not be forced to be sent to driver * regardless of whether its temperature has been changed. * @bf_temp_slow_filter: Send Beacon to driver if delta in temperature values * calculated for this and the last passed beacon is greater than this * threshold. Zero value means that the temperature change is ignored for * beacon filtering; beacons will not be forced to be sent to driver * regardless of whether its temperature has been changed. * @bf_enable_beacon_filter: 1, beacon filtering is enabled; 0, disabled. * @bf_filter_escape_timer: Send beacons to the driver if no beacons were passed * for a specific period of time. Units: Beacons. * @ba_escape_timer: Fully receive and parse beacon if no beacons were passed * for a longer period of time then this escape-timeout. Units: Beacons. * @ba_enable_beacon_abort: 1, beacon abort is enabled; 0, disabled. */ struct iwm_beacon_filter_cmd { uint32_t bf_energy_delta; uint32_t bf_roaming_energy_delta; uint32_t bf_roaming_state; uint32_t bf_temp_threshold; uint32_t bf_temp_fast_filter; uint32_t bf_temp_slow_filter; uint32_t bf_enable_beacon_filter; uint32_t bf_debug_flag; uint32_t bf_escape_timer; uint32_t ba_escape_timer; uint32_t ba_enable_beacon_abort; } __packed; /* Beacon filtering and beacon abort */ #define IWM_BF_ENERGY_DELTA_DEFAULT 5 #define IWM_BF_ENERGY_DELTA_MAX 255 #define IWM_BF_ENERGY_DELTA_MIN 0 #define IWM_BF_ROAMING_ENERGY_DELTA_DEFAULT 1 #define IWM_BF_ROAMING_ENERGY_DELTA_MAX 255 #define IWM_BF_ROAMING_ENERGY_DELTA_MIN 0 #define IWM_BF_ROAMING_STATE_DEFAULT 72 #define IWM_BF_ROAMING_STATE_MAX 255 #define IWM_BF_ROAMING_STATE_MIN 0 #define IWM_BF_TEMP_THRESHOLD_DEFAULT 112 #define IWM_BF_TEMP_THRESHOLD_MAX 255 #define IWM_BF_TEMP_THRESHOLD_MIN 0 #define IWM_BF_TEMP_FAST_FILTER_DEFAULT 1 #define IWM_BF_TEMP_FAST_FILTER_MAX 255 #define IWM_BF_TEMP_FAST_FILTER_MIN 0 #define IWM_BF_TEMP_SLOW_FILTER_DEFAULT 5 #define IWM_BF_TEMP_SLOW_FILTER_MAX 255 #define IWM_BF_TEMP_SLOW_FILTER_MIN 0 #define IWM_BF_ENABLE_BEACON_FILTER_DEFAULT 1 #define IWM_BF_DEBUG_FLAG_DEFAULT 0 #define IWM_BF_ESCAPE_TIMER_DEFAULT 50 #define IWM_BF_ESCAPE_TIMER_MAX 1024 #define IWM_BF_ESCAPE_TIMER_MIN 0 #define IWM_BA_ESCAPE_TIMER_DEFAULT 6 #define IWM_BA_ESCAPE_TIMER_D3 9 #define IWM_BA_ESCAPE_TIMER_MAX 1024 #define IWM_BA_ESCAPE_TIMER_MIN 0 #define IWM_BA_ENABLE_BEACON_ABORT_DEFAULT 1 #define IWM_BF_CMD_CONFIG_DEFAULTS \ .bf_energy_delta = htole32(IWM_BF_ENERGY_DELTA_DEFAULT), \ .bf_roaming_energy_delta = \ htole32(IWM_BF_ROAMING_ENERGY_DELTA_DEFAULT), \ .bf_roaming_state = htole32(IWM_BF_ROAMING_STATE_DEFAULT), \ .bf_temp_threshold = htole32(IWM_BF_TEMP_THRESHOLD_DEFAULT), \ .bf_temp_fast_filter = htole32(IWM_BF_TEMP_FAST_FILTER_DEFAULT), \ .bf_temp_slow_filter = htole32(IWM_BF_TEMP_SLOW_FILTER_DEFAULT), \ .bf_debug_flag = htole32(IWM_BF_DEBUG_FLAG_DEFAULT), \ .bf_escape_timer = htole32(IWM_BF_ESCAPE_TIMER_DEFAULT), \ .ba_escape_timer = htole32(IWM_BA_ESCAPE_TIMER_DEFAULT) /* * END mvm/fw-api-power.h */ /* * BEGIN mvm/fw-api-rs.h */ /* * These serve as indexes into * struct iwm_rate_info fw_rate_idx_to_plcp[IWM_RATE_COUNT]; * TODO: avoid overlap between legacy and HT rates */ enum { IWM_RATE_1M_INDEX = 0, IWM_FIRST_CCK_RATE = IWM_RATE_1M_INDEX, IWM_RATE_2M_INDEX, IWM_RATE_5M_INDEX, IWM_RATE_11M_INDEX, IWM_LAST_CCK_RATE = IWM_RATE_11M_INDEX, IWM_RATE_6M_INDEX, IWM_FIRST_OFDM_RATE = IWM_RATE_6M_INDEX, IWM_RATE_MCS_0_INDEX = IWM_RATE_6M_INDEX, IWM_FIRST_HT_RATE = IWM_RATE_MCS_0_INDEX, IWM_FIRST_VHT_RATE = IWM_RATE_MCS_0_INDEX, IWM_RATE_9M_INDEX, IWM_RATE_12M_INDEX, IWM_RATE_MCS_1_INDEX = IWM_RATE_12M_INDEX, IWM_RATE_18M_INDEX, IWM_RATE_MCS_2_INDEX = IWM_RATE_18M_INDEX, IWM_RATE_24M_INDEX, IWM_RATE_MCS_3_INDEX = IWM_RATE_24M_INDEX, IWM_RATE_36M_INDEX, IWM_RATE_MCS_4_INDEX = IWM_RATE_36M_INDEX, IWM_RATE_48M_INDEX, IWM_RATE_MCS_5_INDEX = IWM_RATE_48M_INDEX, IWM_RATE_54M_INDEX, IWM_RATE_MCS_6_INDEX = IWM_RATE_54M_INDEX, IWM_LAST_NON_HT_RATE = IWM_RATE_54M_INDEX, IWM_RATE_60M_INDEX, IWM_RATE_MCS_7_INDEX = IWM_RATE_60M_INDEX, IWM_LAST_HT_RATE = IWM_RATE_MCS_7_INDEX, IWM_RATE_MCS_8_INDEX, IWM_RATE_MCS_9_INDEX, IWM_LAST_VHT_RATE = IWM_RATE_MCS_9_INDEX, IWM_RATE_COUNT_LEGACY = IWM_LAST_NON_HT_RATE + 1, IWM_RATE_COUNT = IWM_LAST_VHT_RATE + 1, }; #define IWM_RATE_BIT_MSK(r) (1 << (IWM_RATE_##r##M_INDEX)) /* fw API values for legacy bit rates, both OFDM and CCK */ enum { IWM_RATE_6M_PLCP = 13, IWM_RATE_9M_PLCP = 15, IWM_RATE_12M_PLCP = 5, IWM_RATE_18M_PLCP = 7, IWM_RATE_24M_PLCP = 9, IWM_RATE_36M_PLCP = 11, IWM_RATE_48M_PLCP = 1, IWM_RATE_54M_PLCP = 3, IWM_RATE_1M_PLCP = 10, IWM_RATE_2M_PLCP = 20, IWM_RATE_5M_PLCP = 55, IWM_RATE_11M_PLCP = 110, IWM_RATE_INVM_PLCP = -1, }; /* * rate_n_flags bit fields * * The 32-bit value has different layouts in the low 8 bites depending on the * format. There are three formats, HT, VHT and legacy (11abg, with subformats * for CCK and OFDM). * * High-throughput (HT) rate format * bit 8 is 1, bit 26 is 0, bit 9 is 0 (OFDM) * Very High-throughput (VHT) rate format * bit 8 is 0, bit 26 is 1, bit 9 is 0 (OFDM) * Legacy OFDM rate format for bits 7:0 * bit 8 is 0, bit 26 is 0, bit 9 is 0 (OFDM) * Legacy CCK rate format for bits 7:0: * bit 8 is 0, bit 26 is 0, bit 9 is 1 (CCK) */ /* Bit 8: (1) HT format, (0) legacy or VHT format */ #define IWM_RATE_MCS_HT_POS 8 #define IWM_RATE_MCS_HT_MSK (1 << IWM_RATE_MCS_HT_POS) /* Bit 9: (1) CCK, (0) OFDM. HT (bit 8) must be "0" for this bit to be valid */ #define IWM_RATE_MCS_CCK_POS 9 #define IWM_RATE_MCS_CCK_MSK (1 << IWM_RATE_MCS_CCK_POS) /* Bit 26: (1) VHT format, (0) legacy format in bits 8:0 */ #define IWM_RATE_MCS_VHT_POS 26 #define IWM_RATE_MCS_VHT_MSK (1 << IWM_RATE_MCS_VHT_POS) /* * High-throughput (HT) rate format for bits 7:0 * * 2-0: MCS rate base * 0) 6 Mbps * 1) 12 Mbps * 2) 18 Mbps * 3) 24 Mbps * 4) 36 Mbps * 5) 48 Mbps * 6) 54 Mbps * 7) 60 Mbps * 4-3: 0) Single stream (SISO) * 1) Dual stream (MIMO) * 2) Triple stream (MIMO) * 5: Value of 0x20 in bits 7:0 indicates 6 Mbps HT40 duplicate data * (bits 7-6 are zero) * * Together the low 5 bits work out to the MCS index because we don't * support MCSes above 15/23, and 0-7 have one stream, 8-15 have two * streams and 16-23 have three streams. We could also support MCS 32 * which is the duplicate 20 MHz MCS (bit 5 set, all others zero.) */ #define IWM_RATE_HT_MCS_RATE_CODE_MSK 0x7 #define IWM_RATE_HT_MCS_NSS_POS 3 #define IWM_RATE_HT_MCS_NSS_MSK (3 << IWM_RATE_HT_MCS_NSS_POS) /* Bit 10: (1) Use Green Field preamble */ #define IWM_RATE_HT_MCS_GF_POS 10 #define IWM_RATE_HT_MCS_GF_MSK (1 << IWM_RATE_HT_MCS_GF_POS) #define IWM_RATE_HT_MCS_INDEX_MSK 0x3f /* * Very High-throughput (VHT) rate format for bits 7:0 * * 3-0: VHT MCS (0-9) * 5-4: number of streams - 1: * 0) Single stream (SISO) * 1) Dual stream (MIMO) * 2) Triple stream (MIMO) */ /* Bit 4-5: (0) SISO, (1) MIMO2 (2) MIMO3 */ #define IWM_RATE_VHT_MCS_RATE_CODE_MSK 0xf #define IWM_RATE_VHT_MCS_NSS_POS 4 #define IWM_RATE_VHT_MCS_NSS_MSK (3 << IWM_RATE_VHT_MCS_NSS_POS) /* * Legacy OFDM rate format for bits 7:0 * * 3-0: 0xD) 6 Mbps * 0xF) 9 Mbps * 0x5) 12 Mbps * 0x7) 18 Mbps * 0x9) 24 Mbps * 0xB) 36 Mbps * 0x1) 48 Mbps * 0x3) 54 Mbps * (bits 7-4 are 0) * * Legacy CCK rate format for bits 7:0: * bit 8 is 0, bit 26 is 0, bit 9 is 1 (CCK): * * 6-0: 10) 1 Mbps * 20) 2 Mbps * 55) 5.5 Mbps * 110) 11 Mbps * (bit 7 is 0) */ #define IWM_RATE_LEGACY_RATE_MSK 0xff /* * Bit 11-12: (0) 20MHz, (1) 40MHz, (2) 80MHz, (3) 160MHz * 0 and 1 are valid for HT and VHT, 2 and 3 only for VHT */ #define IWM_RATE_MCS_CHAN_WIDTH_POS 11 #define IWM_RATE_MCS_CHAN_WIDTH_MSK (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) #define IWM_RATE_MCS_CHAN_WIDTH_20 (0 << IWM_RATE_MCS_CHAN_WIDTH_POS) #define IWM_RATE_MCS_CHAN_WIDTH_40 (1 << IWM_RATE_MCS_CHAN_WIDTH_POS) #define IWM_RATE_MCS_CHAN_WIDTH_80 (2 << IWM_RATE_MCS_CHAN_WIDTH_POS) #define IWM_RATE_MCS_CHAN_WIDTH_160 (3 << IWM_RATE_MCS_CHAN_WIDTH_POS) /* Bit 13: (1) Short guard interval (0.4 usec), (0) normal GI (0.8 usec) */ #define IWM_RATE_MCS_SGI_POS 13 #define IWM_RATE_MCS_SGI_MSK (1 << IWM_RATE_MCS_SGI_POS) /* Bit 14-16: Antenna selection (1) Ant A, (2) Ant B, (4) Ant C */ #define IWM_RATE_MCS_ANT_POS 14 #define IWM_RATE_MCS_ANT_A_MSK (1 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_B_MSK (2 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_C_MSK (4 << IWM_RATE_MCS_ANT_POS) #define IWM_RATE_MCS_ANT_AB_MSK (IWM_RATE_MCS_ANT_A_MSK | \ IWM_RATE_MCS_ANT_B_MSK) #define IWM_RATE_MCS_ANT_ABC_MSK (IWM_RATE_MCS_ANT_AB_MSK | \ IWM_RATE_MCS_ANT_C_MSK) #define IWM_RATE_MCS_ANT_MSK IWM_RATE_MCS_ANT_ABC_MSK #define IWM_RATE_MCS_ANT_NUM 3 /* Bit 17-18: (0) SS, (1) SS*2 */ #define IWM_RATE_MCS_STBC_POS 17 #define IWM_RATE_MCS_STBC_MSK (1 << IWM_RATE_MCS_STBC_POS) /* Bit 19: (0) Beamforming is off, (1) Beamforming is on */ #define IWM_RATE_MCS_BF_POS 19 #define IWM_RATE_MCS_BF_MSK (1 << IWM_RATE_MCS_BF_POS) /* Bit 20: (0) ZLF is off, (1) ZLF is on */ #define IWM_RATE_MCS_ZLF_POS 20 #define IWM_RATE_MCS_ZLF_MSK (1 << IWM_RATE_MCS_ZLF_POS) /* Bit 24-25: (0) 20MHz (no dup), (1) 2x20MHz, (2) 4x20MHz, 3 8x20MHz */ #define IWM_RATE_MCS_DUP_POS 24 #define IWM_RATE_MCS_DUP_MSK (3 << IWM_RATE_MCS_DUP_POS) /* Bit 27: (1) LDPC enabled, (0) LDPC disabled */ #define IWM_RATE_MCS_LDPC_POS 27 #define IWM_RATE_MCS_LDPC_MSK (1 << IWM_RATE_MCS_LDPC_POS) /* Link Quality definitions */ /* # entries in rate scale table to support Tx retries */ #define IWM_LQ_MAX_RETRY_NUM 16 /* Link quality command flags bit fields */ /* Bit 0: (0) Don't use RTS (1) Use RTS */ #define IWM_LQ_FLAG_USE_RTS_POS 0 #define IWM_LQ_FLAG_USE_RTS_MSK (1 << IWM_LQ_FLAG_USE_RTS_POS) /* Bit 1-3: LQ command color. Used to match responses to LQ commands */ #define IWM_LQ_FLAG_COLOR_POS 1 #define IWM_LQ_FLAG_COLOR_MSK (7 << IWM_LQ_FLAG_COLOR_POS) /* Bit 4-5: Tx RTS BW Signalling * (0) No RTS BW signalling * (1) Static BW signalling * (2) Dynamic BW signalling */ #define IWM_LQ_FLAG_RTS_BW_SIG_POS 4 #define IWM_LQ_FLAG_RTS_BW_SIG_NONE (0 << IWM_LQ_FLAG_RTS_BW_SIG_POS) #define IWM_LQ_FLAG_RTS_BW_SIG_STATIC (1 << IWM_LQ_FLAG_RTS_BW_SIG_POS) #define IWM_LQ_FLAG_RTS_BW_SIG_DYNAMIC (2 << IWM_LQ_FLAG_RTS_BW_SIG_POS) /* Bit 6: (0) No dynamic BW selection (1) Allow dynamic BW selection * Dyanmic BW selection allows Tx with narrower BW then requested in rates */ #define IWM_LQ_FLAG_DYNAMIC_BW_POS 6 #define IWM_LQ_FLAG_DYNAMIC_BW_MSK (1 << IWM_LQ_FLAG_DYNAMIC_BW_POS) /* Single Stream Tx Parameters (lq_cmd->ss_params) * Flags to control a smart FW decision about whether BFER/STBC/SISO will be * used for single stream Tx. */ /* Bit 0-1: Max STBC streams allowed. Can be 0-3. * (0) - No STBC allowed * (1) - 2x1 STBC allowed (HT/VHT) * (2) - 4x2 STBC allowed (HT/VHT) * (3) - 3x2 STBC allowed (HT only) * All our chips are at most 2 antennas so only (1) is valid for now. */ #define IWM_LQ_SS_STBC_ALLOWED_POS 0 #define IWM_LQ_SS_STBC_ALLOWED_MSK (3 << IWM_LQ_SS_STBC_ALLOWED_MSK) /* 2x1 STBC is allowed */ #define IWM_LQ_SS_STBC_1SS_ALLOWED (1 << IWM_LQ_SS_STBC_ALLOWED_POS) /* Bit 2: Beamformer (VHT only) is allowed */ #define IWM_LQ_SS_BFER_ALLOWED_POS 2 #define IWM_LQ_SS_BFER_ALLOWED (1 << IWM_LQ_SS_BFER_ALLOWED_POS) /* Bit 3: Force BFER or STBC for testing * If this is set: * If BFER is allowed then force the ucode to choose BFER else * If STBC is allowed then force the ucode to choose STBC over SISO */ #define IWM_LQ_SS_FORCE_POS 3 #define IWM_LQ_SS_FORCE (1 << IWM_LQ_SS_FORCE_POS) /* Bit 31: ss_params field is valid. Used for FW backward compatibility * with other drivers which don't support the ss_params API yet */ #define IWM_LQ_SS_PARAMS_VALID_POS 31 #define IWM_LQ_SS_PARAMS_VALID (1 << IWM_LQ_SS_PARAMS_VALID_POS) /** * struct iwm_lq_cmd - link quality command * @sta_id: station to update * @control: not used * @flags: combination of IWM_LQ_FLAG_* * @mimo_delim: the first SISO index in rs_table, which separates MIMO * and SISO rates * @single_stream_ant_msk: best antenna for SISO (can be dual in CDD). * Should be ANT_[ABC] * @dual_stream_ant_msk: best antennas for MIMO, combination of ANT_[ABC] * @initial_rate_index: first index from rs_table per AC category * @agg_time_limit: aggregation max time threshold in usec/100, meaning * value of 100 is one usec. Range is 100 to 8000 * @agg_disable_start_th: try-count threshold for starting aggregation. * If a frame has higher try-count, it should not be selected for * starting an aggregation sequence. * @agg_frame_cnt_limit: max frame count in an aggregation. * 0: no limit * 1: no aggregation (one frame per aggregation) * 2 - 0x3f: maximal number of frames (up to 3f == 63) * @rs_table: array of rates for each TX try, each is rate_n_flags, * meaning it is a combination of IWM_RATE_MCS_* and IWM_RATE_*_PLCP * @ss_params: single stream features. declare whether STBC or BFER are allowed. */ struct iwm_lq_cmd { uint8_t sta_id; uint8_t reduced_tpc; uint16_t control; /* LINK_QUAL_GENERAL_PARAMS_API_S_VER_1 */ uint8_t flags; uint8_t mimo_delim; uint8_t single_stream_ant_msk; uint8_t dual_stream_ant_msk; uint8_t initial_rate_index[IWM_AC_NUM]; /* LINK_QUAL_AGG_PARAMS_API_S_VER_1 */ uint16_t agg_time_limit; uint8_t agg_disable_start_th; uint8_t agg_frame_cnt_limit; uint32_t reserved2; uint32_t rs_table[IWM_LQ_MAX_RETRY_NUM]; uint32_t ss_params; }; /* LINK_QUALITY_CMD_API_S_VER_1 */ /* * END mvm/fw-api-rs.h */ /* * BEGIN mvm/fw-api-tx.h */ /** * enum iwm_tx_flags - bitmasks for tx_flags in TX command * @IWM_TX_CMD_FLG_PROT_REQUIRE: use RTS or CTS-to-self to protect the frame * @IWM_TX_CMD_FLG_ACK: expect ACK from receiving station * @IWM_TX_CMD_FLG_STA_RATE: use RS table with initial index from the TX command. * Otherwise, use rate_n_flags from the TX command * @IWM_TX_CMD_FLG_BA: this frame is a block ack * @IWM_TX_CMD_FLG_BAR: this frame is a BA request, immediate BAR is expected * Must set IWM_TX_CMD_FLG_ACK with this flag. * @IWM_TX_CMD_FLG_TXOP_PROT: protect frame with full TXOP protection * @IWM_TX_CMD_FLG_VHT_NDPA: mark frame is NDPA for VHT beamformer sequence * @IWM_TX_CMD_FLG_HT_NDPA: mark frame is NDPA for HT beamformer sequence * @IWM_TX_CMD_FLG_CSI_FDBK2HOST: mark to send feedback to host (only if good CRC) * @IWM_TX_CMD_FLG_BT_DIS: disable BT priority for this frame * @IWM_TX_CMD_FLG_SEQ_CTL: set if FW should override the sequence control. * Should be set for mgmt, non-QOS data, mcast, bcast and in scan command * @IWM_TX_CMD_FLG_MORE_FRAG: this frame is non-last MPDU * @IWM_TX_CMD_FLG_NEXT_FRAME: this frame includes information of the next frame * @IWM_TX_CMD_FLG_TSF: FW should calculate and insert TSF in the frame * Should be set for beacons and probe responses * @IWM_TX_CMD_FLG_CALIB: activate PA TX power calibrations * @IWM_TX_CMD_FLG_KEEP_SEQ_CTL: if seq_ctl is set, don't increase inner seq count * @IWM_TX_CMD_FLG_AGG_START: allow this frame to start aggregation * @IWM_TX_CMD_FLG_MH_PAD: driver inserted 2 byte padding after MAC header. * Should be set for 26/30 length MAC headers * @IWM_TX_CMD_FLG_RESP_TO_DRV: zero this if the response should go only to FW * @IWM_TX_CMD_FLG_TKIP_MIC_DONE: FW already performed TKIP MIC calculation * @IWM_TX_CMD_FLG_DUR: disable duration overwriting used in PS-Poll Assoc-id * @IWM_TX_CMD_FLG_FW_DROP: FW should mark frame to be dropped * @IWM_TX_CMD_FLG_EXEC_PAPD: execute PAPD * @IWM_TX_CMD_FLG_PAPD_TYPE: 0 for reference power, 1 for nominal power * @IWM_TX_CMD_FLG_HCCA_CHUNK: mark start of TSPEC chunk */ enum iwm_tx_flags { IWM_TX_CMD_FLG_PROT_REQUIRE = (1 << 0), IWM_TX_CMD_FLG_ACK = (1 << 3), IWM_TX_CMD_FLG_STA_RATE = (1 << 4), IWM_TX_CMD_FLG_BA = (1 << 5), IWM_TX_CMD_FLG_BAR = (1 << 6), IWM_TX_CMD_FLG_TXOP_PROT = (1 << 7), IWM_TX_CMD_FLG_VHT_NDPA = (1 << 8), IWM_TX_CMD_FLG_HT_NDPA = (1 << 9), IWM_TX_CMD_FLG_CSI_FDBK2HOST = (1 << 10), IWM_TX_CMD_FLG_BT_DIS = (1 << 12), IWM_TX_CMD_FLG_SEQ_CTL = (1 << 13), IWM_TX_CMD_FLG_MORE_FRAG = (1 << 14), IWM_TX_CMD_FLG_NEXT_FRAME = (1 << 15), IWM_TX_CMD_FLG_TSF = (1 << 16), IWM_TX_CMD_FLG_CALIB = (1 << 17), IWM_TX_CMD_FLG_KEEP_SEQ_CTL = (1 << 18), IWM_TX_CMD_FLG_AGG_START = (1 << 19), IWM_TX_CMD_FLG_MH_PAD = (1 << 20), IWM_TX_CMD_FLG_RESP_TO_DRV = (1 << 21), IWM_TX_CMD_FLG_TKIP_MIC_DONE = (1 << 23), IWM_TX_CMD_FLG_DUR = (1 << 25), IWM_TX_CMD_FLG_FW_DROP = (1 << 26), IWM_TX_CMD_FLG_EXEC_PAPD = (1 << 27), IWM_TX_CMD_FLG_PAPD_TYPE = (1 << 28), IWM_TX_CMD_FLG_HCCA_CHUNK = (1 << 31) }; /* IWM_TX_FLAGS_BITS_API_S_VER_1 */ /** * enum iwm_tx_pm_timeouts - pm timeout values in TX command * @IWM_PM_FRAME_NONE: no need to suspend sleep mode * @IWM_PM_FRAME_MGMT: fw suspend sleep mode for 100TU * @IWM_PM_FRAME_ASSOC: fw suspend sleep mode for 10sec */ enum iwm_tx_pm_timeouts { IWM_PM_FRAME_NONE = 0, IWM_PM_FRAME_MGMT = 2, IWM_PM_FRAME_ASSOC = 3, }; /* * TX command security control */ #define IWM_TX_CMD_SEC_WEP 0x01 #define IWM_TX_CMD_SEC_CCM 0x02 #define IWM_TX_CMD_SEC_TKIP 0x03 #define IWM_TX_CMD_SEC_EXT 0x04 #define IWM_TX_CMD_SEC_MSK 0x07 #define IWM_TX_CMD_SEC_WEP_KEY_IDX_POS 6 #define IWM_TX_CMD_SEC_WEP_KEY_IDX_MSK 0xc0 #define IWM_TX_CMD_SEC_KEY128 0x08 /* * TX command Frame life time in us - to be written in pm_frame_timeout */ #define IWM_TX_CMD_LIFE_TIME_INFINITE 0xFFFFFFFF #define IWM_TX_CMD_LIFE_TIME_DEFAULT 2000000 /* 2000 ms*/ #define IWM_TX_CMD_LIFE_TIME_PROBE_RESP 40000 /* 40 ms */ #define IWM_TX_CMD_LIFE_TIME_EXPIRED_FRAME 0 /* * TID for non QoS frames - to be written in tid_tspec */ #define IWM_TID_NON_QOS IWM_MAX_TID_COUNT /* * Limits on the retransmissions - to be written in {data,rts}_retry_limit */ #define IWM_DEFAULT_TX_RETRY 15 #define IWM_MGMT_DFAULT_RETRY_LIMIT 3 #define IWM_RTS_DFAULT_RETRY_LIMIT 60 #define IWM_BAR_DFAULT_RETRY_LIMIT 60 #define IWM_LOW_RETRY_LIMIT 7 /* TODO: complete documentation for try_cnt and btkill_cnt */ /** * struct iwm_tx_cmd - TX command struct to FW * ( IWM_TX_CMD = 0x1c ) * @len: in bytes of the payload, see below for details * @next_frame_len: same as len, but for next frame (0 if not applicable) * Used for fragmentation and bursting, but not in 11n aggregation. * @tx_flags: combination of IWM_TX_CMD_FLG_* * @rate_n_flags: rate for *all* Tx attempts, if IWM_TX_CMD_FLG_STA_RATE_MSK is * cleared. Combination of IWM_RATE_MCS_* * @sta_id: index of destination station in FW station table * @sec_ctl: security control, IWM_TX_CMD_SEC_* * @initial_rate_index: index into the rate table for initial TX attempt. * Applied if IWM_TX_CMD_FLG_STA_RATE_MSK is set, normally 0 for data frames. * @key: security key * @reserved3: reserved * @life_time: frame life time (usecs??) * @dram_lsb_ptr: Physical address of scratch area in the command (try_cnt + * btkill_cnd + reserved), first 32 bits. "0" disables usage. * @dram_msb_ptr: upper bits of the scratch physical address * @rts_retry_limit: max attempts for RTS * @data_retry_limit: max attempts to send the data packet * @tid_spec: TID/tspec * @pm_frame_timeout: PM TX frame timeout * @driver_txop: duration od EDCA TXOP, in 32-usec units. Set this if not * specified by HCCA protocol * * The byte count (both len and next_frame_len) includes MAC header * (24/26/30/32 bytes) * + 2 bytes pad if 26/30 header size * + 8 byte IV for CCM or TKIP (not used for WEP) * + Data payload * + 8-byte MIC (not used for CCM/WEP) * It does not include post-MAC padding, i.e., * MIC (CCM) 8 bytes, ICV (WEP/TKIP/CKIP) 4 bytes, CRC 4 bytes. * Range of len: 14-2342 bytes. * * After the struct fields the MAC header is placed, plus any padding, * and then the actial payload. */ struct iwm_tx_cmd { uint16_t len; uint16_t next_frame_len; uint32_t tx_flags; struct { uint8_t try_cnt; uint8_t btkill_cnt; uint16_t reserved; } scratch; /* DRAM_SCRATCH_API_U_VER_1 */ uint32_t rate_n_flags; uint8_t sta_id; uint8_t sec_ctl; uint8_t initial_rate_index; uint8_t reserved2; uint8_t key[16]; uint16_t next_frame_flags; uint16_t reserved3; uint32_t life_time; uint32_t dram_lsb_ptr; uint8_t dram_msb_ptr; uint8_t rts_retry_limit; uint8_t data_retry_limit; uint8_t tid_tspec; uint16_t pm_frame_timeout; uint16_t driver_txop; uint8_t payload[0]; struct ieee80211_frame hdr[0]; } __packed; /* IWM_TX_CMD_API_S_VER_3 */ /* * TX response related data */ /* * enum iwm_tx_status - status that is returned by the fw after attempts to Tx * @IWM_TX_STATUS_SUCCESS: * @IWM_TX_STATUS_DIRECT_DONE: * @IWM_TX_STATUS_POSTPONE_DELAY: * @IWM_TX_STATUS_POSTPONE_FEW_BYTES: * @IWM_TX_STATUS_POSTPONE_BT_PRIO: * @IWM_TX_STATUS_POSTPONE_QUIET_PERIOD: * @IWM_TX_STATUS_POSTPONE_CALC_TTAK: * @IWM_TX_STATUS_FAIL_INTERNAL_CROSSED_RETRY: * @IWM_TX_STATUS_FAIL_SHORT_LIMIT: * @IWM_TX_STATUS_FAIL_LONG_LIMIT: * @IWM_TX_STATUS_FAIL_UNDERRUN: * @IWM_TX_STATUS_FAIL_DRAIN_FLOW: * @IWM_TX_STATUS_FAIL_RFKILL_FLUSH: * @IWM_TX_STATUS_FAIL_LIFE_EXPIRE: * @IWM_TX_STATUS_FAIL_DEST_PS: * @IWM_TX_STATUS_FAIL_HOST_ABORTED: * @IWM_TX_STATUS_FAIL_BT_RETRY: * @IWM_TX_STATUS_FAIL_STA_INVALID: * @IWM_TX_TATUS_FAIL_FRAG_DROPPED: * @IWM_TX_STATUS_FAIL_TID_DISABLE: * @IWM_TX_STATUS_FAIL_FIFO_FLUSHED: * @IWM_TX_STATUS_FAIL_SMALL_CF_POLL: * @IWM_TX_STATUS_FAIL_FW_DROP: * @IWM_TX_STATUS_FAIL_STA_COLOR_MISMATCH: mismatch between color of Tx cmd and * STA table * @IWM_TX_FRAME_STATUS_INTERNAL_ABORT: * @IWM_TX_MODE_MSK: * @IWM_TX_MODE_NO_BURST: * @IWM_TX_MODE_IN_BURST_SEQ: * @IWM_TX_MODE_FIRST_IN_BURST: * @IWM_TX_QUEUE_NUM_MSK: * * Valid only if frame_count =1 * TODO: complete documentation */ enum iwm_tx_status { IWM_TX_STATUS_MSK = 0x000000ff, IWM_TX_STATUS_SUCCESS = 0x01, IWM_TX_STATUS_DIRECT_DONE = 0x02, /* postpone TX */ IWM_TX_STATUS_POSTPONE_DELAY = 0x40, IWM_TX_STATUS_POSTPONE_FEW_BYTES = 0x41, IWM_TX_STATUS_POSTPONE_BT_PRIO = 0x42, IWM_TX_STATUS_POSTPONE_QUIET_PERIOD = 0x43, IWM_TX_STATUS_POSTPONE_CALC_TTAK = 0x44, /* abort TX */ IWM_TX_STATUS_FAIL_INTERNAL_CROSSED_RETRY = 0x81, IWM_TX_STATUS_FAIL_SHORT_LIMIT = 0x82, IWM_TX_STATUS_FAIL_LONG_LIMIT = 0x83, IWM_TX_STATUS_FAIL_UNDERRUN = 0x84, IWM_TX_STATUS_FAIL_DRAIN_FLOW = 0x85, IWM_TX_STATUS_FAIL_RFKILL_FLUSH = 0x86, IWM_TX_STATUS_FAIL_LIFE_EXPIRE = 0x87, IWM_TX_STATUS_FAIL_DEST_PS = 0x88, IWM_TX_STATUS_FAIL_HOST_ABORTED = 0x89, IWM_TX_STATUS_FAIL_BT_RETRY = 0x8a, IWM_TX_STATUS_FAIL_STA_INVALID = 0x8b, IWM_TX_STATUS_FAIL_FRAG_DROPPED = 0x8c, IWM_TX_STATUS_FAIL_TID_DISABLE = 0x8d, IWM_TX_STATUS_FAIL_FIFO_FLUSHED = 0x8e, IWM_TX_STATUS_FAIL_SMALL_CF_POLL = 0x8f, IWM_TX_STATUS_FAIL_FW_DROP = 0x90, IWM_TX_STATUS_FAIL_STA_COLOR_MISMATCH = 0x91, IWM_TX_STATUS_INTERNAL_ABORT = 0x92, IWM_TX_MODE_MSK = 0x00000f00, IWM_TX_MODE_NO_BURST = 0x00000000, IWM_TX_MODE_IN_BURST_SEQ = 0x00000100, IWM_TX_MODE_FIRST_IN_BURST = 0x00000200, IWM_TX_QUEUE_NUM_MSK = 0x0001f000, IWM_TX_NARROW_BW_MSK = 0x00060000, IWM_TX_NARROW_BW_1DIV2 = 0x00020000, IWM_TX_NARROW_BW_1DIV4 = 0x00040000, IWM_TX_NARROW_BW_1DIV8 = 0x00060000, }; /* * enum iwm_tx_agg_status - TX aggregation status * @IWM_AGG_TX_STATE_STATUS_MSK: * @IWM_AGG_TX_STATE_TRANSMITTED: * @IWM_AGG_TX_STATE_UNDERRUN: * @IWM_AGG_TX_STATE_BT_PRIO: * @IWM_AGG_TX_STATE_FEW_BYTES: * @IWM_AGG_TX_STATE_ABORT: * @IWM_AGG_TX_STATE_LAST_SENT_TTL: * @IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT: * @IWM_AGG_TX_STATE_LAST_SENT_BT_KILL: * @IWM_AGG_TX_STATE_SCD_QUERY: * @IWM_AGG_TX_STATE_TEST_BAD_CRC32: * @IWM_AGG_TX_STATE_RESPONSE: * @IWM_AGG_TX_STATE_DUMP_TX: * @IWM_AGG_TX_STATE_DELAY_TX: * @IWM_AGG_TX_STATE_TRY_CNT_MSK: Retry count for 1st frame in aggregation (retries * occur if tx failed for this frame when it was a member of a previous * aggregation block). If rate scaling is used, retry count indicates the * rate table entry used for all frames in the new agg. *@ IWM_AGG_TX_STATE_SEQ_NUM_MSK: Command ID and sequence number of Tx command for * this frame * * TODO: complete documentation */ enum iwm_tx_agg_status { IWM_AGG_TX_STATE_STATUS_MSK = 0x00fff, IWM_AGG_TX_STATE_TRANSMITTED = 0x000, IWM_AGG_TX_STATE_UNDERRUN = 0x001, IWM_AGG_TX_STATE_BT_PRIO = 0x002, IWM_AGG_TX_STATE_FEW_BYTES = 0x004, IWM_AGG_TX_STATE_ABORT = 0x008, IWM_AGG_TX_STATE_LAST_SENT_TTL = 0x010, IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT = 0x020, IWM_AGG_TX_STATE_LAST_SENT_BT_KILL = 0x040, IWM_AGG_TX_STATE_SCD_QUERY = 0x080, IWM_AGG_TX_STATE_TEST_BAD_CRC32 = 0x0100, IWM_AGG_TX_STATE_RESPONSE = 0x1ff, IWM_AGG_TX_STATE_DUMP_TX = 0x200, IWM_AGG_TX_STATE_DELAY_TX = 0x400, IWM_AGG_TX_STATE_TRY_CNT_POS = 12, IWM_AGG_TX_STATE_TRY_CNT_MSK = 0xf << IWM_AGG_TX_STATE_TRY_CNT_POS, }; #define IWM_AGG_TX_STATE_LAST_SENT_MSK (IWM_AGG_TX_STATE_LAST_SENT_TTL| \ IWM_AGG_TX_STATE_LAST_SENT_TRY_CNT| \ IWM_AGG_TX_STATE_LAST_SENT_BT_KILL) /* * The mask below describes a status where we are absolutely sure that the MPDU * wasn't sent. For BA/Underrun we cannot be that sure. All we know that we've * written the bytes to the TXE, but we know nothing about what the DSP did. */ #define IWM_AGG_TX_STAT_FRAME_NOT_SENT (IWM_AGG_TX_STATE_FEW_BYTES | \ IWM_AGG_TX_STATE_ABORT | \ IWM_AGG_TX_STATE_SCD_QUERY) /* * IWM_REPLY_TX = 0x1c (response) * * This response may be in one of two slightly different formats, indicated * by the frame_count field: * * 1) No aggregation (frame_count == 1). This reports Tx results for a single * frame. Multiple attempts, at various bit rates, may have been made for * this frame. * * 2) Aggregation (frame_count > 1). This reports Tx results for two or more * frames that used block-acknowledge. All frames were transmitted at * same rate. Rate scaling may have been used if first frame in this new * agg block failed in previous agg block(s). * * Note that, for aggregation, ACK (block-ack) status is not delivered * here; block-ack has not been received by the time the device records * this status. * This status relates to reasons the tx might have been blocked or aborted * within the device, rather than whether it was received successfully by * the destination station. */ /** * struct iwm_agg_tx_status - per packet TX aggregation status * @status: enum iwm_tx_agg_status * @sequence: Sequence # for this frame's Tx cmd (not SSN!) */ struct iwm_agg_tx_status { uint16_t status; uint16_t sequence; } __packed; /* * definitions for initial rate index field * bits [3:0] initial rate index * bits [6:4] rate table color, used for the initial rate * bit-7 invalid rate indication */ #define IWM_TX_RES_INIT_RATE_INDEX_MSK 0x0f #define IWM_TX_RES_RATE_TABLE_COLOR_MSK 0x70 #define IWM_TX_RES_INV_RATE_INDEX_MSK 0x80 #define IWM_MVM_TX_RES_GET_TID(_ra_tid) ((_ra_tid) & 0x0f) #define IWM_MVM_TX_RES_GET_RA(_ra_tid) ((_ra_tid) >> 4) /** * struct iwm_mvm_tx_resp - notifies that fw is TXing a packet * ( IWM_REPLY_TX = 0x1c ) * @frame_count: 1 no aggregation, >1 aggregation * @bt_kill_count: num of times blocked by bluetooth (unused for agg) * @failure_rts: num of failures due to unsuccessful RTS * @failure_frame: num failures due to no ACK (unused for agg) * @initial_rate: for non-agg: rate of the successful Tx. For agg: rate of the * Tx of all the batch. IWM_RATE_MCS_* * @wireless_media_time: for non-agg: RTS + CTS + frame tx attempts time + ACK. * for agg: RTS + CTS + aggregation tx time + block-ack time. * in usec. * @pa_status: tx power info * @pa_integ_res_a: tx power info * @pa_integ_res_b: tx power info * @pa_integ_res_c: tx power info * @measurement_req_id: tx power info * @tfd_info: TFD information set by the FH * @seq_ctl: sequence control from the Tx cmd * @byte_cnt: byte count from the Tx cmd * @tlc_info: TLC rate info * @ra_tid: bits [3:0] = ra, bits [7:4] = tid * @frame_ctrl: frame control * @status: for non-agg: frame status IWM_TX_STATUS_* * for agg: status of 1st frame, IWM_AGG_TX_STATE_*; other frame status fields * follow this one, up to frame_count. * * After the array of statuses comes the SSN of the SCD. Look at * %iwm_mvm_get_scd_ssn for more details. */ struct iwm_mvm_tx_resp { uint8_t frame_count; uint8_t bt_kill_count; uint8_t failure_rts; uint8_t failure_frame; uint32_t initial_rate; uint16_t wireless_media_time; uint8_t pa_status; uint8_t pa_integ_res_a[3]; uint8_t pa_integ_res_b[3]; uint8_t pa_integ_res_c[3]; uint16_t measurement_req_id; uint8_t reduced_tpc; uint8_t reserved; uint32_t tfd_info; uint16_t seq_ctl; uint16_t byte_cnt; uint8_t tlc_info; uint8_t ra_tid; uint16_t frame_ctrl; struct iwm_agg_tx_status status; } __packed; /* IWM_TX_RSP_API_S_VER_3 */ /** * struct iwm_mvm_ba_notif - notifies about reception of BA * ( IWM_BA_NOTIF = 0xc5 ) * @sta_addr_lo32: lower 32 bits of the MAC address * @sta_addr_hi16: upper 16 bits of the MAC address * @sta_id: Index of recipient (BA-sending) station in fw's station table * @tid: tid of the session * @seq_ctl: * @bitmap: the bitmap of the BA notification as seen in the air * @scd_flow: the tx queue this BA relates to * @scd_ssn: the index of the last contiguously sent packet * @txed: number of Txed frames in this batch * @txed_2_done: number of Acked frames in this batch */ struct iwm_mvm_ba_notif { uint32_t sta_addr_lo32; uint16_t sta_addr_hi16; uint16_t reserved; uint8_t sta_id; uint8_t tid; uint16_t seq_ctl; uint64_t bitmap; uint16_t scd_flow; uint16_t scd_ssn; uint8_t txed; uint8_t txed_2_done; uint16_t reserved1; } __packed; /* * struct iwm_mac_beacon_cmd - beacon template command * @tx: the tx commands associated with the beacon frame * @template_id: currently equal to the mac context id of the coresponding * mac. * @tim_idx: the offset of the tim IE in the beacon * @tim_size: the length of the tim IE * @frame: the template of the beacon frame */ struct iwm_mac_beacon_cmd { struct iwm_tx_cmd tx; uint32_t template_id; uint32_t tim_idx; uint32_t tim_size; struct ieee80211_frame frame[0]; } __packed; struct iwm_beacon_notif { struct iwm_mvm_tx_resp beacon_notify_hdr; uint64_t tsf; uint32_t ibss_mgr_status; } __packed; /** * enum iwm_dump_control - dump (flush) control flags * @IWM_DUMP_TX_FIFO_FLUSH: Dump MSDUs until the FIFO is empty * and the TFD queues are empty. */ enum iwm_dump_control { IWM_DUMP_TX_FIFO_FLUSH = (1 << 1), }; /** * struct iwm_tx_path_flush_cmd -- queue/FIFO flush command * @queues_ctl: bitmap of queues to flush * @flush_ctl: control flags * @reserved: reserved */ struct iwm_tx_path_flush_cmd { uint32_t queues_ctl; uint16_t flush_ctl; uint16_t reserved; } __packed; /* IWM_TX_PATH_FLUSH_CMD_API_S_VER_1 */ /** * iwm_mvm_get_scd_ssn - returns the SSN of the SCD * @tx_resp: the Tx response from the fw (agg or non-agg) * * When the fw sends an AMPDU, it fetches the MPDUs one after the other. Since * it can't know that everything will go well until the end of the AMPDU, it * can't know in advance the number of MPDUs that will be sent in the current * batch. This is why it writes the agg Tx response while it fetches the MPDUs. * Hence, it can't know in advance what the SSN of the SCD will be at the end * of the batch. This is why the SSN of the SCD is written at the end of the * whole struct at a variable offset. This function knows how to cope with the * variable offset and returns the SSN of the SCD. */ static inline uint32_t iwm_mvm_get_scd_ssn(struct iwm_mvm_tx_resp *tx_resp) { return le32_to_cpup((uint32_t *)&tx_resp->status + tx_resp->frame_count) & 0xfff; } /* * END mvm/fw-api-tx.h */ /* * BEGIN mvm/fw-api-scan.h */ /** * struct iwm_scd_txq_cfg_cmd - New txq hw scheduler config command * @token: * @sta_id: station id * @tid: * @scd_queue: scheduler queue to confiug * @enable: 1 queue enable, 0 queue disable * @aggregate: 1 aggregated queue, 0 otherwise * @tx_fifo: %enum iwm_mvm_tx_fifo * @window: BA window size * @ssn: SSN for the BA agreement */ struct iwm_scd_txq_cfg_cmd { uint8_t token; uint8_t sta_id; uint8_t tid; uint8_t scd_queue; uint8_t enable; uint8_t aggregate; uint8_t tx_fifo; uint8_t window; uint16_t ssn; uint16_t reserved; } __packed; /* SCD_QUEUE_CFG_CMD_API_S_VER_1 */ /** * struct iwm_scd_txq_cfg_rsp * @token: taken from the command * @sta_id: station id from the command * @tid: tid from the command * @scd_queue: scd_queue from the command */ struct iwm_scd_txq_cfg_rsp { uint8_t token; uint8_t sta_id; uint8_t tid; uint8_t scd_queue; } __packed; /* SCD_QUEUE_CFG_RSP_API_S_VER_1 */ /* Scan Commands, Responses, Notifications */ /* Masks for iwm_scan_channel.type flags */ #define IWM_SCAN_CHANNEL_TYPE_ACTIVE (1 << 0) #define IWM_SCAN_CHANNEL_NSSIDS(x) (((1 << (x)) - 1) << 1) /* Max number of IEs for direct SSID scans in a command */ #define IWM_PROBE_OPTION_MAX 20 /** * struct iwm_ssid_ie - directed scan network information element * * Up to 20 of these may appear in IWM_REPLY_SCAN_CMD, * selected by "type" bit field in struct iwm_scan_channel; * each channel may select different ssids from among the 20 entries. * SSID IEs get transmitted in reverse order of entry. */ struct iwm_ssid_ie { uint8_t id; uint8_t len; uint8_t ssid[IEEE80211_NWID_LEN]; } __packed; /* IWM_SCAN_DIRECT_SSID_IE_API_S_VER_1 */ /* scan offload */ #define IWM_SCAN_MAX_BLACKLIST_LEN 64 #define IWM_SCAN_SHORT_BLACKLIST_LEN 16 #define IWM_SCAN_MAX_PROFILES 11 #define IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE 512 /* Default watchdog (in MS) for scheduled scan iteration */ #define IWM_SCHED_SCAN_WATCHDOG cpu_to_le16(15000) #define IWM_GOOD_CRC_TH_DEFAULT cpu_to_le16(1) #define IWM_CAN_ABORT_STATUS 1 #define IWM_FULL_SCAN_MULTIPLIER 5 #define IWM_FAST_SCHED_SCAN_ITERATIONS 3 #define IWM_MAX_SCHED_SCAN_PLANS 2 /** * iwm_scan_schedule_lmac - schedule of scan offload * @delay: delay between iterations, in seconds. * @iterations: num of scan iterations * @full_scan_mul: number of partial scans before each full scan */ struct iwm_scan_schedule_lmac { uint16_t delay; uint8_t iterations; uint8_t full_scan_mul; } __packed; /* SCAN_SCHEDULE_API_S */ /** * iwm_scan_req_tx_cmd - SCAN_REQ_TX_CMD_API_S * @tx_flags: combination of TX_CMD_FLG_* * @rate_n_flags: rate for *all* Tx attempts, if TX_CMD_FLG_STA_RATE_MSK is * cleared. Combination of RATE_MCS_* * @sta_id: index of destination station in FW station table * @reserved: for alignment and future use */ struct iwm_scan_req_tx_cmd { uint32_t tx_flags; uint32_t rate_n_flags; uint8_t sta_id; uint8_t reserved[3]; } __packed; enum iwm_scan_channel_flags_lmac { IWM_UNIFIED_SCAN_CHANNEL_FULL = (1 << 27), IWM_UNIFIED_SCAN_CHANNEL_PARTIAL = (1 << 28), }; /** * iwm_scan_channel_cfg_lmac - SCAN_CHANNEL_CFG_S_VER2 * @flags: bits 1-20: directed scan to i'th ssid * other bits &enum iwm_scan_channel_flags_lmac * @channel_number: channel number 1-13 etc * @iter_count: scan iteration on this channel * @iter_interval: interval in seconds between iterations on one channel */ struct iwm_scan_channel_cfg_lmac { uint32_t flags; uint16_t channel_num; uint16_t iter_count; uint32_t iter_interval; } __packed; /* * iwm_scan_probe_segment - PROBE_SEGMENT_API_S_VER_1 * @offset: offset in the data block * @len: length of the segment */ struct iwm_scan_probe_segment { uint16_t offset; uint16_t len; } __packed; /* iwm_scan_probe_req - PROBE_REQUEST_FRAME_API_S_VER_2 * @mac_header: first (and common) part of the probe * @band_data: band specific data * @common_data: last (and common) part of the probe * @buf: raw data block */ struct iwm_scan_probe_req { struct iwm_scan_probe_segment mac_header; struct iwm_scan_probe_segment band_data[2]; struct iwm_scan_probe_segment common_data; uint8_t buf[IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE]; } __packed; enum iwm_scan_channel_flags { IWM_SCAN_CHANNEL_FLAG_EBS = (1 << 0), IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE = (1 << 1), IWM_SCAN_CHANNEL_FLAG_CACHE_ADD = (1 << 2), }; /* iwm_scan_channel_opt - CHANNEL_OPTIMIZATION_API_S * @flags: enum iwm_scan_channel_flags * @non_ebs_ratio: defines the ratio of number of scan iterations where EBS is * involved. * 1 - EBS is disabled. * 2 - every second scan will be full scan(and so on). */ struct iwm_scan_channel_opt { uint16_t flags; uint16_t non_ebs_ratio; } __packed; /** * iwm_mvm_lmac_scan_flags * @IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL: pass all beacons and probe responses * without filtering. * @IWM_MVM_LMAC_SCAN_FLAG_PASSIVE: force passive scan on all channels * @IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION: single channel scan * @IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE: send iteration complete notification * @IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS multiple SSID matching * @IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED: all passive scans will be fragmented * @IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED: insert WFA vendor-specific TPC report * and DS parameter set IEs into probe requests. * @IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL: use extended dwell time on channels * 1, 6 and 11. * @IWM_MVM_LMAC_SCAN_FLAG_MATCH: Send match found notification on matches */ enum iwm_mvm_lmac_scan_flags { IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL = (1 << 0), IWM_MVM_LMAC_SCAN_FLAG_PASSIVE = (1 << 1), IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION = (1 << 2), IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE = (1 << 3), IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS = (1 << 4), IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED = (1 << 5), IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED = (1 << 6), IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL = (1 << 7), IWM_MVM_LMAC_SCAN_FLAG_MATCH = (1 << 9), }; enum iwm_scan_priority { IWM_SCAN_PRIORITY_LOW, IWM_SCAN_PRIORITY_MEDIUM, IWM_SCAN_PRIORITY_HIGH, }; /** * iwm_scan_req_lmac - SCAN_REQUEST_CMD_API_S_VER_1 * @reserved1: for alignment and future use * @channel_num: num of channels to scan * @active-dwell: dwell time for active channels * @passive-dwell: dwell time for passive channels * @fragmented-dwell: dwell time for fragmented passive scan * @extended_dwell: dwell time for channels 1, 6 and 11 (in certain cases) * @reserved2: for alignment and future use * @rx_chain_selct: PHY_RX_CHAIN_* flags * @scan_flags: &enum iwm_mvm_lmac_scan_flags * @max_out_time: max time (in TU) to be out of associated channel * @suspend_time: pause scan this long (TUs) when returning to service channel * @flags: RXON flags * @filter_flags: RXON filter * @tx_cmd: tx command for active scan; for 2GHz and for 5GHz * @direct_scan: list of SSIDs for directed active scan * @scan_prio: enum iwm_scan_priority * @iter_num: number of scan iterations * @delay: delay in seconds before first iteration * @schedule: two scheduling plans. The first one is finite, the second one can * be infinite. * @channel_opt: channel optimization options, for full and partial scan * @data: channel configuration and probe request packet. */ struct iwm_scan_req_lmac { /* SCAN_REQUEST_FIXED_PART_API_S_VER_7 */ uint32_t reserved1; uint8_t n_channels; uint8_t active_dwell; uint8_t passive_dwell; uint8_t fragmented_dwell; uint8_t extended_dwell; uint8_t reserved2; uint16_t rx_chain_select; uint32_t scan_flags; uint32_t max_out_time; uint32_t suspend_time; /* RX_ON_FLAGS_API_S_VER_1 */ uint32_t flags; uint32_t filter_flags; struct iwm_scan_req_tx_cmd tx_cmd[2]; struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; uint32_t scan_prio; /* SCAN_REQ_PERIODIC_PARAMS_API_S */ uint32_t iter_num; uint32_t delay; struct iwm_scan_schedule_lmac schedule[IWM_MAX_SCHED_SCAN_PLANS]; struct iwm_scan_channel_opt channel_opt[2]; uint8_t data[]; } __packed; /** * iwm_scan_offload_complete - PERIODIC_SCAN_COMPLETE_NTF_API_S_VER_2 * @last_schedule_line: last schedule line executed (fast or regular) * @last_schedule_iteration: last scan iteration executed before scan abort * @status: enum iwm_scan_offload_complete_status * @ebs_status: EBS success status &enum iwm_scan_ebs_status * @time_after_last_iter; time in seconds elapsed after last iteration */ struct iwm_periodic_scan_complete { uint8_t last_schedule_line; uint8_t last_schedule_iteration; uint8_t status; uint8_t ebs_status; uint32_t time_after_last_iter; uint32_t reserved; } __packed; /** * struct iwm_scan_results_notif - scan results for one channel - * SCAN_RESULT_NTF_API_S_VER_3 * @channel: which channel the results are from * @band: 0 for 5.2 GHz, 1 for 2.4 GHz * @probe_status: IWM_SCAN_PROBE_STATUS_*, indicates success of probe request * @num_probe_not_sent: # of request that weren't sent due to not enough time * @duration: duration spent in channel, in usecs */ struct iwm_scan_results_notif { uint8_t channel; uint8_t band; uint8_t probe_status; uint8_t num_probe_not_sent; uint32_t duration; } __packed; enum iwm_scan_framework_client { IWM_SCAN_CLIENT_SCHED_SCAN = (1 << 0), IWM_SCAN_CLIENT_NETDETECT = (1 << 1), IWM_SCAN_CLIENT_ASSET_TRACKING = (1 << 2), }; /** * iwm_scan_offload_blacklist - IWM_SCAN_OFFLOAD_BLACKLIST_S * @ssid: MAC address to filter out * @reported_rssi: AP rssi reported to the host * @client_bitmap: clients ignore this entry - enum scan_framework_client */ struct iwm_scan_offload_blacklist { uint8_t ssid[IEEE80211_ADDR_LEN]; uint8_t reported_rssi; uint8_t client_bitmap; } __packed; enum iwm_scan_offload_network_type { IWM_NETWORK_TYPE_BSS = 1, IWM_NETWORK_TYPE_IBSS = 2, IWM_NETWORK_TYPE_ANY = 3, }; enum iwm_scan_offload_band_selection { IWM_SCAN_OFFLOAD_SELECT_2_4 = 0x4, IWM_SCAN_OFFLOAD_SELECT_5_2 = 0x8, IWM_SCAN_OFFLOAD_SELECT_ANY = 0xc, }; /** * iwm_scan_offload_profile - IWM_SCAN_OFFLOAD_PROFILE_S * @ssid_index: index to ssid list in fixed part * @unicast_cipher: encryption olgorithm to match - bitmap * @aut_alg: authentication olgorithm to match - bitmap * @network_type: enum iwm_scan_offload_network_type * @band_selection: enum iwm_scan_offload_band_selection * @client_bitmap: clients waiting for match - enum scan_framework_client */ struct iwm_scan_offload_profile { uint8_t ssid_index; uint8_t unicast_cipher; uint8_t auth_alg; uint8_t network_type; uint8_t band_selection; uint8_t client_bitmap; uint8_t reserved[2]; } __packed; /** * iwm_scan_offload_profile_cfg - IWM_SCAN_OFFLOAD_PROFILES_CFG_API_S_VER_1 * @blaclist: AP list to filter off from scan results * @profiles: profiles to search for match * @blacklist_len: length of blacklist * @num_profiles: num of profiles in the list * @match_notify: clients waiting for match found notification * @pass_match: clients waiting for the results * @active_clients: active clients bitmap - enum scan_framework_client * @any_beacon_notify: clients waiting for match notification without match */ struct iwm_scan_offload_profile_cfg { struct iwm_scan_offload_profile profiles[IWM_SCAN_MAX_PROFILES]; uint8_t blacklist_len; uint8_t num_profiles; uint8_t match_notify; uint8_t pass_match; uint8_t active_clients; uint8_t any_beacon_notify; uint8_t reserved[2]; } __packed; enum iwm_scan_offload_complete_status { IWM_SCAN_OFFLOAD_COMPLETED = 1, IWM_SCAN_OFFLOAD_ABORTED = 2, }; enum iwm_scan_ebs_status { IWM_SCAN_EBS_SUCCESS, IWM_SCAN_EBS_FAILED, IWM_SCAN_EBS_CHAN_NOT_FOUND, IWM_SCAN_EBS_INACTIVE, }; /** * struct iwm_lmac_scan_complete_notif - notifies end of scanning (all channels) * SCAN_COMPLETE_NTF_API_S_VER_3 * @scanned_channels: number of channels scanned (and number of valid results) * @status: one of SCAN_COMP_STATUS_* * @bt_status: BT on/off status * @last_channel: last channel that was scanned * @tsf_low: TSF timer (lower half) in usecs * @tsf_high: TSF timer (higher half) in usecs * @results: an array of scan results, only "scanned_channels" of them are valid */ struct iwm_lmac_scan_complete_notif { uint8_t scanned_channels; uint8_t status; uint8_t bt_status; uint8_t last_channel; uint32_t tsf_low; uint32_t tsf_high; struct iwm_scan_results_notif results[]; } __packed; /* * END mvm/fw-api-scan.h */ /* * BEGIN mvm/fw-api-sta.h */ /* UMAC Scan API */ /* The maximum of either of these cannot exceed 8, because we use an * 8-bit mask (see IWM_MVM_SCAN_MASK). */ #define IWM_MVM_MAX_UMAC_SCANS 8 #define IWM_MVM_MAX_LMAC_SCANS 1 enum iwm_scan_config_flags { IWM_SCAN_CONFIG_FLAG_ACTIVATE = (1 << 0), IWM_SCAN_CONFIG_FLAG_DEACTIVATE = (1 << 1), IWM_SCAN_CONFIG_FLAG_FORBID_CHUB_REQS = (1 << 2), IWM_SCAN_CONFIG_FLAG_ALLOW_CHUB_REQS = (1 << 3), IWM_SCAN_CONFIG_FLAG_SET_TX_CHAINS = (1 << 8), IWM_SCAN_CONFIG_FLAG_SET_RX_CHAINS = (1 << 9), IWM_SCAN_CONFIG_FLAG_SET_AUX_STA_ID = (1 << 10), IWM_SCAN_CONFIG_FLAG_SET_ALL_TIMES = (1 << 11), IWM_SCAN_CONFIG_FLAG_SET_EFFECTIVE_TIMES = (1 << 12), IWM_SCAN_CONFIG_FLAG_SET_CHANNEL_FLAGS = (1 << 13), IWM_SCAN_CONFIG_FLAG_SET_LEGACY_RATES = (1 << 14), IWM_SCAN_CONFIG_FLAG_SET_MAC_ADDR = (1 << 15), IWM_SCAN_CONFIG_FLAG_SET_FRAGMENTED = (1 << 16), IWM_SCAN_CONFIG_FLAG_CLEAR_FRAGMENTED = (1 << 17), IWM_SCAN_CONFIG_FLAG_SET_CAM_MODE = (1 << 18), IWM_SCAN_CONFIG_FLAG_CLEAR_CAM_MODE = (1 << 19), IWM_SCAN_CONFIG_FLAG_SET_PROMISC_MODE = (1 << 20), IWM_SCAN_CONFIG_FLAG_CLEAR_PROMISC_MODE = (1 << 21), /* Bits 26-31 are for num of channels in channel_array */ #define IWM_SCAN_CONFIG_N_CHANNELS(n) ((n) << 26) }; enum iwm_scan_config_rates { /* OFDM basic rates */ IWM_SCAN_CONFIG_RATE_6M = (1 << 0), IWM_SCAN_CONFIG_RATE_9M = (1 << 1), IWM_SCAN_CONFIG_RATE_12M = (1 << 2), IWM_SCAN_CONFIG_RATE_18M = (1 << 3), IWM_SCAN_CONFIG_RATE_24M = (1 << 4), IWM_SCAN_CONFIG_RATE_36M = (1 << 5), IWM_SCAN_CONFIG_RATE_48M = (1 << 6), IWM_SCAN_CONFIG_RATE_54M = (1 << 7), /* CCK basic rates */ IWM_SCAN_CONFIG_RATE_1M = (1 << 8), IWM_SCAN_CONFIG_RATE_2M = (1 << 9), IWM_SCAN_CONFIG_RATE_5M = (1 << 10), IWM_SCAN_CONFIG_RATE_11M = (1 << 11), /* Bits 16-27 are for supported rates */ #define IWM_SCAN_CONFIG_SUPPORTED_RATE(rate) ((rate) << 16) }; enum iwm_channel_flags { IWM_CHANNEL_FLAG_EBS = (1 << 0), IWM_CHANNEL_FLAG_ACCURATE_EBS = (1 << 1), IWM_CHANNEL_FLAG_EBS_ADD = (1 << 2), IWM_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE = (1 << 3), }; /** * struct iwm_scan_config * @flags: enum scan_config_flags * @tx_chains: valid_tx antenna - ANT_* definitions * @rx_chains: valid_rx antenna - ANT_* definitions * @legacy_rates: default legacy rates - enum scan_config_rates * @out_of_channel_time: default max out of serving channel time * @suspend_time: default max suspend time * @dwell_active: default dwell time for active scan * @dwell_passive: default dwell time for passive scan * @dwell_fragmented: default dwell time for fragmented scan * @dwell_extended: default dwell time for channels 1, 6 and 11 * @mac_addr: default mac address to be used in probes * @bcast_sta_id: the index of the station in the fw * @channel_flags: default channel flags - enum iwm_channel_flags * scan_config_channel_flag * @channel_array: default supported channels */ struct iwm_scan_config { uint32_t flags; uint32_t tx_chains; uint32_t rx_chains; uint32_t legacy_rates; uint32_t out_of_channel_time; uint32_t suspend_time; uint8_t dwell_active; uint8_t dwell_passive; uint8_t dwell_fragmented; uint8_t dwell_extended; uint8_t mac_addr[IEEE80211_ADDR_LEN]; uint8_t bcast_sta_id; uint8_t channel_flags; uint8_t channel_array[]; } __packed; /* SCAN_CONFIG_DB_CMD_API_S */ /** * iwm_umac_scan_flags *@IWM_UMAC_SCAN_FLAG_PREEMPTIVE: scan process triggered by this scan request * can be preempted by other scan requests with higher priority. * The low priority scan will be resumed when the higher proirity scan is * completed. *@IWM_UMAC_SCAN_FLAG_START_NOTIF: notification will be sent to the driver * when scan starts. */ enum iwm_umac_scan_flags { IWM_UMAC_SCAN_FLAG_PREEMPTIVE = (1 << 0), IWM_UMAC_SCAN_FLAG_START_NOTIF = (1 << 1), }; enum iwm_umac_scan_uid_offsets { IWM_UMAC_SCAN_UID_TYPE_OFFSET = 0, IWM_UMAC_SCAN_UID_SEQ_OFFSET = 8, }; enum iwm_umac_scan_general_flags { IWM_UMAC_SCAN_GEN_FLAGS_PERIODIC = (1 << 0), IWM_UMAC_SCAN_GEN_FLAGS_OVER_BT = (1 << 1), IWM_UMAC_SCAN_GEN_FLAGS_PASS_ALL = (1 << 2), IWM_UMAC_SCAN_GEN_FLAGS_PASSIVE = (1 << 3), IWM_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT = (1 << 4), IWM_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE = (1 << 5), IWM_UMAC_SCAN_GEN_FLAGS_MULTIPLE_SSID = (1 << 6), IWM_UMAC_SCAN_GEN_FLAGS_FRAGMENTED = (1 << 7), IWM_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED = (1 << 8), IWM_UMAC_SCAN_GEN_FLAGS_MATCH = (1 << 9), IWM_UMAC_SCAN_GEN_FLAGS_EXTENDED_DWELL = (1 << 10), }; /** * struct iwm_scan_channel_cfg_umac * @flags: bitmap - 0-19: directed scan to i'th ssid. * @channel_num: channel number 1-13 etc. * @iter_count: repetition count for the channel. * @iter_interval: interval between two scan iterations on one channel. */ struct iwm_scan_channel_cfg_umac { uint32_t flags; #define IWM_SCAN_CHANNEL_UMAC_NSSIDS(x) ((1 << (x)) - 1) uint8_t channel_num; uint8_t iter_count; uint16_t iter_interval; } __packed; /* SCAN_CHANNEL_CFG_S_VER2 */ /** * struct iwm_scan_umac_schedule * @interval: interval in seconds between scan iterations * @iter_count: num of scan iterations for schedule plan, 0xff for infinite loop * @reserved: for alignment and future use */ struct iwm_scan_umac_schedule { uint16_t interval; uint8_t iter_count; uint8_t reserved; } __packed; /* SCAN_SCHED_PARAM_API_S_VER_1 */ /** * struct iwm_scan_req_umac_tail - the rest of the UMAC scan request command * parameters following channels configuration array. * @schedule: two scheduling plans. * @delay: delay in TUs before starting the first scan iteration * @reserved: for future use and alignment * @preq: probe request with IEs blocks * @direct_scan: list of SSIDs for directed active scan */ struct iwm_scan_req_umac_tail { /* SCAN_PERIODIC_PARAMS_API_S_VER_1 */ struct iwm_scan_umac_schedule schedule[IWM_MAX_SCHED_SCAN_PLANS]; uint16_t delay; uint16_t reserved; /* SCAN_PROBE_PARAMS_API_S_VER_1 */ struct iwm_scan_probe_req preq; struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; } __packed; /** * struct iwm_scan_req_umac * @flags: &enum iwm_umac_scan_flags * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @ooc_priority: out of channel priority - &enum iwm_scan_priority * @general_flags: &enum iwm_umac_scan_general_flags * @extended_dwell: dwell time for channels 1, 6 and 11 * @active_dwell: dwell time for active scan * @passive_dwell: dwell time for passive scan * @fragmented_dwell: dwell time for fragmented passive scan * @max_out_time: max out of serving channel time * @suspend_time: max suspend time * @scan_priority: scan internal prioritization &enum iwm_scan_priority * @channel_flags: &enum iwm_scan_channel_flags * @n_channels: num of channels in scan request * @reserved: for future use and alignment * @data: &struct iwm_scan_channel_cfg_umac and * &struct iwm_scan_req_umac_tail */ struct iwm_scan_req_umac { uint32_t flags; uint32_t uid; uint32_t ooc_priority; /* SCAN_GENERAL_PARAMS_API_S_VER_1 */ uint32_t general_flags; uint8_t extended_dwell; uint8_t active_dwell; uint8_t passive_dwell; uint8_t fragmented_dwell; uint32_t max_out_time; uint32_t suspend_time; uint32_t scan_priority; /* SCAN_CHANNEL_PARAMS_API_S_VER_1 */ uint8_t channel_flags; uint8_t n_channels; uint16_t reserved; uint8_t data[]; } __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_1 */ /** * struct iwm_umac_scan_abort * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @flags: reserved */ struct iwm_umac_scan_abort { uint32_t uid; uint32_t flags; } __packed; /* SCAN_ABORT_CMD_UMAC_API_S_VER_1 */ /** * struct iwm_umac_scan_complete * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @last_schedule: last scheduling line * @last_iter: last scan iteration number * @scan status: &enum iwm_scan_offload_complete_status * @ebs_status: &enum iwm_scan_ebs_status * @time_from_last_iter: time elapsed from last iteration * @reserved: for future use */ struct iwm_umac_scan_complete { uint32_t uid; uint8_t last_schedule; uint8_t last_iter; uint8_t status; uint8_t ebs_status; uint32_t time_from_last_iter; uint32_t reserved; } __packed; /* SCAN_COMPLETE_NTF_UMAC_API_S_VER_1 */ #define IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN 5 /** * struct iwm_scan_offload_profile_match - match information * @bssid: matched bssid * @channel: channel where the match occurred * @energy: * @matching_feature: * @matching_channels: bitmap of channels that matched, referencing * the channels passed in tue scan offload request */ struct iwm_scan_offload_profile_match { uint8_t bssid[IEEE80211_ADDR_LEN]; uint16_t reserved; uint8_t channel; uint8_t energy; uint8_t matching_feature; uint8_t matching_channels[IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN]; } __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_1 */ /** * struct iwm_scan_offload_profiles_query - match results query response * @matched_profiles: bitmap of matched profiles, referencing the * matches passed in the scan offload request * @last_scan_age: age of the last offloaded scan * @n_scans_done: number of offloaded scans done * @gp2_d0u: GP2 when D0U occurred * @gp2_invoked: GP2 when scan offload was invoked * @resume_while_scanning: not used * @self_recovery: obsolete * @reserved: reserved * @matches: array of match information, one for each match */ struct iwm_scan_offload_profiles_query { uint32_t matched_profiles; uint32_t last_scan_age; uint32_t n_scans_done; uint32_t gp2_d0u; uint32_t gp2_invoked; uint8_t resume_while_scanning; uint8_t self_recovery; uint16_t reserved; struct iwm_scan_offload_profile_match matches[IWM_SCAN_MAX_PROFILES]; } __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_2 */ /** * struct iwm_umac_scan_iter_complete_notif - notifies end of scanning iteration * @uid: scan id, &enum iwm_umac_scan_uid_offsets * @scanned_channels: number of channels scanned and number of valid elements in * results array * @status: one of SCAN_COMP_STATUS_* * @bt_status: BT on/off status * @last_channel: last channel that was scanned * @tsf_low: TSF timer (lower half) in usecs * @tsf_high: TSF timer (higher half) in usecs * @results: array of scan results, only "scanned_channels" of them are valid */ struct iwm_umac_scan_iter_complete_notif { uint32_t uid; uint8_t scanned_channels; uint8_t status; uint8_t bt_status; uint8_t last_channel; uint32_t tsf_low; uint32_t tsf_high; struct iwm_scan_results_notif results[]; } __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_1 */ /* Please keep this enum *SORTED* by hex value. * Needed for binary search, otherwise a warning will be triggered. */ enum iwm_scan_subcmd_ids { IWM_GSCAN_START_CMD = 0x0, IWM_GSCAN_STOP_CMD = 0x1, IWM_GSCAN_SET_HOTLIST_CMD = 0x2, IWM_GSCAN_RESET_HOTLIST_CMD = 0x3, IWM_GSCAN_SET_SIGNIFICANT_CHANGE_CMD = 0x4, IWM_GSCAN_RESET_SIGNIFICANT_CHANGE_CMD = 0x5, IWM_GSCAN_SIGNIFICANT_CHANGE_EVENT = 0xFD, IWM_GSCAN_HOTLIST_CHANGE_EVENT = 0xFE, IWM_GSCAN_RESULTS_AVAILABLE_EVENT = 0xFF, }; /* STA API */ /** * enum iwm_sta_flags - flags for the ADD_STA host command * @IWM_STA_FLG_REDUCED_TX_PWR_CTRL: * @IWM_STA_FLG_REDUCED_TX_PWR_DATA: * @IWM_STA_FLG_DISABLE_TX: set if TX should be disabled * @IWM_STA_FLG_PS: set if STA is in Power Save * @IWM_STA_FLG_INVALID: set if STA is invalid * @IWM_STA_FLG_DLP_EN: Direct Link Protocol is enabled * @IWM_STA_FLG_SET_ALL_KEYS: the current key applies to all key IDs * @IWM_STA_FLG_DRAIN_FLOW: drain flow * @IWM_STA_FLG_PAN: STA is for PAN interface * @IWM_STA_FLG_CLASS_AUTH: * @IWM_STA_FLG_CLASS_ASSOC: * @IWM_STA_FLG_CLASS_MIMO_PROT: * @IWM_STA_FLG_MAX_AGG_SIZE_MSK: maximal size for A-MPDU * @IWM_STA_FLG_AGG_MPDU_DENS_MSK: maximal MPDU density for Tx aggregation * @IWM_STA_FLG_FAT_EN_MSK: support for channel width (for Tx). This flag is * initialised by driver and can be updated by fw upon reception of * action frames that can change the channel width. When cleared the fw * will send all the frames in 20MHz even when FAT channel is requested. * @IWM_STA_FLG_MIMO_EN_MSK: support for MIMO. This flag is initialised by the * driver and can be updated by fw upon reception of action frames. * @IWM_STA_FLG_MFP_EN: Management Frame Protection */ enum iwm_sta_flags { IWM_STA_FLG_REDUCED_TX_PWR_CTRL = (1 << 3), IWM_STA_FLG_REDUCED_TX_PWR_DATA = (1 << 6), IWM_STA_FLG_DISABLE_TX = (1 << 4), IWM_STA_FLG_PS = (1 << 8), IWM_STA_FLG_DRAIN_FLOW = (1 << 12), IWM_STA_FLG_PAN = (1 << 13), IWM_STA_FLG_CLASS_AUTH = (1 << 14), IWM_STA_FLG_CLASS_ASSOC = (1 << 15), IWM_STA_FLG_RTS_MIMO_PROT = (1 << 17), IWM_STA_FLG_MAX_AGG_SIZE_SHIFT = 19, IWM_STA_FLG_MAX_AGG_SIZE_8K = (0 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_16K = (1 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_32K = (2 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_64K = (3 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_128K = (4 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_256K = (5 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_512K = (6 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_1024K = (7 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_MAX_AGG_SIZE_MSK = (7 << IWM_STA_FLG_MAX_AGG_SIZE_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_SHIFT = 23, IWM_STA_FLG_AGG_MPDU_DENS_2US = (4 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_4US = (5 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_8US = (6 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_16US = (7 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_AGG_MPDU_DENS_MSK = (7 << IWM_STA_FLG_AGG_MPDU_DENS_SHIFT), IWM_STA_FLG_FAT_EN_20MHZ = (0 << 26), IWM_STA_FLG_FAT_EN_40MHZ = (1 << 26), IWM_STA_FLG_FAT_EN_80MHZ = (2 << 26), IWM_STA_FLG_FAT_EN_160MHZ = (3 << 26), IWM_STA_FLG_FAT_EN_MSK = (3 << 26), IWM_STA_FLG_MIMO_EN_SISO = (0 << 28), IWM_STA_FLG_MIMO_EN_MIMO2 = (1 << 28), IWM_STA_FLG_MIMO_EN_MIMO3 = (2 << 28), IWM_STA_FLG_MIMO_EN_MSK = (3 << 28), }; /** * enum iwm_sta_key_flag - key flags for the ADD_STA host command * @IWM_STA_KEY_FLG_NO_ENC: no encryption * @IWM_STA_KEY_FLG_WEP: WEP encryption algorithm * @IWM_STA_KEY_FLG_CCM: CCMP encryption algorithm * @IWM_STA_KEY_FLG_TKIP: TKIP encryption algorithm * @IWM_STA_KEY_FLG_EXT: extended cipher algorithm (depends on the FW support) * @IWM_STA_KEY_FLG_CMAC: CMAC encryption algorithm * @IWM_STA_KEY_FLG_ENC_UNKNOWN: unknown encryption algorithm * @IWM_STA_KEY_FLG_EN_MSK: mask for encryption algorithmi value * @IWM_STA_KEY_FLG_WEP_KEY_MAP: wep is either a group key (0 - legacy WEP) or from * station info array (1 - n 1X mode) * @IWM_STA_KEY_FLG_KEYID_MSK: the index of the key * @IWM_STA_KEY_NOT_VALID: key is invalid * @IWM_STA_KEY_FLG_WEP_13BYTES: set for 13 bytes WEP key * @IWM_STA_KEY_MULTICAST: set for multical key * @IWM_STA_KEY_MFP: key is used for Management Frame Protection */ enum iwm_sta_key_flag { IWM_STA_KEY_FLG_NO_ENC = (0 << 0), IWM_STA_KEY_FLG_WEP = (1 << 0), IWM_STA_KEY_FLG_CCM = (2 << 0), IWM_STA_KEY_FLG_TKIP = (3 << 0), IWM_STA_KEY_FLG_EXT = (4 << 0), IWM_STA_KEY_FLG_CMAC = (6 << 0), IWM_STA_KEY_FLG_ENC_UNKNOWN = (7 << 0), IWM_STA_KEY_FLG_EN_MSK = (7 << 0), IWM_STA_KEY_FLG_WEP_KEY_MAP = (1 << 3), IWM_STA_KEY_FLG_KEYID_POS = 8, IWM_STA_KEY_FLG_KEYID_MSK = (3 << IWM_STA_KEY_FLG_KEYID_POS), IWM_STA_KEY_NOT_VALID = (1 << 11), IWM_STA_KEY_FLG_WEP_13BYTES = (1 << 12), IWM_STA_KEY_MULTICAST = (1 << 14), IWM_STA_KEY_MFP = (1 << 15), }; /** * enum iwm_sta_modify_flag - indicate to the fw what flag are being changed * @IWM_STA_MODIFY_QUEUE_REMOVAL: this command removes a queue * @IWM_STA_MODIFY_TID_DISABLE_TX: this command modifies %tid_disable_tx * @IWM_STA_MODIFY_TX_RATE: unused * @IWM_STA_MODIFY_ADD_BA_TID: this command modifies %add_immediate_ba_tid * @IWM_STA_MODIFY_REMOVE_BA_TID: this command modifies %remove_immediate_ba_tid * @IWM_STA_MODIFY_SLEEPING_STA_TX_COUNT: this command modifies %sleep_tx_count * @IWM_STA_MODIFY_PROT_TH: * @IWM_STA_MODIFY_QUEUES: modify the queues used by this station */ enum iwm_sta_modify_flag { IWM_STA_MODIFY_QUEUE_REMOVAL = (1 << 0), IWM_STA_MODIFY_TID_DISABLE_TX = (1 << 1), IWM_STA_MODIFY_TX_RATE = (1 << 2), IWM_STA_MODIFY_ADD_BA_TID = (1 << 3), IWM_STA_MODIFY_REMOVE_BA_TID = (1 << 4), IWM_STA_MODIFY_SLEEPING_STA_TX_COUNT = (1 << 5), IWM_STA_MODIFY_PROT_TH = (1 << 6), IWM_STA_MODIFY_QUEUES = (1 << 7), }; #define IWM_STA_MODE_MODIFY 1 /** * enum iwm_sta_sleep_flag - type of sleep of the station * @IWM_STA_SLEEP_STATE_AWAKE: * @IWM_STA_SLEEP_STATE_PS_POLL: * @IWM_STA_SLEEP_STATE_UAPSD: * @IWM_STA_SLEEP_STATE_MOREDATA: set more-data bit on * (last) released frame */ enum iwm_sta_sleep_flag { IWM_STA_SLEEP_STATE_AWAKE = 0, IWM_STA_SLEEP_STATE_PS_POLL = (1 << 0), IWM_STA_SLEEP_STATE_UAPSD = (1 << 1), IWM_STA_SLEEP_STATE_MOREDATA = (1 << 2), }; /* STA ID and color bits definitions */ #define IWM_STA_ID_SEED (0x0f) #define IWM_STA_ID_POS (0) #define IWM_STA_ID_MSK (IWM_STA_ID_SEED << IWM_STA_ID_POS) #define IWM_STA_COLOR_SEED (0x7) #define IWM_STA_COLOR_POS (4) #define IWM_STA_COLOR_MSK (IWM_STA_COLOR_SEED << IWM_STA_COLOR_POS) #define IWM_STA_ID_N_COLOR_GET_COLOR(id_n_color) \ (((id_n_color) & IWM_STA_COLOR_MSK) >> IWM_STA_COLOR_POS) #define IWM_STA_ID_N_COLOR_GET_ID(id_n_color) \ (((id_n_color) & IWM_STA_ID_MSK) >> IWM_STA_ID_POS) #define IWM_STA_KEY_MAX_NUM (16) #define IWM_STA_KEY_IDX_INVALID (0xff) #define IWM_STA_KEY_MAX_DATA_KEY_NUM (4) #define IWM_MAX_GLOBAL_KEYS (4) #define IWM_STA_KEY_LEN_WEP40 (5) #define IWM_STA_KEY_LEN_WEP104 (13) /** * struct iwm_mvm_keyinfo - key information * @key_flags: type %iwm_sta_key_flag * @tkip_rx_tsc_byte2: TSC[2] for key mix ph1 detection * @tkip_rx_ttak: 10-byte unicast TKIP TTAK for Rx * @key_offset: key offset in the fw's key table * @key: 16-byte unicast decryption key * @tx_secur_seq_cnt: initial RSC / PN needed for replay check * @hw_tkip_mic_rx_key: byte: MIC Rx Key - used for TKIP only * @hw_tkip_mic_tx_key: byte: MIC Tx Key - used for TKIP only */ struct iwm_mvm_keyinfo { uint16_t key_flags; uint8_t tkip_rx_tsc_byte2; uint8_t reserved1; uint16_t tkip_rx_ttak[5]; uint8_t key_offset; uint8_t reserved2; uint8_t key[16]; uint64_t tx_secur_seq_cnt; uint64_t hw_tkip_mic_rx_key; uint64_t hw_tkip_mic_tx_key; } __packed; #define IWM_ADD_STA_STATUS_MASK 0xFF #define IWM_ADD_STA_BAID_VALID_MASK 0x8000 #define IWM_ADD_STA_BAID_MASK 0x7F00 #define IWM_ADD_STA_BAID_SHIFT 8 /** * struct iwm_mvm_add_sta_cmd - Add/modify a station in the fw's sta table. * ( REPLY_ADD_STA = 0x18 ) * @add_modify: 1: modify existing, 0: add new station * @awake_acs: * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable * AMPDU for tid x. Set %IWM_STA_MODIFY_TID_DISABLE_TX to change this field. * @mac_id_n_color: the Mac context this station belongs to * @addr[IEEE80211_ADDR_LEN]: station's MAC address * @sta_id: index of station in uCode's station table * @modify_mask: IWM_STA_MODIFY_*, selects which parameters to modify vs. leave * alone. 1 - modify, 0 - don't change. * @station_flags: look at %iwm_sta_flags * @station_flags_msk: what of %station_flags have changed * @add_immediate_ba_tid: tid for which to add block-ack support (Rx) * Set %IWM_STA_MODIFY_ADD_BA_TID to use this field, and also set * add_immediate_ba_ssn. * @remove_immediate_ba_tid: tid for which to remove block-ack support (Rx) * Set %IWM_STA_MODIFY_REMOVE_BA_TID to use this field * @add_immediate_ba_ssn: ssn for the Rx block-ack session. Used together with * add_immediate_ba_tid. * @sleep_tx_count: number of packets to transmit to station even though it is * asleep. Used to synchronise PS-poll and u-APSD responses while ucode * keeps track of STA sleep state. * @sleep_state_flags: Look at %iwm_sta_sleep_flag. * @assoc_id: assoc_id to be sent in VHT PLCP (9-bit), for grp use 0, for AP * mac-addr. * @beamform_flags: beam forming controls * @tfd_queue_msk: tfd queues used by this station * * The device contains an internal table of per-station information, with info * on security keys, aggregation parameters, and Tx rates for initial Tx * attempt and any retries (set by IWM_REPLY_TX_LINK_QUALITY_CMD). * * ADD_STA sets up the table entry for one station, either creating a new * entry, or modifying a pre-existing one. */ struct iwm_mvm_add_sta_cmd { uint8_t add_modify; uint8_t awake_acs; uint16_t tid_disable_tx; uint32_t mac_id_n_color; uint8_t addr[IEEE80211_ADDR_LEN]; /* _STA_ID_MODIFY_INFO_API_S_VER_1 */ uint16_t reserved2; uint8_t sta_id; uint8_t modify_mask; uint16_t reserved3; uint32_t station_flags; uint32_t station_flags_msk; uint8_t add_immediate_ba_tid; uint8_t remove_immediate_ba_tid; uint16_t add_immediate_ba_ssn; uint16_t sleep_tx_count; uint16_t sleep_state_flags; uint16_t assoc_id; uint16_t beamform_flags; uint32_t tfd_queue_msk; } __packed; /* ADD_STA_CMD_API_S_VER_7 */ /** * struct iwm_mvm_add_sta_key_cmd - add/modify sta key * ( IWM_REPLY_ADD_STA_KEY = 0x17 ) * @sta_id: index of station in uCode's station table * @key_offset: key offset in key storage * @key_flags: type %iwm_sta_key_flag * @key: key material data * @key2: key material data * @rx_secur_seq_cnt: RX security sequence counter for the key * @tkip_rx_tsc_byte2: TSC[2] for key mix ph1 detection * @tkip_rx_ttak: 10-byte unicast TKIP TTAK for Rx */ struct iwm_mvm_add_sta_key_cmd { uint8_t sta_id; uint8_t key_offset; uint16_t key_flags; uint8_t key[16]; uint8_t key2[16]; uint8_t rx_secur_seq_cnt[16]; uint8_t tkip_rx_tsc_byte2; uint8_t reserved; uint16_t tkip_rx_ttak[5]; } __packed; /* IWM_ADD_MODIFY_STA_KEY_API_S_VER_1 */ /** * enum iwm_mvm_add_sta_rsp_status - status in the response to ADD_STA command * @IWM_ADD_STA_SUCCESS: operation was executed successfully * @IWM_ADD_STA_STATIONS_OVERLOAD: no room left in the fw's station table * @IWM_ADD_STA_IMMEDIATE_BA_FAILURE: can't add Rx block ack session * @IWM_ADD_STA_MODIFY_NON_EXISTING_STA: driver requested to modify a station * that doesn't exist. */ enum iwm_mvm_add_sta_rsp_status { IWM_ADD_STA_SUCCESS = 0x1, IWM_ADD_STA_STATIONS_OVERLOAD = 0x2, IWM_ADD_STA_IMMEDIATE_BA_FAILURE = 0x4, IWM_ADD_STA_MODIFY_NON_EXISTING_STA = 0x8, }; /** * struct iwm_mvm_rm_sta_cmd - Add / modify a station in the fw's station table * ( IWM_REMOVE_STA = 0x19 ) * @sta_id: the station id of the station to be removed */ struct iwm_mvm_rm_sta_cmd { uint8_t sta_id; uint8_t reserved[3]; } __packed; /* IWM_REMOVE_STA_CMD_API_S_VER_2 */ /** * struct iwm_mvm_mgmt_mcast_key_cmd * ( IWM_MGMT_MCAST_KEY = 0x1f ) * @ctrl_flags: %iwm_sta_key_flag * @IGTK: * @K1: IGTK master key * @K2: IGTK sub key * @sta_id: station ID that support IGTK * @key_id: * @receive_seq_cnt: initial RSC/PN needed for replay check */ struct iwm_mvm_mgmt_mcast_key_cmd { uint32_t ctrl_flags; uint8_t IGTK[16]; uint8_t K1[16]; uint8_t K2[16]; uint32_t key_id; uint32_t sta_id; uint64_t receive_seq_cnt; } __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_1 */ struct iwm_mvm_wep_key { uint8_t key_index; uint8_t key_offset; uint16_t reserved1; uint8_t key_size; uint8_t reserved2[3]; uint8_t key[16]; } __packed; struct iwm_mvm_wep_key_cmd { uint32_t mac_id_n_color; uint8_t num_keys; uint8_t decryption_type; uint8_t flags; uint8_t reserved; struct iwm_mvm_wep_key wep_key[0]; } __packed; /* SEC_CURR_WEP_KEY_CMD_API_S_VER_2 */ /* * END mvm/fw-api-sta.h */ /* * BT coex */ enum iwm_bt_coex_mode { IWM_BT_COEX_DISABLE = 0x0, IWM_BT_COEX_NW = 0x1, IWM_BT_COEX_BT = 0x2, IWM_BT_COEX_WIFI = 0x3, }; /* BT_COEX_MODES_E */ enum iwm_bt_coex_enabled_modules { IWM_BT_COEX_MPLUT_ENABLED = (1 << 0), IWM_BT_COEX_MPLUT_BOOST_ENABLED = (1 << 1), IWM_BT_COEX_SYNC2SCO_ENABLED = (1 << 2), IWM_BT_COEX_CORUN_ENABLED = (1 << 3), IWM_BT_COEX_HIGH_BAND_RET = (1 << 4), }; /* BT_COEX_MODULES_ENABLE_E_VER_1 */ /** * struct iwm_bt_coex_cmd - bt coex configuration command * @mode: enum %iwm_bt_coex_mode * @enabled_modules: enum %iwm_bt_coex_enabled_modules * * The structure is used for the BT_COEX command. */ struct iwm_bt_coex_cmd { uint32_t mode; uint32_t enabled_modules; } __packed; /* BT_COEX_CMD_API_S_VER_6 */ /* * Location Aware Regulatory (LAR) API - MCC updates */ /** * struct iwm_mcc_update_cmd_v1 - Request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: the source from where we got the MCC, see iwm_mcc_source * @reserved: reserved for alignment */ struct iwm_mcc_update_cmd_v1 { uint16_t mcc; uint8_t source_id; uint8_t reserved; } __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_1 */ /** * struct iwm_mcc_update_cmd - Request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: the source from where we got the MCC, see iwm_mcc_source * @reserved: reserved for alignment * @key: integrity key for MCC API OEM testing * @reserved2: reserved */ struct iwm_mcc_update_cmd { uint16_t mcc; uint8_t source_id; uint8_t reserved; uint32_t key; uint32_t reserved2[5]; } __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_2 */ /** * iwm_mcc_update_resp_v1 - response to MCC_UPDATE_CMD. * Contains the new channel control profile map, if changed, and the new MCC * (mobile country code). * The new MCC may be different than what was requested in MCC_UPDATE_CMD. * @status: see &enum iwm_mcc_update_status * @mcc: the new applied MCC * @cap: capabilities for all channels which matches the MCC * @source_id: the MCC source, see iwm_mcc_source * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 * channels, depending on platform) * @channels: channel control data map, DWORD for each channel. Only the first * 16bits are used. */ struct iwm_mcc_update_resp_v1 { uint32_t status; uint16_t mcc; uint8_t cap; uint8_t source_id; uint32_t n_channels; uint32_t channels[0]; } __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_1 */ /** * iwm_mcc_update_resp - response to MCC_UPDATE_CMD. * Contains the new channel control profile map, if changed, and the new MCC * (mobile country code). * The new MCC may be different than what was requested in MCC_UPDATE_CMD. * @status: see &enum iwm_mcc_update_status * @mcc: the new applied MCC * @cap: capabilities for all channels which matches the MCC * @source_id: the MCC source, see iwm_mcc_source * @time: time elapsed from the MCC test start (in 30 seconds TU) * @reserved: reserved. * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 * channels, depending on platform) * @channels: channel control data map, DWORD for each channel. Only the first * 16bits are used. */ struct iwm_mcc_update_resp { uint32_t status; uint16_t mcc; uint8_t cap; uint8_t source_id; uint16_t time; uint16_t reserved; uint32_t n_channels; uint32_t channels[0]; } __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_2 */ /** * struct iwm_mcc_chub_notif - chub notifies of mcc change * (MCC_CHUB_UPDATE_CMD = 0xc9) * The Chub (Communication Hub, CommsHUB) is a HW component that connects to * the cellular and connectivity cores that gets updates of the mcc, and * notifies the ucode directly of any mcc change. * The ucode requests the driver to request the device to update geographic * regulatory profile according to the given MCC (Mobile Country Code). * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the * MCC in the cmd response will be the relevant MCC in the NVM. * @mcc: given mobile country code * @source_id: identity of the change originator, see iwm_mcc_source * @reserved1: reserved for alignment */ struct iwm_mcc_chub_notif { uint16_t mcc; uint8_t source_id; uint8_t reserved1; } __packed; /* LAR_MCC_NOTIFY_S */ enum iwm_mcc_update_status { IWM_MCC_RESP_NEW_CHAN_PROFILE, IWM_MCC_RESP_SAME_CHAN_PROFILE, IWM_MCC_RESP_INVALID, IWM_MCC_RESP_NVM_DISABLED, IWM_MCC_RESP_ILLEGAL, IWM_MCC_RESP_LOW_PRIORITY, IWM_MCC_RESP_TEST_MODE_ACTIVE, IWM_MCC_RESP_TEST_MODE_NOT_ACTIVE, IWM_MCC_RESP_TEST_MODE_DENIAL_OF_SERVICE, }; enum iwm_mcc_source { IWM_MCC_SOURCE_OLD_FW = 0, IWM_MCC_SOURCE_ME = 1, IWM_MCC_SOURCE_BIOS = 2, IWM_MCC_SOURCE_3G_LTE_HOST = 3, IWM_MCC_SOURCE_3G_LTE_DEVICE = 4, IWM_MCC_SOURCE_WIFI = 5, IWM_MCC_SOURCE_RESERVED = 6, IWM_MCC_SOURCE_DEFAULT = 7, IWM_MCC_SOURCE_UNINITIALIZED = 8, IWM_MCC_SOURCE_MCC_API = 9, IWM_MCC_SOURCE_GET_CURRENT = 0x10, IWM_MCC_SOURCE_GETTING_MCC_TEST_MODE = 0x11, }; /** * struct iwm_dts_measurement_notif_v1 - measurements notification * * @temp: the measured temperature * @voltage: the measured voltage */ struct iwm_dts_measurement_notif_v1 { int32_t temp; int32_t voltage; } __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_1*/ /** * struct iwm_dts_measurement_notif_v2 - measurements notification * * @temp: the measured temperature * @voltage: the measured voltage * @threshold_idx: the trip index that was crossed */ struct iwm_dts_measurement_notif_v2 { int32_t temp; int32_t voltage; int32_t threshold_idx; } __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_2 */ /* * Some cherry-picked definitions */ #define IWM_FRAME_LIMIT 64 /* * These functions retrieve specific information from the id field in * the iwm_host_cmd struct which contains the command id, the group id, * and the version of the command and vice versa. */ static inline uint8_t iwm_cmd_opcode(uint32_t cmdid) { return cmdid & 0xff; } static inline uint8_t iwm_cmd_groupid(uint32_t cmdid) { return ((cmdid & 0xff00) >> 8); } static inline uint8_t iwm_cmd_version(uint32_t cmdid) { return ((cmdid & 0xff0000) >> 16); } static inline uint32_t iwm_cmd_id(uint8_t opcode, uint8_t groupid, uint8_t version) { return opcode + (groupid << 8) + (version << 16); } /* make uint16_t wide id out of uint8_t group and opcode */ #define IWM_WIDE_ID(grp, opcode) ((grp << 8) | opcode) /* due to the conversion, this group is special */ #define IWM_ALWAYS_LONG_GROUP 1 struct iwm_cmd_header { uint8_t code; uint8_t flags; uint8_t idx; uint8_t qid; } __packed; struct iwm_cmd_header_wide { uint8_t opcode; uint8_t group_id; uint8_t idx; uint8_t qid; uint16_t length; uint8_t reserved; uint8_t version; } __packed; /** * enum iwm_power_scheme * @IWM_POWER_LEVEL_CAM - Continuously Active Mode * @IWM_POWER_LEVEL_BPS - Balanced Power Save (default) * @IWM_POWER_LEVEL_LP - Low Power */ enum iwm_power_scheme { IWM_POWER_SCHEME_CAM = 1, IWM_POWER_SCHEME_BPS, IWM_POWER_SCHEME_LP }; #define IWM_DEF_CMD_PAYLOAD_SIZE 320 #define IWM_MAX_CMD_PAYLOAD_SIZE ((4096 - 4) - sizeof(struct iwm_cmd_header)) #define IWM_CMD_FAILED_MSK 0x40 /** * struct iwm_device_cmd * * For allocation of the command and tx queues, this establishes the overall * size of the largest command we send to uCode, except for commands that * aren't fully copied and use other TFD space. */ struct iwm_device_cmd { union { struct { struct iwm_cmd_header hdr; uint8_t data[IWM_DEF_CMD_PAYLOAD_SIZE]; }; struct { struct iwm_cmd_header_wide hdr_wide; uint8_t data_wide[IWM_DEF_CMD_PAYLOAD_SIZE - sizeof(struct iwm_cmd_header_wide) + sizeof(struct iwm_cmd_header)]; }; }; } __packed; struct iwm_rx_packet { /* * The first 4 bytes of the RX frame header contain both the RX frame * size and some flags. * Bit fields: * 31: flag flush RB request * 30: flag ignore TC (terminal counter) request * 29: flag fast IRQ request * 28-14: Reserved * 13-00: RX frame size */ uint32_t len_n_flags; struct iwm_cmd_header hdr; uint8_t data[]; } __packed; #define IWM_FH_RSCSR_FRAME_SIZE_MSK 0x00003fff #define IWM_FH_RSCSR_FRAME_INVALID 0x55550000 #define IWM_FH_RSCSR_FRAME_ALIGN 0x40 static inline uint32_t iwm_rx_packet_len(const struct iwm_rx_packet *pkt) { return le32toh(pkt->len_n_flags) & IWM_FH_RSCSR_FRAME_SIZE_MSK; } static inline uint32_t iwm_rx_packet_payload_len(const struct iwm_rx_packet *pkt) { return iwm_rx_packet_len(pkt) - sizeof(pkt->hdr); } #define IWM_MIN_DBM -100 #define IWM_MAX_DBM -33 /* realistic guess */ #define IWM_READ(sc, reg) \ bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (reg)) #define IWM_WRITE(sc, reg, val) \ bus_space_write_4((sc)->sc_st, (sc)->sc_sh, (reg), (val)) #define IWM_WRITE_1(sc, reg, val) \ bus_space_write_1((sc)->sc_st, (sc)->sc_sh, (reg), (val)) #define IWM_SETBITS(sc, reg, mask) \ IWM_WRITE(sc, reg, IWM_READ(sc, reg) | (mask)) #define IWM_CLRBITS(sc, reg, mask) \ IWM_WRITE(sc, reg, IWM_READ(sc, reg) & ~(mask)) #define IWM_BARRIER_WRITE(sc) \ bus_space_barrier((sc)->sc_st, (sc)->sc_sh, 0, (sc)->sc_sz, \ BUS_SPACE_BARRIER_WRITE) #define IWM_BARRIER_READ_WRITE(sc) \ bus_space_barrier((sc)->sc_st, (sc)->sc_sh, 0, (sc)->sc_sz, \ BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE) #endif /* __IF_IWM_REG_H__ */ Index: projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_cmd.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_cmd.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_cmd.c (revision 345026) @@ -1,3281 +1,3289 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "mlx4.h" #include "fw.h" #include "fw_qos.h" #define CMD_POLL_TOKEN 0xffff #define INBOX_MASK 0xffffffffffffff00ULL #define CMD_CHAN_VER 1 #define CMD_CHAN_IF_REV 1 enum { /* command completed successfully: */ CMD_STAT_OK = 0x00, /* Internal error (such as a bus error) occurred while processing command: */ CMD_STAT_INTERNAL_ERR = 0x01, /* Operation/command not supported or opcode modifier not supported: */ CMD_STAT_BAD_OP = 0x02, /* Parameter not supported or parameter out of range: */ CMD_STAT_BAD_PARAM = 0x03, /* System not enabled or bad system state: */ CMD_STAT_BAD_SYS_STATE = 0x04, /* Attempt to access reserved or unallocaterd resource: */ CMD_STAT_BAD_RESOURCE = 0x05, /* Requested resource is currently executing a command, or is otherwise busy: */ CMD_STAT_RESOURCE_BUSY = 0x06, /* Required capability exceeds device limits: */ CMD_STAT_EXCEED_LIM = 0x08, /* Resource is not in the appropriate state or ownership: */ CMD_STAT_BAD_RES_STATE = 0x09, /* Index out of range: */ CMD_STAT_BAD_INDEX = 0x0a, /* FW image corrupted: */ CMD_STAT_BAD_NVMEM = 0x0b, /* Error in ICM mapping (e.g. not enough auxiliary ICM pages to execute command): */ CMD_STAT_ICM_ERROR = 0x0c, /* Attempt to modify a QP/EE which is not in the presumed state: */ CMD_STAT_BAD_QP_STATE = 0x10, /* Bad segment parameters (Address/Size): */ CMD_STAT_BAD_SEG_PARAM = 0x20, /* Memory Region has Memory Windows bound to: */ CMD_STAT_REG_BOUND = 0x21, /* HCA local attached memory not present: */ CMD_STAT_LAM_NOT_PRE = 0x22, /* Bad management packet (silently discarded): */ CMD_STAT_BAD_PKT = 0x30, /* More outstanding CQEs in CQ than new CQ size: */ CMD_STAT_BAD_SIZE = 0x40, /* Multi Function device support required: */ CMD_STAT_MULTI_FUNC_REQ = 0x50, }; enum { HCR_IN_PARAM_OFFSET = 0x00, HCR_IN_MODIFIER_OFFSET = 0x08, HCR_OUT_PARAM_OFFSET = 0x0c, HCR_TOKEN_OFFSET = 0x14, HCR_STATUS_OFFSET = 0x18, HCR_OPMOD_SHIFT = 12, HCR_T_BIT = 21, HCR_E_BIT = 22, HCR_GO_BIT = 23 }; enum { GO_BIT_TIMEOUT_MSECS = 10000 }; enum mlx4_vlan_transition { MLX4_VLAN_TRANSITION_VST_VST = 0, MLX4_VLAN_TRANSITION_VST_VGT = 1, MLX4_VLAN_TRANSITION_VGT_VST = 2, MLX4_VLAN_TRANSITION_VGT_VGT = 3, }; struct mlx4_cmd_context { struct completion done; int result; int next; u64 out_param; u16 token; u8 fw_status; }; static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, struct mlx4_vhcr_cmd *in_vhcr); static int mlx4_status_to_errno(u8 status) { static const int trans_table[] = { [CMD_STAT_INTERNAL_ERR] = -EIO, [CMD_STAT_BAD_OP] = -EPERM, [CMD_STAT_BAD_PARAM] = -EINVAL, [CMD_STAT_BAD_SYS_STATE] = -ENXIO, [CMD_STAT_BAD_RESOURCE] = -EBADF, [CMD_STAT_RESOURCE_BUSY] = -EBUSY, [CMD_STAT_EXCEED_LIM] = -ENOMEM, [CMD_STAT_BAD_RES_STATE] = -EBADF, [CMD_STAT_BAD_INDEX] = -EBADF, [CMD_STAT_BAD_NVMEM] = -EFAULT, [CMD_STAT_ICM_ERROR] = -ENFILE, [CMD_STAT_BAD_QP_STATE] = -EINVAL, [CMD_STAT_BAD_SEG_PARAM] = -EFAULT, [CMD_STAT_REG_BOUND] = -EBUSY, [CMD_STAT_LAM_NOT_PRE] = -EAGAIN, [CMD_STAT_BAD_PKT] = -EINVAL, [CMD_STAT_BAD_SIZE] = -ENOMEM, [CMD_STAT_MULTI_FUNC_REQ] = -EACCES, }; if (status >= ARRAY_SIZE(trans_table) || (status != CMD_STAT_OK && trans_table[status] == 0)) return -EIO; return trans_table[status]; } static u8 mlx4_errno_to_status(int errno) { switch (errno) { case -EPERM: return CMD_STAT_BAD_OP; case -EINVAL: return CMD_STAT_BAD_PARAM; case -ENXIO: return CMD_STAT_BAD_SYS_STATE; case -EBUSY: return CMD_STAT_RESOURCE_BUSY; case -ENOMEM: return CMD_STAT_EXCEED_LIM; case -ENFILE: return CMD_STAT_ICM_ERROR; default: return CMD_STAT_INTERNAL_ERR; } } static int mlx4_internal_err_ret_value(struct mlx4_dev *dev, u16 op, u8 op_modifier) { switch (op) { case MLX4_CMD_UNMAP_ICM: case MLX4_CMD_UNMAP_ICM_AUX: case MLX4_CMD_UNMAP_FA: case MLX4_CMD_2RST_QP: case MLX4_CMD_HW2SW_EQ: case MLX4_CMD_HW2SW_CQ: case MLX4_CMD_HW2SW_SRQ: case MLX4_CMD_HW2SW_MPT: case MLX4_CMD_CLOSE_HCA: case MLX4_QP_FLOW_STEERING_DETACH: case MLX4_CMD_FREE_RES: case MLX4_CMD_CLOSE_PORT: return CMD_STAT_OK; case MLX4_CMD_QP_ATTACH: /* On Detach case return success */ if (op_modifier == 0) return CMD_STAT_OK; return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); default: return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); } } static int mlx4_closing_cmd_fatal_error(u16 op, u8 fw_status) { /* Any error during the closing commands below is considered fatal */ if (op == MLX4_CMD_CLOSE_HCA || op == MLX4_CMD_HW2SW_EQ || op == MLX4_CMD_HW2SW_CQ || op == MLX4_CMD_2RST_QP || op == MLX4_CMD_HW2SW_SRQ || op == MLX4_CMD_SYNC_TPT || op == MLX4_CMD_UNMAP_ICM || op == MLX4_CMD_UNMAP_ICM_AUX || op == MLX4_CMD_UNMAP_FA) return 1; /* Error on MLX4_CMD_HW2SW_MPT is fatal except when fw status equals * CMD_STAT_REG_BOUND. * This status indicates that memory region has memory windows bound to it * which may result from invalid user space usage and is not fatal. */ if (op == MLX4_CMD_HW2SW_MPT && fw_status != CMD_STAT_REG_BOUND) return 1; return 0; } static int mlx4_cmd_reset_flow(struct mlx4_dev *dev, u16 op, u8 op_modifier, int err) { /* Only if reset flow is really active return code is based on * command, otherwise current error code is returned. */ if (mlx4_internal_err_reset) { mlx4_enter_error_state(dev->persist); err = mlx4_internal_err_ret_value(dev, op, op_modifier); } return err; } static int comm_pending(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); u32 status = readl(&priv->mfunc.comm->slave_read); return (swab32(status) >> 31) != priv->cmd.comm_toggle; } static int mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param) { struct mlx4_priv *priv = mlx4_priv(dev); u32 val; /* To avoid writing to unknown addresses after the device state was * changed to internal error and the function was rest, * check the INTERNAL_ERROR flag which is updated under * device_state_mutex lock. */ mutex_lock(&dev->persist->device_state_mutex); if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { mutex_unlock(&dev->persist->device_state_mutex); return -EIO; } priv->cmd.comm_toggle ^= 1; val = param | (cmd << 16) | (priv->cmd.comm_toggle << 31); __raw_writel((__force u32) cpu_to_be32(val), &priv->mfunc.comm->slave_write); mmiowb(); mutex_unlock(&dev->persist->device_state_mutex); return 0; } static int mlx4_comm_cmd_poll(struct mlx4_dev *dev, u8 cmd, u16 param, unsigned long timeout) { struct mlx4_priv *priv = mlx4_priv(dev); unsigned long end; int err = 0; int ret_from_pending = 0; /* First, verify that the master reports correct status */ if (comm_pending(dev)) { mlx4_warn(dev, "Communication channel is not idle - my toggle is %d (cmd:0x%x)\n", priv->cmd.comm_toggle, cmd); return -EAGAIN; } /* Write command */ down(&priv->cmd.poll_sem); if (mlx4_comm_cmd_post(dev, cmd, param)) { /* Only in case the device state is INTERNAL_ERROR, * mlx4_comm_cmd_post returns with an error */ err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); goto out; } end = msecs_to_jiffies(timeout) + jiffies; while (comm_pending(dev) && time_before(jiffies, end)) cond_resched(); ret_from_pending = comm_pending(dev); if (ret_from_pending) { /* check if the slave is trying to boot in the middle of * FLR process. The only non-zero result in the RESET command * is MLX4_DELAY_RESET_SLAVE*/ if ((MLX4_COMM_CMD_RESET == cmd)) { err = MLX4_DELAY_RESET_SLAVE; goto out; } else { mlx4_warn(dev, "Communication channel command 0x%x timed out\n", cmd); err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); } } if (err) mlx4_enter_error_state(dev->persist); out: up(&priv->cmd.poll_sem); return err; } static int mlx4_comm_cmd_wait(struct mlx4_dev *dev, u8 vhcr_cmd, u16 param, u16 op, unsigned long timeout) { struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; struct mlx4_cmd_context *context; unsigned long end; int err = 0; down(&cmd->event_sem); spin_lock(&cmd->context_lock); BUG_ON(cmd->free_head < 0); context = &cmd->context[cmd->free_head]; context->token += cmd->token_mask + 1; cmd->free_head = context->next; spin_unlock(&cmd->context_lock); reinit_completion(&context->done); if (mlx4_comm_cmd_post(dev, vhcr_cmd, param)) { /* Only in case the device state is INTERNAL_ERROR, * mlx4_comm_cmd_post returns with an error */ err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); goto out; } if (!wait_for_completion_timeout(&context->done, msecs_to_jiffies(timeout))) { mlx4_warn(dev, "communication channel command 0x%x (op=0x%x) timed out\n", vhcr_cmd, op); goto out_reset; } err = context->result; if (err && context->fw_status != CMD_STAT_MULTI_FUNC_REQ) { mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", vhcr_cmd, context->fw_status); if (mlx4_closing_cmd_fatal_error(op, context->fw_status)) goto out_reset; } /* wait for comm channel ready * this is necessary for prevention the race * when switching between event to polling mode * Skipping this section in case the device is in FATAL_ERROR state, * In this state, no commands are sent via the comm channel until * the device has returned from reset. */ if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) { end = msecs_to_jiffies(timeout) + jiffies; while (comm_pending(dev) && time_before(jiffies, end)) cond_resched(); } goto out; out_reset: err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); mlx4_enter_error_state(dev->persist); out: spin_lock(&cmd->context_lock); context->next = cmd->free_head; cmd->free_head = context - cmd->context; spin_unlock(&cmd->context_lock); up(&cmd->event_sem); return err; } int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, u16 op, unsigned long timeout) { if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); if (mlx4_priv(dev)->cmd.use_events) return mlx4_comm_cmd_wait(dev, cmd, param, op, timeout); return mlx4_comm_cmd_poll(dev, cmd, param, timeout); } static int cmd_pending(struct mlx4_dev *dev) { u32 status; if (pci_channel_offline(dev->persist->pdev)) return -EIO; status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); return (status & swab32(1 << HCR_GO_BIT)) || (mlx4_priv(dev)->cmd.toggle == !!(status & swab32(1 << HCR_T_BIT))); } static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token, int event) { struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; u32 __iomem *hcr = cmd->hcr; int ret = -EIO; unsigned long end; mutex_lock(&dev->persist->device_state_mutex); /* To avoid writing to unknown addresses after the device state was * changed to internal error and the chip was reset, * check the INTERNAL_ERROR flag which is updated under * device_state_mutex lock. */ if (pci_channel_offline(dev->persist->pdev) || (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) { /* * Device is going through error recovery * and cannot accept commands. */ goto out; } end = jiffies; if (event) end += msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS); while (cmd_pending(dev)) { if (pci_channel_offline(dev->persist->pdev)) { /* * Device is going through error recovery * and cannot accept commands. */ goto out; } if (time_after_eq(jiffies, end)) { mlx4_err(dev, "%s:cmd_pending failed\n", __func__); goto out; } cond_resched(); } /* * We use writel (instead of something like memcpy_toio) * because writes of less than 32 bits to the HCR don't work * (and some architectures such as ia64 implement memcpy_toio * in terms of writeb). */ __raw_writel((__force u32) cpu_to_be32(in_param >> 32), hcr + 0); __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), hcr + 1); __raw_writel((__force u32) cpu_to_be32(in_modifier), hcr + 2); __raw_writel((__force u32) cpu_to_be32(out_param >> 32), hcr + 3); __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), hcr + 4); __raw_writel((__force u32) cpu_to_be32(token << 16), hcr + 5); /* __raw_writel may not order writes. */ wmb(); __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | (cmd->toggle << HCR_T_BIT) | (event ? (1 << HCR_E_BIT) : 0) | (op_modifier << HCR_OPMOD_SHIFT) | op), hcr + 6); /* * Make sure that our HCR writes don't get mixed in with * writes from another CPU starting a FW command. */ mmiowb(); cmd->toggle = cmd->toggle ^ 1; ret = 0; out: if (ret) mlx4_warn(dev, "Could not post command 0x%x: ret=%d, in_param=0x%llx, in_mod=0x%x, op_mod=0x%x\n", op, ret, (long long)in_param, in_modifier, op_modifier); mutex_unlock(&dev->persist->device_state_mutex); return ret; } static int mlx4_slave_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_vhcr_cmd *vhcr = priv->mfunc.vhcr; int ret; mutex_lock(&priv->cmd.slave_cmd_mutex); vhcr->in_param = cpu_to_be64(in_param); vhcr->out_param = out_param ? cpu_to_be64(*out_param) : 0; vhcr->in_modifier = cpu_to_be32(in_modifier); vhcr->opcode = cpu_to_be16((((u16) op_modifier) << 12) | (op & 0xfff)); vhcr->token = cpu_to_be16(CMD_POLL_TOKEN); vhcr->status = 0; vhcr->flags = !!(priv->cmd.use_events) << 6; if (mlx4_is_master(dev)) { ret = mlx4_master_process_vhcr(dev, dev->caps.function, vhcr); if (!ret) { if (out_is_imm) { if (out_param) *out_param = be64_to_cpu(vhcr->out_param); else { mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", op); vhcr->status = CMD_STAT_BAD_PARAM; } } ret = mlx4_status_to_errno(vhcr->status); } if (ret && dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) ret = mlx4_internal_err_ret_value(dev, op, op_modifier); } else { ret = mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_POST, 0, op, MLX4_COMM_TIME + timeout); if (!ret) { if (out_is_imm) { if (out_param) *out_param = be64_to_cpu(vhcr->out_param); else { mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", op); vhcr->status = CMD_STAT_BAD_PARAM; } } ret = mlx4_status_to_errno(vhcr->status); } else { if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) ret = mlx4_internal_err_ret_value(dev, op, op_modifier); else mlx4_err(dev, "failed execution of VHCR_POST command opcode 0x%x\n", op); } } mutex_unlock(&priv->cmd.slave_cmd_mutex); return ret; } static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { struct mlx4_priv *priv = mlx4_priv(dev); void __iomem *hcr = priv->cmd.hcr; int err = 0; unsigned long end; u32 stat; down(&priv->cmd.poll_sem); if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { /* * Device is going through error recovery * and cannot accept commands. */ err = mlx4_internal_err_ret_value(dev, op, op_modifier); goto out; } if (out_is_imm && !out_param) { mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", op); err = -EINVAL; goto out; } err = mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); if (err) goto out_reset; end = msecs_to_jiffies(timeout) + jiffies; while (cmd_pending(dev) && time_before(jiffies, end)) { if (pci_channel_offline(dev->persist->pdev)) { /* * Device is going through error recovery * and cannot accept commands. */ err = -EIO; goto out_reset; } if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { err = mlx4_internal_err_ret_value(dev, op, op_modifier); goto out; } cond_resched(); } if (cmd_pending(dev)) { mlx4_warn(dev, "command 0x%x timed out (go bit not cleared)\n", op); err = -EIO; goto out_reset; } if (out_is_imm) *out_param = (u64) be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_OUT_PARAM_OFFSET)) << 32 | (u64) be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_OUT_PARAM_OFFSET + 4)); stat = be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; err = mlx4_status_to_errno(stat); if (err) { mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", op, stat); if (mlx4_closing_cmd_fatal_error(op, stat)) goto out_reset; goto out; } out_reset: if (err) err = mlx4_cmd_reset_flow(dev, op, op_modifier, err); out: up(&priv->cmd.poll_sem); return err; } void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_context *context = &priv->cmd.context[token & priv->cmd.token_mask]; /* previously timed out command completing at long last */ if (token != context->token) return; context->fw_status = status; context->result = mlx4_status_to_errno(status); context->out_param = out_param; complete(&context->done); } static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; struct mlx4_cmd_context *context; long ret_wait; int err = 0; down(&cmd->event_sem); spin_lock(&cmd->context_lock); BUG_ON(cmd->free_head < 0); context = &cmd->context[cmd->free_head]; context->token += cmd->token_mask + 1; cmd->free_head = context->next; spin_unlock(&cmd->context_lock); if (out_is_imm && !out_param) { mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", op); err = -EINVAL; goto out; } reinit_completion(&context->done); err = mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, in_modifier, op_modifier, op, context->token, 1); if (err) goto out_reset; if (op == MLX4_CMD_SENSE_PORT) { ret_wait = wait_for_completion_interruptible_timeout(&context->done, msecs_to_jiffies(timeout)); if (ret_wait < 0) { context->fw_status = 0; context->out_param = 0; context->result = 0; } } else { ret_wait = (long)wait_for_completion_timeout(&context->done, msecs_to_jiffies(timeout)); } if (!ret_wait) { mlx4_warn(dev, "command 0x%x timed out (go bit not cleared)\n", op); if (op == MLX4_CMD_NOP) { err = -EBUSY; goto out; } else { err = -EIO; goto out_reset; } } err = context->result; if (err) { /* Since we do not want to have this error message always * displayed at driver start when there are ConnectX2 HCAs * on the host, we deprecate the error message for this * specific command/input_mod/opcode_mod/fw-status to be debug. */ if (op == MLX4_CMD_SET_PORT && (in_modifier == 1 || in_modifier == 2) && op_modifier == MLX4_SET_PORT_IB_OPCODE && context->fw_status == CMD_STAT_BAD_SIZE) mlx4_dbg(dev, "command 0x%x failed: fw status = 0x%x\n", op, context->fw_status); else mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", op, context->fw_status); if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) err = mlx4_internal_err_ret_value(dev, op, op_modifier); else if (mlx4_closing_cmd_fatal_error(op, context->fw_status)) goto out_reset; goto out; } if (out_is_imm) *out_param = context->out_param; out_reset: if (err) err = mlx4_cmd_reset_flow(dev, op, op_modifier, err); out: spin_lock(&cmd->context_lock); context->next = cmd->free_head; cmd->free_head = context - cmd->context; spin_unlock(&cmd->context_lock); up(&cmd->event_sem); return err; } int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, int native) { if (pci_channel_offline(dev->persist->pdev)) return mlx4_cmd_reset_flow(dev, op, op_modifier, -EIO); if (!mlx4_is_mfunc(dev) || (native && mlx4_is_master(dev))) { int ret; if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) return mlx4_internal_err_ret_value(dev, op, op_modifier); down_read(&mlx4_priv(dev)->cmd.switch_sem); if (mlx4_priv(dev)->cmd.use_events) ret = mlx4_cmd_wait(dev, in_param, out_param, out_is_imm, in_modifier, op_modifier, op, timeout); else ret = mlx4_cmd_poll(dev, in_param, out_param, out_is_imm, in_modifier, op_modifier, op, timeout); up_read(&mlx4_priv(dev)->cmd.switch_sem); return ret; } return mlx4_slave_cmd(dev, in_param, out_param, out_is_imm, in_modifier, op_modifier, op, timeout); } EXPORT_SYMBOL_GPL(__mlx4_cmd); int mlx4_ARM_COMM_CHANNEL(struct mlx4_dev *dev) { return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_ARM_COMM_CHANNEL, MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, int slave, u64 slave_addr, int size, int is_read) { u64 in_param; u64 out_param; if ((slave_addr & 0xfff) | (master_addr & 0xfff) | (slave & ~0x7f) | (size & 0xff)) { mlx4_err(dev, "Bad access mem params - slave_addr:0x%llx " "master_addr:0x%llx slave_id:%d size:%d\n", (unsigned long long)slave_addr, (unsigned long long)master_addr, slave, size); return -EINVAL; } if (is_read) { in_param = (u64) slave | slave_addr; out_param = (u64) dev->caps.function | master_addr; } else { in_param = (u64) dev->caps.function | master_addr; out_param = (u64) slave | slave_addr; } return mlx4_cmd_imm(dev, in_param, &out_param, size, 0, MLX4_CMD_ACCESS_MEM, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } static int query_pkey_block(struct mlx4_dev *dev, u8 port, u16 index, u16 *pkey, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox) { struct ib_smp *in_mad = (struct ib_smp *)(inbox->buf); struct ib_smp *out_mad = (struct ib_smp *)(outbox->buf); int err; int i; if (index & 0x1f) return -EINVAL; in_mad->attr_mod = cpu_to_be32(index / 32); err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); if (err) return err; for (i = 0; i < 32; ++i) pkey[i] = be16_to_cpu(((__be16 *) out_mad->data)[i]); return err; } static int get_full_pkey_table(struct mlx4_dev *dev, u8 port, u16 *table, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox) { int i; int err; for (i = 0; i < dev->caps.pkey_table_len[port]; i += 32) { err = query_pkey_block(dev, port, i, table + i, inbox, outbox); if (err) return err; } return 0; } #define PORT_CAPABILITY_LOCATION_IN_SMP 20 #define PORT_STATE_OFFSET 32 static enum ib_port_state vf_port_state(struct mlx4_dev *dev, int port, int vf) { if (mlx4_get_slave_port_state(dev, vf, port) == SLAVE_PORT_UP) return IB_PORT_ACTIVE; else return IB_PORT_DOWN; } static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox, struct mlx4_cmd_info *cmd) { struct ib_smp *smp = inbox->buf; u32 index; u8 port, slave_port; u8 opcode_modifier; u16 *table; int err; int vidx, pidx; int network_view; struct mlx4_priv *priv = mlx4_priv(dev); struct ib_smp *outsmp = outbox->buf; __be16 *outtab = (__be16 *)(outsmp->data); __be32 slave_cap_mask; __be64 slave_node_guid; slave_port = vhcr->in_modifier; port = mlx4_slave_convert_port(dev, slave, slave_port); /* network-view bit is for driver use only, and should not be passed to FW */ opcode_modifier = vhcr->op_modifier & ~0x8; /* clear netw view bit */ network_view = !!(vhcr->op_modifier & 0x8); if (smp->base_version == 1 && smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && smp->class_version == 1) { /* host view is paravirtualized */ if (!network_view && smp->method == IB_MGMT_METHOD_GET) { if (smp->attr_id == IB_SMP_ATTR_PKEY_TABLE) { index = be32_to_cpu(smp->attr_mod); if (port < 1 || port > dev->caps.num_ports) return -EINVAL; table = kcalloc((dev->caps.pkey_table_len[port] / 32) + 1, sizeof(*table) * 32, GFP_KERNEL); if (!table) return -ENOMEM; /* need to get the full pkey table because the paravirtualized * pkeys may be scattered among several pkey blocks. */ err = get_full_pkey_table(dev, port, table, inbox, outbox); if (!err) { for (vidx = index * 32; vidx < (index + 1) * 32; ++vidx) { pidx = priv->virt2phys_pkey[slave][port - 1][vidx]; outtab[vidx % 32] = cpu_to_be16(table[pidx]); } } kfree(table); return err; } if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) { /*get the slave specific caps:*/ /*do the command */ smp->attr_mod = cpu_to_be32(port); err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); /* modify the response for slaves */ if (!err && slave != mlx4_master_func_num(dev)) { u8 *state = outsmp->data + PORT_STATE_OFFSET; if (port < 1 || port > dev->caps.num_ports) return -EINVAL; *state = (*state & 0xf0) | vf_port_state(dev, port, slave); slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; memcpy(outsmp->data + PORT_CAPABILITY_LOCATION_IN_SMP, &slave_cap_mask, 4); } return err; } if (smp->attr_id == IB_SMP_ATTR_GUID_INFO) { __be64 guid; if (port < 1 || port > dev->caps.num_ports) return -EINVAL; guid = mlx4_get_admin_guid(dev, slave, port); /* set the PF admin guid to the FW/HW burned * GUID, if it wasn't yet set */ if (slave == 0 && guid == 0) { smp->attr_mod = 0; err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, vhcr->in_modifier, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); if (err) return err; mlx4_set_admin_guid(dev, *(__be64 *)outsmp-> data, slave, port); } else { memcpy(outsmp->data, &guid, 8); } /* clean all other gids */ memset(outsmp->data + 8, 0, 56); return 0; } if (smp->attr_id == IB_SMP_ATTR_NODE_INFO) { err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); if (!err) { slave_node_guid = mlx4_get_slave_node_guid(dev, slave); memcpy(outsmp->data + 12, &slave_node_guid, 8); } return err; } } } /* Non-privileged VFs are only allowed "host" view LID-routed 'Get' MADs. * These are the MADs used by ib verbs (such as ib_query_gids). */ if (slave != mlx4_master_func_num(dev) && !mlx4_vf_smi_enabled(dev, slave, port)) { if (!(smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && smp->method == IB_MGMT_METHOD_GET) || network_view) { mlx4_err(dev, "Unprivileged slave %d is trying to execute a Subnet MGMT MAD, class 0x%x, method 0x%x, view=%s for attr 0x%x. Rejecting\n", slave, smp->mgmt_class, smp->method, network_view ? "Network" : "Host", be16_to_cpu(smp->attr_id)); return -EPERM; } } return mlx4_cmd_box(dev, inbox->dma, outbox->dma, vhcr->in_modifier, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); } static int mlx4_CMD_EPERM_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox, struct mlx4_cmd_info *cmd) { return -EPERM; } int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox, struct mlx4_cmd_info *cmd) { u64 in_param; u64 out_param; int err; in_param = cmd->has_inbox ? (u64) inbox->dma : vhcr->in_param; out_param = cmd->has_outbox ? (u64) outbox->dma : vhcr->out_param; if (cmd->encode_slave_id) { in_param &= 0xffffffffffffff00ll; in_param |= slave; } err = __mlx4_cmd(dev, in_param, &out_param, cmd->out_is_imm, vhcr->in_modifier, vhcr->op_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (cmd->out_is_imm) vhcr->out_param = out_param; return err; } static struct mlx4_cmd_info cmd_info[] = { { .opcode = MLX4_CMD_QUERY_FW, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_FW_wrapper }, { .opcode = MLX4_CMD_QUERY_HCA, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_QUERY_DEV_CAP, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_DEV_CAP_wrapper }, { .opcode = MLX4_CMD_QUERY_FUNC_CAP, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_FUNC_CAP_wrapper }, { .opcode = MLX4_CMD_QUERY_ADAPTER, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_INIT_PORT, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_INIT_PORT_wrapper }, { .opcode = MLX4_CMD_CLOSE_PORT, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CLOSE_PORT_wrapper }, { .opcode = MLX4_CMD_QUERY_PORT, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_PORT_wrapper }, { .opcode = MLX4_CMD_SET_PORT, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SET_PORT_wrapper }, { .opcode = MLX4_CMD_MAP_EQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_MAP_EQ_wrapper }, { .opcode = MLX4_CMD_SW2HW_EQ, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_SW2HW_EQ_wrapper }, { .opcode = MLX4_CMD_HW_HEALTH_CHECK, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_NOP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_CONFIG_DEV, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CONFIG_DEV_wrapper }, { .opcode = MLX4_CMD_ALLOC_RES, .has_inbox = false, .has_outbox = false, .out_is_imm = true, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_ALLOC_RES_wrapper }, { .opcode = MLX4_CMD_FREE_RES, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_FREE_RES_wrapper }, { .opcode = MLX4_CMD_SW2HW_MPT, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_SW2HW_MPT_wrapper }, { .opcode = MLX4_CMD_QUERY_MPT, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_MPT_wrapper }, { .opcode = MLX4_CMD_HW2SW_MPT, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_HW2SW_MPT_wrapper }, { .opcode = MLX4_CMD_READ_MTT, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_WRITE_MTT, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_WRITE_MTT_wrapper }, { .opcode = MLX4_CMD_SYNC_TPT, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, { .opcode = MLX4_CMD_HW2SW_EQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_HW2SW_EQ_wrapper }, { .opcode = MLX4_CMD_QUERY_EQ, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_QUERY_EQ_wrapper }, { .opcode = MLX4_CMD_SW2HW_CQ, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_SW2HW_CQ_wrapper }, { .opcode = MLX4_CMD_HW2SW_CQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_HW2SW_CQ_wrapper }, { .opcode = MLX4_CMD_QUERY_CQ, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_CQ_wrapper }, { .opcode = MLX4_CMD_MODIFY_CQ, .has_inbox = true, .has_outbox = false, .out_is_imm = true, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_MODIFY_CQ_wrapper }, { .opcode = MLX4_CMD_SW2HW_SRQ, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_SW2HW_SRQ_wrapper }, { .opcode = MLX4_CMD_HW2SW_SRQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_HW2SW_SRQ_wrapper }, { .opcode = MLX4_CMD_QUERY_SRQ, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_SRQ_wrapper }, { .opcode = MLX4_CMD_ARM_SRQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_ARM_SRQ_wrapper }, { .opcode = MLX4_CMD_RST2INIT_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = true, .verify = NULL, .wrapper = mlx4_RST2INIT_QP_wrapper }, { .opcode = MLX4_CMD_INIT2INIT_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_INIT2INIT_QP_wrapper }, { .opcode = MLX4_CMD_INIT2RTR_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_INIT2RTR_QP_wrapper }, { .opcode = MLX4_CMD_RTR2RTS_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_RTR2RTS_QP_wrapper }, { .opcode = MLX4_CMD_RTS2RTS_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_RTS2RTS_QP_wrapper }, { .opcode = MLX4_CMD_SQERR2RTS_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SQERR2RTS_QP_wrapper }, { .opcode = MLX4_CMD_2ERR_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_GEN_QP_wrapper }, { .opcode = MLX4_CMD_RTS2SQD_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_GEN_QP_wrapper }, { .opcode = MLX4_CMD_SQD2SQD_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SQD2SQD_QP_wrapper }, { .opcode = MLX4_CMD_SQD2RTS_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SQD2RTS_QP_wrapper }, { .opcode = MLX4_CMD_2RST_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_2RST_QP_wrapper }, { .opcode = MLX4_CMD_QUERY_QP, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_GEN_QP_wrapper }, { .opcode = MLX4_CMD_SUSPEND_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_GEN_QP_wrapper }, { .opcode = MLX4_CMD_UNSUSPEND_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_GEN_QP_wrapper }, { .opcode = MLX4_CMD_UPDATE_QP, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_UPDATE_QP_wrapper }, { .opcode = MLX4_CMD_GET_OP_REQ, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper, }, { .opcode = MLX4_CMD_ALLOCATE_VPP, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper, }, { .opcode = MLX4_CMD_SET_VPORT_QOS, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper, }, { .opcode = MLX4_CMD_CONF_SPECIAL_QP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, /* XXX verify: only demux can do this */ .wrapper = NULL }, { .opcode = MLX4_CMD_MAD_IFC, .has_inbox = true, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_MAD_IFC_wrapper }, { .opcode = MLX4_CMD_MAD_DEMUX, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper }, { .opcode = MLX4_CMD_QUERY_IF_STAT, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QUERY_IF_STAT_wrapper }, { .opcode = MLX4_CMD_ACCESS_REG, .has_inbox = true, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_ACCESS_REG_wrapper, }, { .opcode = MLX4_CMD_CONGESTION_CTRL_OPCODE, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper, }, /* Native multicast commands are not available for guests */ { .opcode = MLX4_CMD_QP_ATTACH, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QP_ATTACH_wrapper }, { .opcode = MLX4_CMD_PROMISC, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_PROMISC_wrapper }, /* Ethernet specific commands */ { .opcode = MLX4_CMD_SET_VLAN_FLTR, .has_inbox = true, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SET_VLAN_FLTR_wrapper }, { .opcode = MLX4_CMD_SET_MCAST_FLTR, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_SET_MCAST_FLTR_wrapper }, { .opcode = MLX4_CMD_DUMP_ETH_STATS, .has_inbox = false, .has_outbox = true, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_DUMP_ETH_STATS_wrapper }, { .opcode = MLX4_CMD_INFORM_FLR_DONE, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = NULL }, /* flow steering commands */ { .opcode = MLX4_QP_FLOW_STEERING_ATTACH, .has_inbox = true, .has_outbox = false, .out_is_imm = true, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QP_FLOW_STEERING_ATTACH_wrapper }, { .opcode = MLX4_QP_FLOW_STEERING_DETACH, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_QP_FLOW_STEERING_DETACH_wrapper }, { .opcode = MLX4_FLOW_STEERING_IB_UC_QP_RANGE, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper }, { .opcode = MLX4_CMD_VIRT_PORT_MAP, .has_inbox = false, .has_outbox = false, .out_is_imm = false, .encode_slave_id = false, .verify = NULL, .wrapper = mlx4_CMD_EPERM_wrapper }, }; static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, struct mlx4_vhcr_cmd *in_vhcr) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_info *cmd = NULL; struct mlx4_vhcr_cmd *vhcr_cmd = in_vhcr ? in_vhcr : priv->mfunc.vhcr; struct mlx4_vhcr *vhcr; struct mlx4_cmd_mailbox *inbox = NULL; struct mlx4_cmd_mailbox *outbox = NULL; u64 in_param; u64 out_param; int ret = 0; int i; int err = 0; /* Create sw representation of Virtual HCR */ vhcr = kzalloc(sizeof(struct mlx4_vhcr), GFP_KERNEL); if (!vhcr) return -ENOMEM; /* DMA in the vHCR */ if (!in_vhcr) { ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, priv->mfunc.master.slave_state[slave].vhcr_dma, ALIGN(sizeof(struct mlx4_vhcr_cmd), MLX4_ACCESS_MEM_ALIGN), 1); if (ret) { if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) mlx4_err(dev, "%s: Failed reading vhcr ret: 0x%x\n", __func__, ret); kfree(vhcr); return ret; } } /* Fill SW VHCR fields */ vhcr->in_param = be64_to_cpu(vhcr_cmd->in_param); vhcr->out_param = be64_to_cpu(vhcr_cmd->out_param); vhcr->in_modifier = be32_to_cpu(vhcr_cmd->in_modifier); vhcr->token = be16_to_cpu(vhcr_cmd->token); vhcr->op = be16_to_cpu(vhcr_cmd->opcode) & 0xfff; vhcr->op_modifier = (u8) (be16_to_cpu(vhcr_cmd->opcode) >> 12); vhcr->e_bit = vhcr_cmd->flags & (1 << 6); /* Lookup command */ for (i = 0; i < ARRAY_SIZE(cmd_info); ++i) { if (vhcr->op == cmd_info[i].opcode) { cmd = &cmd_info[i]; break; } } if (!cmd) { mlx4_err(dev, "Unknown command:0x%x accepted from slave:%d\n", vhcr->op, slave); vhcr_cmd->status = CMD_STAT_BAD_PARAM; goto out_status; } /* Read inbox */ if (cmd->has_inbox) { vhcr->in_param &= INBOX_MASK; inbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(inbox)) { vhcr_cmd->status = CMD_STAT_BAD_SIZE; inbox = NULL; goto out_status; } ret = mlx4_ACCESS_MEM(dev, inbox->dma, slave, vhcr->in_param, MLX4_MAILBOX_SIZE, 1); if (ret) { if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) mlx4_err(dev, "%s: Failed reading inbox (cmd:0x%x)\n", __func__, cmd->opcode); vhcr_cmd->status = CMD_STAT_INTERNAL_ERR; goto out_status; } } /* Apply permission and bound checks if applicable */ if (cmd->verify && cmd->verify(dev, slave, vhcr, inbox)) { mlx4_warn(dev, "Command:0x%x from slave: %d failed protection checks for resource_id:%d\n", vhcr->op, slave, vhcr->in_modifier); vhcr_cmd->status = CMD_STAT_BAD_OP; goto out_status; } /* Allocate outbox */ if (cmd->has_outbox) { outbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(outbox)) { vhcr_cmd->status = CMD_STAT_BAD_SIZE; outbox = NULL; goto out_status; } } /* Execute the command! */ if (cmd->wrapper) { err = cmd->wrapper(dev, slave, vhcr, inbox, outbox, cmd); if (cmd->out_is_imm) vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); } else { in_param = cmd->has_inbox ? (u64) inbox->dma : vhcr->in_param; out_param = cmd->has_outbox ? (u64) outbox->dma : vhcr->out_param; err = __mlx4_cmd(dev, in_param, &out_param, cmd->out_is_imm, vhcr->in_modifier, vhcr->op_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (cmd->out_is_imm) { vhcr->out_param = out_param; vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); } } if (err) { if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) mlx4_warn(dev, "vhcr command:0x%x slave:%d failed with error:%d, status %d\n", vhcr->op, slave, vhcr->errno, err); vhcr_cmd->status = mlx4_errno_to_status(err); goto out_status; } /* Write outbox if command completed successfully */ if (cmd->has_outbox && !vhcr_cmd->status) { ret = mlx4_ACCESS_MEM(dev, outbox->dma, slave, vhcr->out_param, MLX4_MAILBOX_SIZE, MLX4_CMD_WRAPPED); if (ret) { /* If we failed to write back the outbox after the *command was successfully executed, we must fail this * slave, as it is now in undefined state */ if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) mlx4_err(dev, "%s:Failed writing outbox\n", __func__); goto out; } } out_status: /* DMA back vhcr result */ if (!in_vhcr) { ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, priv->mfunc.master.slave_state[slave].vhcr_dma, ALIGN(sizeof(struct mlx4_vhcr), MLX4_ACCESS_MEM_ALIGN), MLX4_CMD_WRAPPED); if (ret) mlx4_err(dev, "%s:Failed writing vhcr result\n", __func__); else if (vhcr->e_bit && mlx4_GEN_EQE(dev, slave, &priv->mfunc.master.cmd_eqe)) mlx4_warn(dev, "Failed to generate command completion eqe for slave %d\n", slave); } out: kfree(vhcr); mlx4_free_cmd_mailbox(dev, inbox); mlx4_free_cmd_mailbox(dev, outbox); return ret; } static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv, int slave, int port) { struct mlx4_vport_oper_state *vp_oper; struct mlx4_vport_state *vp_admin; struct mlx4_vf_immed_vlan_work *work; struct mlx4_dev *dev = &priv->dev; int err; int admin_vlan_ix = NO_INDX; vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; if (vp_oper->state.default_vlan == vp_admin->default_vlan && vp_oper->state.default_qos == vp_admin->default_qos && vp_oper->state.vlan_proto == vp_admin->vlan_proto && vp_oper->state.qos_vport == vp_admin->qos_vport) return 0; if (!(priv->mfunc.master.slave_state[slave].active && dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP)) { /* even if the UPDATE_QP command isn't supported, we still want * to set this VF link according to the admin directive */ return -1; } mlx4_dbg(dev, "updating immediately admin params slave %d port %d\n", slave, port); mlx4_dbg(dev, "vlan %d QoS %d link down\n", vp_admin->default_vlan, vp_admin->default_qos); work = kzalloc(sizeof(*work), GFP_KERNEL); if (!work) return -ENOMEM; if (vp_oper->state.default_vlan != vp_admin->default_vlan) { if (MLX4_VGT != vp_admin->default_vlan) { err = __mlx4_register_vlan(&priv->dev, port, vp_admin->default_vlan, &admin_vlan_ix); if (err) { kfree(work); mlx4_warn(&priv->dev, "No vlan resources slave %d, port %d\n", slave, port); return err; } } else { admin_vlan_ix = NO_INDX; } work->flags |= MLX4_VF_IMMED_VLAN_FLAG_VLAN; mlx4_dbg(&priv->dev, "alloc vlan %d idx %d slave %d port %d\n", (int)(vp_admin->default_vlan), admin_vlan_ix, slave, port); } /* save original vlan ix and vlan id */ work->orig_vlan_id = vp_oper->state.default_vlan; work->orig_vlan_ix = vp_oper->vlan_idx; /* handle new qos */ if (vp_oper->state.default_qos != vp_admin->default_qos) work->flags |= MLX4_VF_IMMED_VLAN_FLAG_QOS; if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN) vp_oper->vlan_idx = admin_vlan_ix; vp_oper->state.default_vlan = vp_admin->default_vlan; vp_oper->state.default_qos = vp_admin->default_qos; vp_oper->state.vlan_proto = vp_admin->vlan_proto; vp_oper->state.qos_vport = vp_admin->qos_vport; if (1 /* vp_admin->link_state == IFLA_VF_LINK_STATE_DISABLE */) work->flags |= MLX4_VF_IMMED_VLAN_FLAG_LINK_DISABLE; /* iterate over QPs owned by this slave, using UPDATE_QP */ work->port = port; work->slave = slave; work->qos = vp_oper->state.default_qos; work->qos_vport = vp_oper->state.qos_vport; work->vlan_id = vp_oper->state.default_vlan; work->vlan_ix = vp_oper->vlan_idx; work->vlan_proto = vp_oper->state.vlan_proto; work->priv = priv; INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler); queue_work(priv->mfunc.master.comm_wq, &work->work); return 0; } static void mlx4_set_default_port_qos(struct mlx4_dev *dev, int port) { struct mlx4_qos_manager *port_qos_ctl; struct mlx4_priv *priv = mlx4_priv(dev); port_qos_ctl = &priv->mfunc.master.qos_ctl[port]; bitmap_zero(port_qos_ctl->priority_bm, MLX4_NUM_UP); /* Enable only default prio at PF init routine */ set_bit(MLX4_DEFAULT_QOS_PRIO, port_qos_ctl->priority_bm); } static void mlx4_allocate_port_vpps(struct mlx4_dev *dev, int port) { int i; int err; int num_vfs; u16 availible_vpp; u8 vpp_param[MLX4_NUM_UP]; struct mlx4_qos_manager *port_qos; struct mlx4_priv *priv = mlx4_priv(dev); err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param); if (err) { mlx4_info(dev, "Failed query availible VPPs\n"); return; } port_qos = &priv->mfunc.master.qos_ctl[port]; num_vfs = (availible_vpp / bitmap_weight(port_qos->priority_bm, MLX4_NUM_UP)); for (i = 0; i < MLX4_NUM_UP; i++) { if (test_bit(i, port_qos->priority_bm)) vpp_param[i] = num_vfs; } err = mlx4_ALLOCATE_VPP_set(dev, port, vpp_param); if (err) { mlx4_info(dev, "Failed allocating VPPs\n"); return; } /* Query actual allocated VPP, just to make sure */ err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param); if (err) { mlx4_info(dev, "Failed query availible VPPs\n"); return; } port_qos->num_of_qos_vfs = num_vfs; mlx4_dbg(dev, "Port %d Availible VPPs %d\n", port, availible_vpp); for (i = 0; i < MLX4_NUM_UP; i++) mlx4_dbg(dev, "Port %d UP %d Allocated %d VPPs\n", port, i, vpp_param[i]); } static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave) { int port, err; struct mlx4_vport_state *vp_admin; struct mlx4_vport_oper_state *vp_oper; struct mlx4_slave_state *slave_state = &priv->mfunc.master.slave_state[slave]; struct mlx4_active_ports actv_ports = mlx4_get_active_ports( &priv->dev, slave); int min_port = find_first_bit(actv_ports.ports, priv->dev.caps.num_ports) + 1; int max_port = min_port - 1 + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); for (port = min_port; port <= max_port; port++) { if (!test_bit(port - 1, actv_ports.ports)) continue; priv->mfunc.master.vf_oper[slave].smi_enabled[port] = priv->mfunc.master.vf_admin[slave].enable_smi[port]; vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; if (vp_admin->vlan_proto != htons(ETH_P_8021AD) || slave_state->vst_qinq_supported) { vp_oper->state.vlan_proto = vp_admin->vlan_proto; vp_oper->state.default_vlan = vp_admin->default_vlan; vp_oper->state.default_qos = vp_admin->default_qos; } vp_oper->state.mac = vp_admin->mac; vp_oper->state.spoofchk = vp_admin->spoofchk; vp_oper->state.tx_rate = vp_admin->tx_rate; vp_oper->state.qos_vport = vp_admin->qos_vport; vp_oper->state.guid = vp_admin->guid; if (MLX4_VGT != vp_admin->default_vlan) { err = __mlx4_register_vlan(&priv->dev, port, vp_admin->default_vlan, &(vp_oper->vlan_idx)); if (err) { vp_oper->vlan_idx = NO_INDX; vp_oper->state.default_vlan = MLX4_VGT; vp_oper->state.vlan_proto = htons(ETH_P_8021Q); mlx4_warn(&priv->dev, "No vlan resources slave %d, port %d\n", slave, port); return err; } mlx4_dbg(&priv->dev, "alloc vlan %d idx %d slave %d port %d\n", (int)(vp_oper->state.default_vlan), vp_oper->vlan_idx, slave, port); } if (vp_admin->spoofchk) { vp_oper->mac_idx = __mlx4_register_mac(&priv->dev, port, vp_admin->mac); if (0 > vp_oper->mac_idx) { err = vp_oper->mac_idx; vp_oper->mac_idx = NO_INDX; mlx4_warn(&priv->dev, "No mac resources slave %d, port %d\n", slave, port); return err; } mlx4_dbg(&priv->dev, "alloc mac %llx idx %d slave %d port %d\n", (unsigned long long) vp_oper->state.mac, vp_oper->mac_idx, slave, port); } } return 0; } static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave) { int port; struct mlx4_vport_oper_state *vp_oper; struct mlx4_active_ports actv_ports = mlx4_get_active_ports( &priv->dev, slave); int min_port = find_first_bit(actv_ports.ports, priv->dev.caps.num_ports) + 1; int max_port = min_port - 1 + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); for (port = min_port; port <= max_port; port++) { if (!test_bit(port - 1, actv_ports.ports)) continue; priv->mfunc.master.vf_oper[slave].smi_enabled[port] = MLX4_VF_SMI_DISABLED; vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; if (NO_INDX != vp_oper->vlan_idx) { __mlx4_unregister_vlan(&priv->dev, port, vp_oper->state.default_vlan); vp_oper->vlan_idx = NO_INDX; } if (NO_INDX != vp_oper->mac_idx) { __mlx4_unregister_mac(&priv->dev, port, vp_oper->state.mac); vp_oper->mac_idx = NO_INDX; } } return; } static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd, u16 param, u8 toggle) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; u32 reply; u8 is_going_down = 0; int i; unsigned long flags; slave_state[slave].comm_toggle ^= 1; reply = (u32) slave_state[slave].comm_toggle << 31; if (toggle != slave_state[slave].comm_toggle) { mlx4_warn(dev, "Incorrect toggle %d from slave %d. *** MASTER STATE COMPROMISED ***\n", toggle, slave); goto reset_slave; } if (cmd == MLX4_COMM_CMD_RESET) { mlx4_warn(dev, "Received reset from slave:%d\n", slave); slave_state[slave].active = false; slave_state[slave].old_vlan_api = false; slave_state[slave].vst_qinq_supported = false; mlx4_master_deactivate_admin_state(priv, slave); for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) { slave_state[slave].event_eq[i].eqn = -1; slave_state[slave].event_eq[i].token = 0; } /*check if we are in the middle of FLR process, if so return "retry" status to the slave*/ if (MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) goto inform_slave_state; mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN, slave); /* write the version in the event field */ reply |= mlx4_comm_get_version(); goto reset_slave; } /*command from slave in the middle of FLR*/ if (cmd != MLX4_COMM_CMD_RESET && MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) { mlx4_warn(dev, "slave:%d is Trying to run cmd(0x%x) in the middle of FLR\n", slave, cmd); return; } switch (cmd) { case MLX4_COMM_CMD_VHCR0: if (slave_state[slave].last_cmd != MLX4_COMM_CMD_RESET) goto reset_slave; slave_state[slave].vhcr_dma = ((u64) param) << 48; priv->mfunc.master.slave_state[slave].cookie = 0; break; case MLX4_COMM_CMD_VHCR1: if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR0) goto reset_slave; slave_state[slave].vhcr_dma |= ((u64) param) << 32; break; case MLX4_COMM_CMD_VHCR2: if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR1) goto reset_slave; slave_state[slave].vhcr_dma |= ((u64) param) << 16; break; case MLX4_COMM_CMD_VHCR_EN: if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR2) goto reset_slave; slave_state[slave].vhcr_dma |= param; if (mlx4_master_activate_admin_state(priv, slave)) goto reset_slave; slave_state[slave].active = true; mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_INIT, slave); break; case MLX4_COMM_CMD_VHCR_POST: if ((slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_EN) && (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_POST)) { mlx4_warn(dev, "slave:%d is out of sync, cmd=0x%x, last command=0x%x, reset is needed\n", slave, cmd, slave_state[slave].last_cmd); goto reset_slave; } mutex_lock(&priv->cmd.slave_cmd_mutex); if (mlx4_master_process_vhcr(dev, slave, NULL)) { mlx4_err(dev, "Failed processing vhcr for slave:%d, resetting slave\n", slave); mutex_unlock(&priv->cmd.slave_cmd_mutex); goto reset_slave; } mutex_unlock(&priv->cmd.slave_cmd_mutex); break; default: mlx4_warn(dev, "Bad comm cmd:%d from slave:%d\n", cmd, slave); goto reset_slave; } spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); if (!slave_state[slave].is_slave_going_down) slave_state[slave].last_cmd = cmd; else is_going_down = 1; spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); if (is_going_down) { mlx4_warn(dev, "Slave is going down aborting command(%d) executing from slave:%d\n", cmd, slave); return; } __raw_writel((__force u32) cpu_to_be32(reply), &priv->mfunc.comm[slave].slave_read); mmiowb(); return; reset_slave: /* cleanup any slave resources */ if (dev->persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_delete_all_resources_for_slave(dev, slave); if (cmd != MLX4_COMM_CMD_RESET) { mlx4_warn(dev, "Turn on internal error to force reset, slave=%d, cmd=0x%x\n", slave, cmd); /* Turn on internal error letting slave reset itself immeditaly, * otherwise it might take till timeout on command is passed */ reply |= ((u32)COMM_CHAN_EVENT_INTERNAL_ERR); } spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); if (!slave_state[slave].is_slave_going_down) slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET; spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); /*with slave in the middle of flr, no need to clean resources again.*/ inform_slave_state: memset(&slave_state[slave].event_eq, 0, sizeof(struct mlx4_slave_event_eq_info)); __raw_writel((__force u32) cpu_to_be32(reply), &priv->mfunc.comm[slave].slave_read); wmb(); } /* master command processing */ void mlx4_master_comm_channel(struct work_struct *work) { struct mlx4_mfunc_master_ctx *master = container_of(work, struct mlx4_mfunc_master_ctx, comm_work); struct mlx4_mfunc *mfunc = container_of(master, struct mlx4_mfunc, master); struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); struct mlx4_dev *dev = &priv->dev; __be32 *bit_vec; u32 comm_cmd; u32 vec; int i, j, slave; int toggle; int served = 0; int reported = 0; u32 slt; bit_vec = master->comm_arm_bit_vector; for (i = 0; i < COMM_CHANNEL_BIT_ARRAY_SIZE; i++) { vec = be32_to_cpu(bit_vec[i]); for (j = 0; j < 32; j++) { if (!(vec & (1 << j))) continue; ++reported; slave = (i * 32) + j; comm_cmd = swab32(readl( &mfunc->comm[slave].slave_write)); slt = swab32(readl(&mfunc->comm[slave].slave_read)) >> 31; toggle = comm_cmd >> 31; if (toggle != slt) { if (master->slave_state[slave].comm_toggle != slt) { pr_info("slave %d out of sync. read toggle %d, state toggle %d. Resynching.\n", slave, slt, master->slave_state[slave].comm_toggle); master->slave_state[slave].comm_toggle = slt; } mlx4_master_do_cmd(dev, slave, comm_cmd >> 16 & 0xff, comm_cmd & 0xffff, toggle); ++served; } } } if (reported && reported != served) mlx4_warn(dev, "Got command event with bitmask from %d slaves but %d were served\n", reported, served); if (mlx4_ARM_COMM_CHANNEL(dev)) mlx4_warn(dev, "Failed to arm comm channel events\n"); } static int sync_toggles(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); u32 wr_toggle; u32 rd_toggle; unsigned long end; wr_toggle = swab32(readl(&priv->mfunc.comm->slave_write)); if (wr_toggle == 0xffffffff) end = jiffies + msecs_to_jiffies(30000); else end = jiffies + msecs_to_jiffies(5000); while (time_before(jiffies, end)) { rd_toggle = swab32(readl(&priv->mfunc.comm->slave_read)); if (wr_toggle == 0xffffffff || rd_toggle == 0xffffffff) { /* PCI might be offline */ msleep(100); wr_toggle = swab32(readl(&priv->mfunc.comm-> slave_write)); continue; } if (rd_toggle >> 31 == wr_toggle >> 31) { priv->cmd.comm_toggle = rd_toggle >> 31; return 0; } cond_resched(); } /* * we could reach here if for example the previous VM using this * function misbehaved and left the channel with unsynced state. We * should fix this here and give this VM a chance to use a properly * synced channel */ mlx4_warn(dev, "recovering from previously mis-behaved VM\n"); __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_read); __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_write); priv->cmd.comm_toggle = 0; return 0; } int mlx4_multi_func_init(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_slave_state *s_state; int i, j, err, port; if (mlx4_is_master(dev)) priv->mfunc.comm = ioremap(pci_resource_start(dev->persist->pdev, priv->fw.comm_bar) + priv->fw.comm_base, MLX4_COMM_PAGESIZE); else priv->mfunc.comm = ioremap(pci_resource_start(dev->persist->pdev, 2) + MLX4_SLAVE_COMM_BASE, MLX4_COMM_PAGESIZE); if (!priv->mfunc.comm) { mlx4_err(dev, "Couldn't map communication vector\n"); goto err_vhcr; } if (mlx4_is_master(dev)) { struct mlx4_vf_oper_state *vf_oper; struct mlx4_vf_admin_state *vf_admin; priv->mfunc.master.slave_state = kzalloc(dev->num_slaves * sizeof(struct mlx4_slave_state), GFP_KERNEL); if (!priv->mfunc.master.slave_state) goto err_comm; priv->mfunc.master.vf_admin = kzalloc(dev->num_slaves * sizeof(struct mlx4_vf_admin_state), GFP_KERNEL); if (!priv->mfunc.master.vf_admin) goto err_comm_admin; priv->mfunc.master.vf_oper = kzalloc(dev->num_slaves * sizeof(struct mlx4_vf_oper_state), GFP_KERNEL); if (!priv->mfunc.master.vf_oper) goto err_comm_oper; for (i = 0; i < dev->num_slaves; ++i) { vf_admin = &priv->mfunc.master.vf_admin[i]; vf_oper = &priv->mfunc.master.vf_oper[i]; s_state = &priv->mfunc.master.slave_state[i]; s_state->last_cmd = MLX4_COMM_CMD_RESET; s_state->vst_qinq_supported = false; mutex_init(&priv->mfunc.master.gen_eqe_mutex[i]); for (j = 0; j < MLX4_EVENT_TYPES_NUM; ++j) s_state->event_eq[j].eqn = -1; __raw_writel((__force u32) 0, &priv->mfunc.comm[i].slave_write); __raw_writel((__force u32) 0, &priv->mfunc.comm[i].slave_read); mmiowb(); for (port = 1; port <= MLX4_MAX_PORTS; port++) { struct mlx4_vport_state *admin_vport; struct mlx4_vport_state *oper_vport; s_state->vlan_filter[port] = kzalloc(sizeof(struct mlx4_vlan_fltr), GFP_KERNEL); if (!s_state->vlan_filter[port]) { if (--port) kfree(s_state->vlan_filter[port]); goto err_slaves; } admin_vport = &vf_admin->vport[port]; oper_vport = &vf_oper->vport[port].state; INIT_LIST_HEAD(&s_state->mcast_filters[port]); admin_vport->default_vlan = MLX4_VGT; oper_vport->default_vlan = MLX4_VGT; admin_vport->qos_vport = MLX4_VPP_DEFAULT_VPORT; oper_vport->qos_vport = MLX4_VPP_DEFAULT_VPORT; admin_vport->vlan_proto = htons(ETH_P_8021Q); oper_vport->vlan_proto = htons(ETH_P_8021Q); vf_oper->vport[port].vlan_idx = NO_INDX; vf_oper->vport[port].mac_idx = NO_INDX; mlx4_set_random_admin_guid(dev, i, port); } spin_lock_init(&s_state->lock); } if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP) { for (port = 1; port <= dev->caps.num_ports; port++) { if (mlx4_is_eth(dev, port)) { mlx4_set_default_port_qos(dev, port); mlx4_allocate_port_vpps(dev, port); } } } memset(&priv->mfunc.master.cmd_eqe, 0, sizeof(struct mlx4_eqe)); priv->mfunc.master.cmd_eqe.type = MLX4_EVENT_TYPE_CMD; INIT_WORK(&priv->mfunc.master.comm_work, mlx4_master_comm_channel); INIT_WORK(&priv->mfunc.master.slave_event_work, mlx4_gen_slave_eqe); INIT_WORK(&priv->mfunc.master.slave_flr_event_work, mlx4_master_handle_slave_flr); spin_lock_init(&priv->mfunc.master.slave_state_lock); spin_lock_init(&priv->mfunc.master.slave_eq.event_lock); priv->mfunc.master.comm_wq = create_singlethread_workqueue("mlx4_comm"); if (!priv->mfunc.master.comm_wq) goto err_slaves; if (mlx4_init_resource_tracker(dev)) goto err_thread; } else { err = sync_toggles(dev); if (err) { mlx4_err(dev, "Couldn't sync toggles\n"); goto err_comm; } } return 0; err_thread: flush_workqueue(priv->mfunc.master.comm_wq); destroy_workqueue(priv->mfunc.master.comm_wq); err_slaves: while (i--) { for (port = 1; port <= MLX4_MAX_PORTS; port++) kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); } kfree(priv->mfunc.master.vf_oper); err_comm_oper: kfree(priv->mfunc.master.vf_admin); err_comm_admin: kfree(priv->mfunc.master.slave_state); err_comm: iounmap(priv->mfunc.comm); priv->mfunc.comm = NULL; err_vhcr: dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, priv->mfunc.vhcr, priv->mfunc.vhcr_dma); priv->mfunc.vhcr = NULL; return -ENOMEM; } int mlx4_cmd_init(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int flags = 0; if (!priv->cmd.initialized) { init_rwsem(&priv->cmd.switch_sem); mutex_init(&priv->cmd.slave_cmd_mutex); sema_init(&priv->cmd.poll_sem, 1); + sema_init(&priv->cmd.event_sem, 0); priv->cmd.use_events = 0; priv->cmd.toggle = 1; priv->cmd.initialized = 1; flags |= MLX4_CMD_CLEANUP_STRUCT; } if (!mlx4_is_slave(dev) && !priv->cmd.hcr) { priv->cmd.hcr = ioremap(pci_resource_start(dev->persist->pdev, 0) + MLX4_HCR_BASE, MLX4_HCR_SIZE); if (!priv->cmd.hcr) { mlx4_err(dev, "Couldn't map command register\n"); goto err; } flags |= MLX4_CMD_CLEANUP_HCR; } if (mlx4_is_mfunc(dev) && !priv->mfunc.vhcr) { priv->mfunc.vhcr = dma_alloc_coherent(&dev->persist->pdev->dev, PAGE_SIZE, &priv->mfunc.vhcr_dma, GFP_KERNEL); if (!priv->mfunc.vhcr) goto err; flags |= MLX4_CMD_CLEANUP_VHCR; } if (!priv->cmd.pool) { priv->cmd.pool = pci_pool_create("mlx4_cmd", dev->persist->pdev, MLX4_MAILBOX_SIZE, MLX4_MAILBOX_SIZE, 0); if (!priv->cmd.pool) goto err; flags |= MLX4_CMD_CLEANUP_POOL; } return 0; err: mlx4_cmd_cleanup(dev, flags); return -ENOMEM; } void mlx4_report_internal_err_comm_event(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int slave; u32 slave_read; /* If the comm channel has not yet been initialized, * skip reporting the internal error event to all * the communication channels. */ if (!priv->mfunc.comm) return; /* Report an internal error event to all * communication channels. */ for (slave = 0; slave < dev->num_slaves; slave++) { slave_read = swab32(readl(&priv->mfunc.comm[slave].slave_read)); slave_read |= (u32)COMM_CHAN_EVENT_INTERNAL_ERR; __raw_writel((__force u32)cpu_to_be32(slave_read), &priv->mfunc.comm[slave].slave_read); /* Make sure that our comm channel write doesn't * get mixed in with writes from another CPU. */ mmiowb(); } } void mlx4_multi_func_cleanup(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i, port; if (mlx4_is_master(dev)) { flush_workqueue(priv->mfunc.master.comm_wq); destroy_workqueue(priv->mfunc.master.comm_wq); for (i = 0; i < dev->num_slaves; i++) { for (port = 1; port <= MLX4_MAX_PORTS; port++) kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); } kfree(priv->mfunc.master.slave_state); kfree(priv->mfunc.master.vf_admin); kfree(priv->mfunc.master.vf_oper); dev->num_slaves = 0; } iounmap(priv->mfunc.comm); priv->mfunc.comm = NULL; } void mlx4_cmd_cleanup(struct mlx4_dev *dev, int cleanup_mask) { struct mlx4_priv *priv = mlx4_priv(dev); if (priv->cmd.pool && (cleanup_mask & MLX4_CMD_CLEANUP_POOL)) { pci_pool_destroy(priv->cmd.pool); priv->cmd.pool = NULL; } if (!mlx4_is_slave(dev) && priv->cmd.hcr && (cleanup_mask & MLX4_CMD_CLEANUP_HCR)) { iounmap(priv->cmd.hcr); priv->cmd.hcr = NULL; } if (mlx4_is_mfunc(dev) && priv->mfunc.vhcr && (cleanup_mask & MLX4_CMD_CLEANUP_VHCR)) { dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, priv->mfunc.vhcr, priv->mfunc.vhcr_dma); priv->mfunc.vhcr = NULL; } if (priv->cmd.initialized && (cleanup_mask & MLX4_CMD_CLEANUP_STRUCT)) priv->cmd.initialized = 0; } /* * Switch to using events to issue FW commands (can only be called * after event queue for command events has been initialized). */ int mlx4_cmd_use_events(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; - int err = 0; + if (priv->cmd.use_events != 0) + return 0; + priv->cmd.context = kmalloc(priv->cmd.max_cmds * sizeof (struct mlx4_cmd_context), GFP_KERNEL); if (!priv->cmd.context) return -ENOMEM; down_write(&priv->cmd.switch_sem); for (i = 0; i < priv->cmd.max_cmds; ++i) { priv->cmd.context[i].token = i; priv->cmd.context[i].next = i + 1; /* To support fatal error flow, initialize all * cmd contexts to allow simulating completions * with complete() at any time. */ init_completion(&priv->cmd.context[i].done); } priv->cmd.context[priv->cmd.max_cmds - 1].next = -1; priv->cmd.free_head = 0; - sema_init(&priv->cmd.event_sem, priv->cmd.max_cmds); + for (i = 0; i != priv->cmd.max_cmds; i++) + up(&priv->cmd.event_sem); for (priv->cmd.token_mask = 1; priv->cmd.token_mask < priv->cmd.max_cmds; priv->cmd.token_mask <<= 1) ; /* nothing */ --priv->cmd.token_mask; down(&priv->cmd.poll_sem); priv->cmd.use_events = 1; up_write(&priv->cmd.switch_sem); - return err; + return 0; } /* * Switch back to polling (used when shutting down the device) */ void mlx4_cmd_use_polling(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; + if (priv->cmd.use_events == 0) + return; + down_write(&priv->cmd.switch_sem); priv->cmd.use_events = 0; for (i = 0; i < priv->cmd.max_cmds; ++i) down(&priv->cmd.event_sem); kfree(priv->cmd.context); + priv->cmd.context = NULL; up(&priv->cmd.poll_sem); up_write(&priv->cmd.switch_sem); } struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) { struct mlx4_cmd_mailbox *mailbox; mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL); if (!mailbox) return ERR_PTR(-ENOMEM); mailbox->buf = pci_pool_alloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL, &mailbox->dma); if (!mailbox->buf) { kfree(mailbox); return ERR_PTR(-ENOMEM); } memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); return mailbox; } EXPORT_SYMBOL_GPL(mlx4_alloc_cmd_mailbox); void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox) { if (!mailbox) return; pci_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox); u32 mlx4_comm_get_version(void) { return ((u32) CMD_CHAN_IF_REV << 8) | (u32) CMD_CHAN_VER; } static int mlx4_get_slave_indx(struct mlx4_dev *dev, int vf) { if ((vf < 0) || (vf >= dev->persist->num_vfs)) { mlx4_err(dev, "Bad vf number:%d (number of activated vf: %d)\n", vf, dev->persist->num_vfs); return -EINVAL; } return vf+1; } int mlx4_get_vf_indx(struct mlx4_dev *dev, int slave) { if (slave < 1 || slave > dev->persist->num_vfs) { mlx4_err(dev, "Bad slave number:%d (number of activated slaves: %lu)\n", slave, dev->num_slaves); return -EINVAL; } return slave - 1; } void mlx4_cmd_wake_completions(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_context *context; int i; spin_lock(&priv->cmd.context_lock); - if (priv->cmd.context) { + if (priv->cmd.context != NULL) { for (i = 0; i < priv->cmd.max_cmds; ++i) { context = &priv->cmd.context[i]; context->fw_status = CMD_STAT_INTERNAL_ERR; - context->result = + context->result = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); complete(&context->done); } } spin_unlock(&priv->cmd.context_lock); } struct mlx4_active_ports mlx4_get_active_ports(struct mlx4_dev *dev, int slave) { struct mlx4_active_ports actv_ports; int vf; bitmap_zero(actv_ports.ports, MLX4_MAX_PORTS); if (slave == 0) { bitmap_fill(actv_ports.ports, dev->caps.num_ports); return actv_ports; } vf = mlx4_get_vf_indx(dev, slave); if (vf < 0) return actv_ports; bitmap_set(actv_ports.ports, dev->dev_vfs[vf].min_port - 1, min((int)dev->dev_vfs[mlx4_get_vf_indx(dev, slave)].n_ports, dev->caps.num_ports)); return actv_ports; } EXPORT_SYMBOL_GPL(mlx4_get_active_ports); int mlx4_slave_convert_port(struct mlx4_dev *dev, int slave, int port) { unsigned n; struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); unsigned m = bitmap_weight(actv_ports.ports, dev->caps.num_ports); if (port <= 0 || port > m) return -EINVAL; n = find_first_bit(actv_ports.ports, dev->caps.num_ports); if (port <= n) port = n + 1; return port; } EXPORT_SYMBOL_GPL(mlx4_slave_convert_port); int mlx4_phys_to_slave_port(struct mlx4_dev *dev, int slave, int port) { struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); if (test_bit(port - 1, actv_ports.ports)) return port - find_first_bit(actv_ports.ports, dev->caps.num_ports); return -1; } EXPORT_SYMBOL_GPL(mlx4_phys_to_slave_port); struct mlx4_slaves_pport mlx4_phys_to_slaves_pport(struct mlx4_dev *dev, int port) { unsigned i; struct mlx4_slaves_pport slaves_pport; bitmap_zero(slaves_pport.slaves, MLX4_MFUNC_MAX); if (port <= 0 || port > dev->caps.num_ports) return slaves_pport; for (i = 0; i < dev->persist->num_vfs + 1; i++) { struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, i); if (test_bit(port - 1, actv_ports.ports)) set_bit(i, slaves_pport.slaves); } return slaves_pport; } EXPORT_SYMBOL_GPL(mlx4_phys_to_slaves_pport); struct mlx4_slaves_pport mlx4_phys_to_slaves_pport_actv( struct mlx4_dev *dev, const struct mlx4_active_ports *crit_ports) { unsigned i; struct mlx4_slaves_pport slaves_pport; bitmap_zero(slaves_pport.slaves, MLX4_MFUNC_MAX); for (i = 0; i < dev->persist->num_vfs + 1; i++) { struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, i); if (bitmap_equal(crit_ports->ports, actv_ports.ports, dev->caps.num_ports)) set_bit(i, slaves_pport.slaves); } return slaves_pport; } EXPORT_SYMBOL_GPL(mlx4_phys_to_slaves_pport_actv); static int mlx4_slaves_closest_port(struct mlx4_dev *dev, int slave, int port) { struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); int min_port = find_first_bit(actv_ports.ports, dev->caps.num_ports) + 1; int max_port = min_port + bitmap_weight(actv_ports.ports, dev->caps.num_ports); if (port < min_port) port = min_port; else if (port >= max_port) port = max_port - 1; return port; } static int mlx4_set_vport_qos(struct mlx4_priv *priv, int slave, int port, int max_tx_rate) { int i; int err; struct mlx4_qos_manager *port_qos; struct mlx4_dev *dev = &priv->dev; struct mlx4_vport_qos_param vpp_qos[MLX4_NUM_UP]; port_qos = &priv->mfunc.master.qos_ctl[port]; memset(vpp_qos, 0, sizeof(struct mlx4_vport_qos_param) * MLX4_NUM_UP); if (slave > port_qos->num_of_qos_vfs) { mlx4_info(dev, "No availible VPP resources for this VF\n"); return -EINVAL; } /* Query for default QoS values from Vport 0 is needed */ err = mlx4_SET_VPORT_QOS_get(dev, port, 0, vpp_qos); if (err) { mlx4_info(dev, "Failed to query Vport 0 QoS values\n"); return err; } for (i = 0; i < MLX4_NUM_UP; i++) { if (test_bit(i, port_qos->priority_bm) && max_tx_rate) { vpp_qos[i].max_avg_bw = max_tx_rate; vpp_qos[i].enable = 1; } else { /* if user supplied tx_rate == 0, meaning no rate limit * configuration is required. so we are leaving the * value of max_avg_bw as queried from Vport 0. */ vpp_qos[i].enable = 0; } } err = mlx4_SET_VPORT_QOS_set(dev, port, slave, vpp_qos); if (err) { mlx4_info(dev, "Failed to set Vport %d QoS values\n", slave); return err; } return 0; } static bool mlx4_is_vf_vst_and_prio_qos(struct mlx4_dev *dev, int port, struct mlx4_vport_state *vf_admin) { struct mlx4_qos_manager *info; struct mlx4_priv *priv = mlx4_priv(dev); if (!mlx4_is_master(dev) || !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP)) return false; info = &priv->mfunc.master.qos_ctl[port]; if (vf_admin->default_vlan != MLX4_VGT && test_bit(vf_admin->default_qos, info->priority_bm)) return true; return false; } static bool mlx4_valid_vf_state_change(struct mlx4_dev *dev, int port, struct mlx4_vport_state *vf_admin, int vlan, int qos) { struct mlx4_vport_state dummy_admin = {0}; if (!mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin) || !vf_admin->tx_rate) return true; dummy_admin.default_qos = qos; dummy_admin.default_vlan = vlan; /* VF wants to move to other VST state which is valid with current * rate limit. Either differnt default vlan in VST or other * supported QoS priority. Otherwise we don't allow this change when * the TX rate is still configured. */ if (mlx4_is_vf_vst_and_prio_qos(dev, port, &dummy_admin)) return true; mlx4_info(dev, "Cannot change VF state to %s while rate is set\n", (vlan == MLX4_VGT) ? "VGT" : "VST"); if (vlan != MLX4_VGT) mlx4_info(dev, "VST priority %d not supported for QoS\n", qos); mlx4_info(dev, "Please set rate to 0 prior to this VF state change\n"); return false; } int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u64 mac) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_vport_state *s_info; int slave; if (!mlx4_is_master(dev)) return -EPROTONOSUPPORT; slave = mlx4_get_slave_indx(dev, vf); if (slave < 0) return -EINVAL; port = mlx4_slaves_closest_port(dev, slave, port); s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; s_info->mac = mac; mlx4_info(dev, "default mac on vf %d port %d to %llX will take effect only after vf restart\n", vf, port, (unsigned long long)s_info->mac); return 0; } EXPORT_SYMBOL_GPL(mlx4_set_vf_mac); int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos, __be16 proto) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_vport_state *vf_admin; struct mlx4_slave_state *slave_state; struct mlx4_vport_oper_state *vf_oper; int slave; if ((!mlx4_is_master(dev)) || !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VLAN_CONTROL)) return -EPROTONOSUPPORT; if ((vlan > 4095) || (qos > 7)) return -EINVAL; if (proto == htons(ETH_P_8021AD) && !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP)) return -EPROTONOSUPPORT; if (proto != htons(ETH_P_8021Q) && proto != htons(ETH_P_8021AD)) return -EINVAL; if ((proto == htons(ETH_P_8021AD)) && ((vlan == 0) || (vlan == MLX4_VGT))) return -EINVAL; slave = mlx4_get_slave_indx(dev, vf); if (slave < 0) return -EINVAL; slave_state = &priv->mfunc.master.slave_state[slave]; if ((proto == htons(ETH_P_8021AD)) && (slave_state->active) && (!slave_state->vst_qinq_supported)) { mlx4_err(dev, "vf %d does not support VST QinQ mode\n", vf); return -EPROTONOSUPPORT; } port = mlx4_slaves_closest_port(dev, slave, port); vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; if (!mlx4_valid_vf_state_change(dev, port, vf_admin, vlan, qos)) return -EPERM; if ((0 == vlan) && (0 == qos)) vf_admin->default_vlan = MLX4_VGT; else vf_admin->default_vlan = vlan; vf_admin->default_qos = qos; vf_admin->vlan_proto = proto; /* If rate was configured prior to VST, we saved the configured rate * in vf_admin->rate and now, if priority supported we enforce the QoS */ if (mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin) && vf_admin->tx_rate) vf_admin->qos_vport = slave; /* Try to activate new vf state without restart, * this option is not supported while moving to VST QinQ mode. */ if ((proto == htons(ETH_P_8021AD) && vf_oper->state.vlan_proto != proto) || mlx4_master_immediate_activate_vlan_qos(priv, slave, port)) mlx4_info(dev, "updating vf %d port %d config will take effect on next VF restart\n", vf, port); return 0; } EXPORT_SYMBOL_GPL(mlx4_set_vf_vlan); int mlx4_set_vf_rate(struct mlx4_dev *dev, int port, int vf, int min_tx_rate, int max_tx_rate) { int err; int slave; struct mlx4_vport_state *vf_admin; struct mlx4_priv *priv = mlx4_priv(dev); if (!mlx4_is_master(dev) || !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP)) return -EPROTONOSUPPORT; if (min_tx_rate) { mlx4_info(dev, "Minimum BW share not supported\n"); return -EPROTONOSUPPORT; } slave = mlx4_get_slave_indx(dev, vf); if (slave < 0) return -EINVAL; port = mlx4_slaves_closest_port(dev, slave, port); vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; err = mlx4_set_vport_qos(priv, slave, port, max_tx_rate); if (err) { mlx4_info(dev, "vf %d failed to set rate %d\n", vf, max_tx_rate); return err; } vf_admin->tx_rate = max_tx_rate; /* if VF is not in supported mode (VST with supported prio), * we do not change vport configuration for its QPs, but save * the rate, so it will be enforced when it moves to supported * mode next time. */ if (!mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin)) { mlx4_info(dev, "rate set for VF %d when not in valid state\n", vf); if (vf_admin->default_vlan != MLX4_VGT) mlx4_info(dev, "VST priority not supported by QoS\n"); else mlx4_info(dev, "VF in VGT mode (needed VST)\n"); mlx4_info(dev, "rate %d take affect when VF moves to valid state\n", max_tx_rate); return 0; } /* If user sets rate 0 assigning default vport for its QPs */ vf_admin->qos_vport = max_tx_rate ? slave : MLX4_VPP_DEFAULT_VPORT; if (priv->mfunc.master.slave_state[slave].active && dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) mlx4_master_immediate_activate_vlan_qos(priv, slave, port); return 0; } EXPORT_SYMBOL_GPL(mlx4_set_vf_rate); /* mlx4_get_slave_default_vlan - * return true if VST ( default vlan) * if VST, will return vlan & qos (if not NULL) */ bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, u16 *vlan, u8 *qos) { struct mlx4_vport_oper_state *vp_oper; struct mlx4_priv *priv; priv = mlx4_priv(dev); port = mlx4_slaves_closest_port(dev, slave, port); vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; if (MLX4_VGT != vp_oper->state.default_vlan) { if (vlan) *vlan = vp_oper->state.default_vlan; if (qos) *qos = vp_oper->state.default_qos; return true; } return false; } EXPORT_SYMBOL_GPL(mlx4_get_slave_default_vlan); int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_vport_state *s_info; int slave; if ((!mlx4_is_master(dev)) || !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FSM)) return -EPROTONOSUPPORT; slave = mlx4_get_slave_indx(dev, vf); if (slave < 0) return -EINVAL; port = mlx4_slaves_closest_port(dev, slave, port); s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; s_info->spoofchk = setting; return 0; } EXPORT_SYMBOL_GPL(mlx4_set_vf_spoofchk); int mlx4_get_counter_stats(struct mlx4_dev *dev, int counter_index, struct mlx4_counter *counter_stats, int reset) { struct mlx4_cmd_mailbox *mailbox = NULL; struct mlx4_counter *tmp_counter; int err; u32 if_stat_in_mod; if (!counter_stats) return -EINVAL; if (counter_index == MLX4_SINK_COUNTER_INDEX(dev)) return 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, sizeof(struct mlx4_counter)); if_stat_in_mod = counter_index; if (reset) if_stat_in_mod |= MLX4_QUERY_IF_STAT_RESET; err = mlx4_cmd_box(dev, 0, mailbox->dma, if_stat_in_mod, 0, MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); if (err) { mlx4_dbg(dev, "%s: failed to read statistics for counter index %d\n", __func__, counter_index); goto if_stat_out; } tmp_counter = (struct mlx4_counter *)mailbox->buf; counter_stats->counter_mode = tmp_counter->counter_mode; if (counter_stats->counter_mode == 0) { counter_stats->rx_frames = cpu_to_be64(be64_to_cpu(counter_stats->rx_frames) + be64_to_cpu(tmp_counter->rx_frames)); counter_stats->tx_frames = cpu_to_be64(be64_to_cpu(counter_stats->tx_frames) + be64_to_cpu(tmp_counter->tx_frames)); counter_stats->rx_bytes = cpu_to_be64(be64_to_cpu(counter_stats->rx_bytes) + be64_to_cpu(tmp_counter->rx_bytes)); counter_stats->tx_bytes = cpu_to_be64(be64_to_cpu(counter_stats->tx_bytes) + be64_to_cpu(tmp_counter->tx_bytes)); } if_stat_out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } EXPORT_SYMBOL_GPL(mlx4_get_counter_stats); int mlx4_vf_smi_enabled(struct mlx4_dev *dev, int slave, int port) { struct mlx4_priv *priv = mlx4_priv(dev); if (slave < 1 || slave >= dev->num_slaves || port < 1 || port > MLX4_MAX_PORTS) return 0; return priv->mfunc.master.vf_oper[slave].smi_enabled[port] == MLX4_VF_SMI_ENABLED; } EXPORT_SYMBOL_GPL(mlx4_vf_smi_enabled); int mlx4_vf_get_enable_smi_admin(struct mlx4_dev *dev, int slave, int port) { struct mlx4_priv *priv = mlx4_priv(dev); if (slave == mlx4_master_func_num(dev)) return 1; if (slave < 1 || slave >= dev->num_slaves || port < 1 || port > MLX4_MAX_PORTS) return 0; return priv->mfunc.master.vf_admin[slave].enable_smi[port] == MLX4_VF_SMI_ENABLED; } EXPORT_SYMBOL_GPL(mlx4_vf_get_enable_smi_admin); int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, int enabled) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_active_ports actv_ports = mlx4_get_active_ports( &priv->dev, slave); int min_port = find_first_bit(actv_ports.ports, priv->dev.caps.num_ports) + 1; int max_port = min_port - 1 + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); if (slave == mlx4_master_func_num(dev)) return 0; if (slave < 1 || slave >= dev->num_slaves || port < 1 || port > MLX4_MAX_PORTS || enabled < 0 || enabled > 1) return -EINVAL; if (min_port == max_port && dev->caps.num_ports > 1) { mlx4_info(dev, "SMI access disallowed for single ported VFs\n"); return -EPROTONOSUPPORT; } priv->mfunc.master.vf_admin[slave].enable_smi[port] = enabled; return 0; } EXPORT_SYMBOL_GPL(mlx4_vf_set_enable_smi_admin); Index: projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_main.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_main.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/mlx4/mlx4_core/mlx4_main.c (revision 345026) @@ -1,4239 +1,4236 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define LINUXKPI_PARAM_PREFIX mlx4_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mlx4.h" #include "fw.h" #include "icm.h" #include MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); MODULE_LICENSE("Dual BSD/GPL"); struct workqueue_struct *mlx4_wq; #ifdef CONFIG_MLX4_DEBUG int mlx4_debug_level = 0; module_param_named(debug_level, mlx4_debug_level, int, 0644); MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); #endif /* CONFIG_MLX4_DEBUG */ #ifdef CONFIG_PCI_MSI static int msi_x = 1; module_param(msi_x, int, 0444); MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); #else /* CONFIG_PCI_MSI */ #define msi_x (0) #endif /* CONFIG_PCI_MSI */ static uint8_t num_vfs[3] = {0, 0, 0}; static int num_vfs_argc; module_param_array(num_vfs, byte , &num_vfs_argc, 0444); MODULE_PARM_DESC(num_vfs, "enable #num_vfs functions if num_vfs > 0\n" "num_vfs=port1,port2,port1+2"); static uint8_t probe_vf[3] = {0, 0, 0}; static int probe_vfs_argc; module_param_array(probe_vf, byte, &probe_vfs_argc, 0444); MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (num_vfs > 0)\n" "probe_vf=port1,port2,port1+2"); int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; module_param_named(log_num_mgm_entry_size, mlx4_log_num_mgm_entry_size, int, 0444); MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num" " of qp per mcg, for example:" " 10 gives 248.range: 7 <=" " log_num_mgm_entry_size <= 12." " To activate device managed" " flow steering when available, set to -1"); static bool enable_64b_cqe_eqe = true; module_param(enable_64b_cqe_eqe, bool, 0444); MODULE_PARM_DESC(enable_64b_cqe_eqe, "Enable 64 byte CQEs/EQEs when the FW supports this (default: True)"); static bool enable_4k_uar; module_param(enable_4k_uar, bool, 0444); MODULE_PARM_DESC(enable_4k_uar, "Enable using 4K UAR. Should not be enabled if have VFs which do not support 4K UARs (default: false)"); #define PF_CONTEXT_BEHAVIOUR_MASK (MLX4_FUNC_CAP_64B_EQE_CQE | \ MLX4_FUNC_CAP_EQE_CQE_STRIDE | \ MLX4_FUNC_CAP_DMFS_A0_STATIC) #define RESET_PERSIST_MASK_FLAGS (MLX4_FLAG_SRIOV) static char mlx4_description[] = "Mellanox driver" " (" DRV_VERSION ")"; static char mlx4_version[] = DRV_NAME ": Mellanox ConnectX core driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; static struct mlx4_profile default_profile = { .num_qp = 1 << 18, .num_srq = 1 << 16, .rdmarc_per_qp = 1 << 4, .num_cq = 1 << 16, .num_mcg = 1 << 13, .num_mpt = 1 << 19, .num_mtt = 1 << 20, /* It is really num mtt segements */ }; static struct mlx4_profile low_mem_profile = { .num_qp = 1 << 17, .num_srq = 1 << 6, .rdmarc_per_qp = 1 << 4, .num_cq = 1 << 8, .num_mcg = 1 << 8, .num_mpt = 1 << 9, .num_mtt = 1 << 7, }; static int log_num_mac = 7; module_param_named(log_num_mac, log_num_mac, int, 0444); MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); static int log_num_vlan; module_param_named(log_num_vlan, log_num_vlan, int, 0444); MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)"); /* Log2 max number of VLANs per ETH port (0-7) */ #define MLX4_LOG_NUM_VLANS 7 #define MLX4_MIN_LOG_NUM_VLANS 0 #define MLX4_MIN_LOG_NUM_MAC 1 static bool use_prio; module_param_named(use_prio, use_prio, bool, 0444); MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)"); int log_mtts_per_seg = ilog2(MLX4_MTT_ENTRY_PER_SEG); module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-7)"); static int port_type_array[2] = {MLX4_PORT_TYPE_NONE, MLX4_PORT_TYPE_NONE}; struct mlx4_port_config { struct list_head list; enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; struct pci_dev *pdev; }; static atomic_t pf_loading = ATOMIC_INIT(0); static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { /* The reserved_uars is calculated by system page size unit. * Therefore, adjustment is added when the uar page size is less * than the system page size */ dev->caps.reserved_uars = max_t(int, mlx4_get_num_reserved_uar(dev), dev_cap->reserved_uars / (1 << (PAGE_SHIFT - dev->uar_page_shift))); } int mlx4_check_port_params(struct mlx4_dev *dev, enum mlx4_port_type *port_type) { int i; if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) { for (i = 0; i < dev->caps.num_ports - 1; i++) { if (port_type[i] != port_type[i + 1]) { mlx4_err(dev, "Only same port types supported on this HCA, aborting\n"); return -EINVAL; } } } for (i = 0; i < dev->caps.num_ports; i++) { if (!(port_type[i] & dev->caps.supported_type[i+1])) { mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n", i + 1); return -EINVAL; } } return 0; } static void mlx4_set_port_mask(struct mlx4_dev *dev) { int i; for (i = 1; i <= dev->caps.num_ports; ++i) dev->caps.port_mask[i] = dev->caps.port_type[i]; } enum { MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0, }; static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { int err = 0; struct mlx4_func func; if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { err = mlx4_QUERY_FUNC(dev, &func, 0); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); return err; } dev_cap->max_eqs = func.max_eq; dev_cap->reserved_eqs = func.rsvd_eqs; dev_cap->reserved_uars = func.rsvd_uars; err |= MLX4_QUERY_FUNC_NUM_SYS_EQS; } return err; } static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev) { struct mlx4_caps *dev_cap = &dev->caps; /* FW not supporting or cancelled by user */ if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) || !(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE)) return; /* Must have 64B CQE_EQE enabled by FW to use bigger stride * When FW has NCSI it may decide not to report 64B CQE/EQEs */ if (!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_EQE) || !(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_CQE)) { dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; return; } if (cache_line_size() == 128 || cache_line_size() == 256) { mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n"); /* Changing the real data inside CQE size to 32B */ dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; if (mlx4_is_master(dev)) dev_cap->function_caps |= MLX4_FUNC_CAP_EQE_CQE_STRIDE; } else { if (cache_line_size() != 32 && cache_line_size() != 64) mlx4_dbg(dev, "Disabling CQE stride, cacheLine size unsupported\n"); dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; } } static int _mlx4_dev_port(struct mlx4_dev *dev, int port, struct mlx4_port_cap *port_cap) { dev->caps.vl_cap[port] = port_cap->max_vl; dev->caps.ib_mtu_cap[port] = port_cap->ib_mtu; dev->phys_caps.gid_phys_table_len[port] = port_cap->max_gids; dev->phys_caps.pkey_phys_table_len[port] = port_cap->max_pkeys; /* set gid and pkey table operating lengths by default * to non-sriov values */ dev->caps.gid_table_len[port] = port_cap->max_gids; dev->caps.pkey_table_len[port] = port_cap->max_pkeys; dev->caps.port_width_cap[port] = port_cap->max_port_width; dev->caps.eth_mtu_cap[port] = port_cap->eth_mtu; dev->caps.max_tc_eth = port_cap->max_tc_eth; dev->caps.def_mac[port] = port_cap->def_mac; dev->caps.supported_type[port] = port_cap->supported_port_types; dev->caps.suggested_type[port] = port_cap->suggested_type; dev->caps.default_sense[port] = port_cap->default_sense; dev->caps.trans_type[port] = port_cap->trans_type; dev->caps.vendor_oui[port] = port_cap->vendor_oui; dev->caps.wavelength[port] = port_cap->wavelength; dev->caps.trans_code[port] = port_cap->trans_code; return 0; } static int mlx4_dev_port(struct mlx4_dev *dev, int port, struct mlx4_port_cap *port_cap) { int err = 0; err = mlx4_QUERY_PORT(dev, port, port_cap); if (err) mlx4_err(dev, "QUERY_PORT command failed.\n"); return err; } static inline void mlx4_enable_ignore_fcs(struct mlx4_dev *dev) { if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS)) return; if (mlx4_is_mfunc(dev)) { mlx4_dbg(dev, "SRIOV mode - Disabling Ignore FCS"); dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS; return; } if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) { mlx4_dbg(dev, "Keep FCS is not supported - Disabling Ignore FCS"); dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS; return; } } #define MLX4_A0_STEERING_TABLE_SIZE 256 static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { int err; int i; err = mlx4_QUERY_DEV_CAP(dev, dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); return err; } mlx4_dev_cap_dump(dev, dev_cap); if (dev_cap->min_page_sz > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n", dev_cap->min_page_sz, (long)PAGE_SIZE); return -ENODEV; } if (dev_cap->num_ports > MLX4_MAX_PORTS) { mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n", dev_cap->num_ports, MLX4_MAX_PORTS); return -ENODEV; } if (dev_cap->uar_size > pci_resource_len(dev->persist->pdev, 2)) { mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n", dev_cap->uar_size, (unsigned long long) pci_resource_len(dev->persist->pdev, 2)); return -ENODEV; } dev->caps.num_ports = dev_cap->num_ports; dev->caps.num_sys_eqs = dev_cap->num_sys_eqs; dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ? dev->caps.num_sys_eqs : MLX4_MAX_EQ_NUM; for (i = 1; i <= dev->caps.num_ports; ++i) { err = _mlx4_dev_port(dev, i, dev_cap->port_cap + i); if (err) { mlx4_err(dev, "QUERY_PORT command failed, aborting\n"); return err; } } dev->caps.uar_page_size = PAGE_SIZE; dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; dev->caps.bf_reg_size = dev_cap->bf_reg_size; dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page; dev->caps.max_sq_sg = dev_cap->max_sq_sg; dev->caps.max_rq_sg = dev_cap->max_rq_sg; dev->caps.max_wqes = dev_cap->max_qp_sz; dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp; dev->caps.max_srq_wqes = dev_cap->max_srq_sz; dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1; dev->caps.reserved_srqs = dev_cap->reserved_srqs; dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz; dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz; /* * Subtract 1 from the limit because we need to allocate a * spare CQE so the HCA HW can tell the difference between an * empty CQ and a full CQ. */ dev->caps.max_cqes = dev_cap->max_cq_sz - 1; dev->caps.reserved_cqs = dev_cap->reserved_cqs; dev->caps.reserved_eqs = dev_cap->reserved_eqs; dev->caps.reserved_mtts = dev_cap->reserved_mtts; dev->caps.reserved_mrws = dev_cap->reserved_mrws; dev->caps.reserved_pds = dev_cap->reserved_pds; dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? dev_cap->reserved_xrcds : 0; dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? dev_cap->max_xrcds : 0; dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz; dev->caps.max_msg_sz = dev_cap->max_msg_sz; dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1); dev->caps.flags = dev_cap->flags; dev->caps.flags2 = dev_cap->flags2; dev->caps.bmme_flags = dev_cap->bmme_flags; dev->caps.reserved_lkey = dev_cap->reserved_lkey; dev->caps.stat_rate_support = dev_cap->stat_rate_support; dev->caps.max_gso_sz = dev_cap->max_gso_sz; dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz; /* Save uar page shift */ if (!mlx4_is_slave(dev)) { /* Virtual PCI function needs to determine UAR page size from * firmware. Only master PCI function can set the uar page size */ if (enable_4k_uar) dev->uar_page_shift = DEFAULT_UAR_PAGE_SHIFT; else dev->uar_page_shift = PAGE_SHIFT; mlx4_set_num_reserved_uars(dev, dev_cap); } if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN) { struct mlx4_init_hca_param hca_param; memset(&hca_param, 0, sizeof(hca_param)); err = mlx4_QUERY_HCA(dev, &hca_param); /* Turn off PHV_EN flag in case phv_check_en is set. * phv_check_en is a HW check that parse the packet and verify * phv bit was reported correctly in the wqe. To allow QinQ * PHV_EN flag should be set and phv_check_en must be cleared * otherwise QinQ packets will be drop by the HW. */ if (err || hca_param.phv_check_en) dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_PHV_EN; } /* Sense port always allowed on supported devices for ConnectX-1 and -2 */ if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT) dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; /* Don't do sense port on multifunction devices (for now at least) */ if (mlx4_is_mfunc(dev)) dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; if (mlx4_low_memory_profile()) { dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC; dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS; } else { dev->caps.log_num_macs = log_num_mac; dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; } for (i = 1; i <= dev->caps.num_ports; ++i) { dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE; if (dev->caps.supported_type[i]) { /* if only ETH is supported - assign ETH */ if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH) dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH; /* if only IB is supported, assign IB */ else if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_IB) dev->caps.port_type[i] = MLX4_PORT_TYPE_IB; else { /* if IB and ETH are supported, we set the port * type according to user selection of port type; * if user selected none, take the FW hint */ if (port_type_array[i - 1] == MLX4_PORT_TYPE_NONE) dev->caps.port_type[i] = dev->caps.suggested_type[i] ? MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB; else dev->caps.port_type[i] = port_type_array[i - 1]; } } /* * Link sensing is allowed on the port if 3 conditions are true: * 1. Both protocols are supported on the port. * 2. Different types are supported on the port * 3. FW declared that it supports link sensing */ mlx4_priv(dev)->sense.sense_allowed[i] = ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) && (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)); /* * If "default_sense" bit is set, we move the port to "AUTO" mode * and perform sense_port FW command to try and set the correct * port type from beginning */ if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) { enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE; dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO; mlx4_SENSE_PORT(dev, i, &sensed_port); if (sensed_port != MLX4_PORT_TYPE_NONE) dev->caps.port_type[i] = sensed_port; } else { dev->caps.possible_type[i] = dev->caps.port_type[i]; } if (dev->caps.log_num_macs > dev_cap->port_cap[i].log_max_macs) { dev->caps.log_num_macs = dev_cap->port_cap[i].log_max_macs; mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n", i, 1 << dev->caps.log_num_macs); } if (dev->caps.log_num_vlans > dev_cap->port_cap[i].log_max_vlans) { dev->caps.log_num_vlans = dev_cap->port_cap[i].log_max_vlans; mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n", i, 1 << dev->caps.log_num_vlans); } } if (mlx4_is_master(dev) && (dev->caps.num_ports == 2) && (port_type_array[0] == MLX4_PORT_TYPE_IB) && (port_type_array[1] == MLX4_PORT_TYPE_ETH)) { mlx4_warn(dev, "Granular QoS per VF not supported with IB/Eth configuration\n"); dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_QOS_VPP; } dev->caps.max_counters = dev_cap->max_counters; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] = (1 << dev->caps.log_num_macs) * (1 << dev->caps.log_num_vlans) * dev->caps.num_ports; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH; if (dev_cap->dmfs_high_rate_qpn_base > 0 && dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) dev->caps.dmfs_high_rate_qpn_base = dev_cap->dmfs_high_rate_qpn_base; else dev->caps.dmfs_high_rate_qpn_base = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; if (dev_cap->dmfs_high_rate_qpn_range > 0 && dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) { dev->caps.dmfs_high_rate_qpn_range = dev_cap->dmfs_high_rate_qpn_range; dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DEFAULT; dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_FS_A0; } else { dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_NOT_SUPPORTED; dev->caps.dmfs_high_rate_qpn_base = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE; } dev->caps.rl_caps = dev_cap->rl_caps; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] = dev->caps.dmfs_high_rate_qpn_range; dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH]; dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0; if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) { if (dev_cap->flags & (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) { mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n"); dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; } if (dev_cap->flags2 & (MLX4_DEV_CAP_FLAG2_CQE_STRIDE | MLX4_DEV_CAP_FLAG2_EQE_STRIDE)) { mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n"); dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; } } if ((dev->caps.flags & (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) && mlx4_is_master(dev)) dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE; if (!mlx4_is_slave(dev)) { mlx4_enable_cqe_eqe_stride(dev); dev->caps.alloc_res_qp_mask = (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) | MLX4_RESERVE_A0_QP; if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) && dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) { mlx4_warn(dev, "Old device ETS support detected\n"); mlx4_warn(dev, "Consider upgrading device FW.\n"); dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG; } } else { dev->caps.alloc_res_qp_mask = 0; } mlx4_enable_ignore_fcs(dev); return 0; } static int mlx4_get_pcie_dev_link_caps(struct mlx4_dev *dev, enum pci_bus_speed *speed, enum pcie_link_width *width) { u32 lnkcap1, lnkcap2; int err1, err2; #define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ *speed = PCI_SPEED_UNKNOWN; *width = PCIE_LNK_WIDTH_UNKNOWN; err1 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP, &lnkcap1); err2 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP2, &lnkcap2); if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) *speed = PCIE_SPEED_8_0GT; else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) *speed = PCIE_SPEED_5_0GT; else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) *speed = PCIE_SPEED_2_5GT; } if (!err1) { *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; if (!lnkcap2) { /* pre-r3.0 */ if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) *speed = PCIE_SPEED_5_0GT; else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) *speed = PCIE_SPEED_2_5GT; } } if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) { return err1 ? err1 : err2 ? err2 : -EINVAL; } return 0; } static void mlx4_check_pcie_caps(struct mlx4_dev *dev) { enum pcie_link_width width, width_cap; enum pci_bus_speed speed, speed_cap; int err; #define PCIE_SPEED_STR(speed) \ (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ "Unknown") err = mlx4_get_pcie_dev_link_caps(dev, &speed_cap, &width_cap); if (err) { mlx4_warn(dev, "Unable to determine PCIe device BW capabilities\n"); return; } err = pcie_get_minimum_link(dev->persist->pdev, &speed, &width); if (err || speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { mlx4_warn(dev, "Unable to determine PCI device chain minimum BW\n"); return; } if (width != width_cap || speed != speed_cap) mlx4_warn(dev, "PCIe BW is different than device's capability\n"); mlx4_info(dev, "PCIe link speed is %s, device supports %s\n", PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); mlx4_info(dev, "PCIe link width is x%d, device supports x%d\n", width, width_cap); return; } /*The function checks if there are live vf, return the num of them*/ static int mlx4_how_many_lives_vf(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_slave_state *s_state; int i; int ret = 0; for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) { s_state = &priv->mfunc.master.slave_state[i]; if (s_state->active && s_state->last_cmd != MLX4_COMM_CMD_RESET) { mlx4_warn(dev, "%s: slave: %d is still active\n", __func__, i); ret++; } } return ret; } int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey) { u32 qk = MLX4_RESERVED_QKEY_BASE; if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX || qpn < dev->phys_caps.base_proxy_sqpn) return -EINVAL; if (qpn >= dev->phys_caps.base_tunnel_sqpn) /* tunnel qp */ qk += qpn - dev->phys_caps.base_tunnel_sqpn; else qk += qpn - dev->phys_caps.base_proxy_sqpn; *qkey = qk; return 0; } EXPORT_SYMBOL(mlx4_get_parav_qkey); void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val) { struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); if (!mlx4_is_master(dev)) return; priv->virt2phys_pkey[slave][port - 1][i] = val; } EXPORT_SYMBOL(mlx4_sync_pkey_table); void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid) { struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); if (!mlx4_is_master(dev)) return; priv->slave_node_guids[slave] = guid; } EXPORT_SYMBOL(mlx4_put_slave_node_guid); __be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave) { struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); if (!mlx4_is_master(dev)) return 0; return priv->slave_node_guids[slave]; } EXPORT_SYMBOL(mlx4_get_slave_node_guid); int mlx4_is_slave_active(struct mlx4_dev *dev, int slave) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_slave_state *s_slave; if (!mlx4_is_master(dev)) return 0; s_slave = &priv->mfunc.master.slave_state[slave]; return !!s_slave->active; } EXPORT_SYMBOL(mlx4_is_slave_active); static void slave_adjust_steering_mode(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, struct mlx4_init_hca_param *hca_param) { dev->caps.steering_mode = hca_param->steering_mode; if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; dev->caps.fs_log_max_ucast_qp_range_size = dev_cap->fs_log_max_ucast_qp_range_size; } else dev->caps.num_qp_per_mgm = 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2); mlx4_dbg(dev, "Steering mode is: %s\n", mlx4_steering_mode_str(dev->caps.steering_mode)); } static int mlx4_slave_cap(struct mlx4_dev *dev) { int err; u32 page_size; struct mlx4_dev_cap dev_cap; struct mlx4_func_cap func_cap; struct mlx4_init_hca_param hca_param; u8 i; memset(&hca_param, 0, sizeof(hca_param)); err = mlx4_QUERY_HCA(dev, &hca_param); if (err) { mlx4_err(dev, "QUERY_HCA command failed, aborting\n"); return err; } /* fail if the hca has an unknown global capability * at this time global_caps should be always zeroed */ if (hca_param.global_caps) { mlx4_err(dev, "Unknown hca global capabilities\n"); return -ENOSYS; } mlx4_log_num_mgm_entry_size = hca_param.log_mc_entry_sz; dev->caps.hca_core_clock = hca_param.hca_core_clock; memset(&dev_cap, 0, sizeof(dev_cap)); dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp; err = mlx4_dev_cap(dev, &dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); return err; } err = mlx4_QUERY_FW(dev); if (err) mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n"); page_size = ~dev->caps.page_size_cap + 1; mlx4_warn(dev, "HCA minimum page size:%d\n", page_size); if (page_size > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n", page_size, (long)PAGE_SIZE); return -ENODEV; } /* Set uar_page_shift for VF */ dev->uar_page_shift = hca_param.uar_page_sz + 12; /* Make sure the master uar page size is valid */ if (dev->uar_page_shift > PAGE_SHIFT) { mlx4_err(dev, "Invalid configuration: uar page size is larger than system page size\n"); return -ENODEV; } /* Set reserved_uars based on the uar_page_shift */ mlx4_set_num_reserved_uars(dev, &dev_cap); /* Although uar page size in FW differs from system page size, * upper software layers (mlx4_ib, mlx4_en and part of mlx4_core) * still works with assumption that uar page size == system page size */ dev->caps.uar_page_size = PAGE_SIZE; memset(&func_cap, 0, sizeof(func_cap)); err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap); if (err) { mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n", err); return err; } if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) != PF_CONTEXT_BEHAVIOUR_MASK) { mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n", func_cap.pf_context_behaviour, PF_CONTEXT_BEHAVIOUR_MASK); return -ENOSYS; } dev->caps.num_ports = func_cap.num_ports; dev->quotas.qp = func_cap.qp_quota; dev->quotas.srq = func_cap.srq_quota; dev->quotas.cq = func_cap.cq_quota; dev->quotas.mpt = func_cap.mpt_quota; dev->quotas.mtt = func_cap.mtt_quota; dev->caps.num_qps = 1 << hca_param.log_num_qps; dev->caps.num_srqs = 1 << hca_param.log_num_srqs; dev->caps.num_cqs = 1 << hca_param.log_num_cqs; dev->caps.num_mpts = 1 << hca_param.log_mpt_sz; dev->caps.num_eqs = func_cap.max_eq; dev->caps.reserved_eqs = func_cap.reserved_eq; dev->caps.reserved_lkey = func_cap.reserved_lkey; dev->caps.num_pds = MLX4_NUM_PDS; dev->caps.num_mgms = 0; dev->caps.num_amgms = 0; if (dev->caps.num_ports > MLX4_MAX_PORTS) { mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n", dev->caps.num_ports, MLX4_MAX_PORTS); return -ENODEV; } mlx4_replace_zero_macs(dev); dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL); dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy || !dev->caps.qp0_qkey) { err = -ENOMEM; goto err_mem; } for (i = 1; i <= dev->caps.num_ports; ++i) { err = mlx4_QUERY_FUNC_CAP(dev, i, &func_cap); if (err) { mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n", i, err); goto err_mem; } dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey; dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn; dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn; dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn; dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn; dev->caps.port_mask[i] = dev->caps.port_type[i]; dev->caps.phys_port_id[i] = func_cap.phys_port_id; err = mlx4_get_slave_pkey_gid_tbl_len(dev, i, &dev->caps.gid_table_len[i], &dev->caps.pkey_table_len[i]); if (err) goto err_mem; } if (dev->caps.uar_page_size * (dev->caps.num_uars - dev->caps.reserved_uars) > pci_resource_len(dev->persist->pdev, 2)) { mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n", dev->caps.uar_page_size * dev->caps.num_uars, (unsigned long long) pci_resource_len(dev->persist->pdev, 2)); err = -ENOMEM; goto err_mem; } if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) { dev->caps.eqe_size = 64; dev->caps.eqe_factor = 1; } else { dev->caps.eqe_size = 32; dev->caps.eqe_factor = 0; } if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) { dev->caps.cqe_size = 64; dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; } else { dev->caps.cqe_size = 32; } if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) { dev->caps.eqe_size = hca_param.eqe_size; dev->caps.eqe_factor = 0; } if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) { dev->caps.cqe_size = hca_param.cqe_size; /* User still need to know when CQE > 32B */ dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; } dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; mlx4_warn(dev, "Timestamping is not supported in slave mode\n"); slave_adjust_steering_mode(dev, &dev_cap, &hca_param); mlx4_dbg(dev, "RSS support for IP fragments is %s\n", hca_param.rss_ip_frags ? "on" : "off"); if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP && dev->caps.bf_reg_size) dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP; if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP) dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP; return 0; err_mem: kfree(dev->caps.qp0_qkey); kfree(dev->caps.qp0_tunnel); kfree(dev->caps.qp0_proxy); kfree(dev->caps.qp1_tunnel); kfree(dev->caps.qp1_proxy); dev->caps.qp0_qkey = NULL; dev->caps.qp0_tunnel = NULL; dev->caps.qp0_proxy = NULL; dev->caps.qp1_tunnel = NULL; dev->caps.qp1_proxy = NULL; return err; } static void mlx4_request_modules(struct mlx4_dev *dev) { int port; int has_ib_port = false; int has_eth_port = false; #define EN_DRV_NAME "mlx4_en" #define IB_DRV_NAME "mlx4_ib" for (port = 1; port <= dev->caps.num_ports; port++) { if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) has_ib_port = true; else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) has_eth_port = true; } if (has_eth_port) request_module_nowait(EN_DRV_NAME); if (has_ib_port || (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE)) request_module_nowait(IB_DRV_NAME); } /* * Change the port configuration of the device. * Every user of this function must hold the port mutex. */ int mlx4_change_port_types(struct mlx4_dev *dev, enum mlx4_port_type *port_types) { int err = 0; int change = 0; int port; for (port = 0; port < dev->caps.num_ports; port++) { /* Change the port type only if the new type is different * from the current, and not set to Auto */ if (port_types[port] != dev->caps.port_type[port + 1]) change = 1; } if (change) { mlx4_unregister_device(dev); for (port = 1; port <= dev->caps.num_ports; port++) { mlx4_CLOSE_PORT(dev, port); dev->caps.port_type[port] = port_types[port - 1]; err = mlx4_SET_PORT(dev, port, -1); if (err) { mlx4_err(dev, "Failed to set port %d, aborting\n", port); goto out; } } mlx4_set_port_mask(dev); err = mlx4_register_device(dev); if (err) { mlx4_err(dev, "Failed to register device\n"); goto out; } mlx4_request_modules(dev); } out: return err; } static ssize_t show_port_type(struct device *dev, struct device_attribute *attr, char *buf) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_attr); struct mlx4_dev *mdev = info->dev; char type[8]; sprintf(type, "%s", (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ? "ib" : "eth"); if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO) sprintf(buf, "auto (%s)\n", type); else sprintf(buf, "%s\n", type); return strlen(buf); } static int __set_port_type(struct mlx4_port_info *info, enum mlx4_port_type port_type) { struct mlx4_dev *mdev = info->dev; struct mlx4_priv *priv = mlx4_priv(mdev); enum mlx4_port_type types[MLX4_MAX_PORTS]; enum mlx4_port_type new_types[MLX4_MAX_PORTS]; int i; int err = 0; if ((port_type & mdev->caps.supported_type[info->port]) != port_type) { mlx4_err(mdev, "Requested port type for port %d is not supported on this HCA\n", info->port); err = -EINVAL; goto err_sup; } mlx4_stop_sense(mdev); mutex_lock(&priv->port_mutex); info->tmp_type = port_type; /* Possible type is always the one that was delivered */ mdev->caps.possible_type[info->port] = info->tmp_type; for (i = 0; i < mdev->caps.num_ports; i++) { types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type : mdev->caps.possible_type[i+1]; if (types[i] == MLX4_PORT_TYPE_AUTO) types[i] = mdev->caps.port_type[i+1]; } if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) { for (i = 1; i <= mdev->caps.num_ports; i++) { if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { mdev->caps.possible_type[i] = mdev->caps.port_type[i]; err = -EINVAL; } } } if (err) { mlx4_err(mdev, "Auto sensing is not supported on this HCA. Set only 'eth' or 'ib' for both ports (should be the same)\n"); goto out; } mlx4_do_sense_ports(mdev, new_types, types); err = mlx4_check_port_params(mdev, new_types); if (err) goto out; /* We are about to apply the changes after the configuration * was verified, no need to remember the temporary types * any more */ for (i = 0; i < mdev->caps.num_ports; i++) priv->port[i + 1].tmp_type = 0; err = mlx4_change_port_types(mdev, new_types); out: mutex_unlock(&priv->port_mutex); mlx4_start_sense(mdev); err_sup: return err; } static ssize_t set_port_type(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_attr); struct mlx4_dev *mdev = info->dev; enum mlx4_port_type port_type; static DEFINE_MUTEX(set_port_type_mutex); int err; mutex_lock(&set_port_type_mutex); if (!strcmp(buf, "ib\n")) { port_type = MLX4_PORT_TYPE_IB; } else if (!strcmp(buf, "eth\n")) { port_type = MLX4_PORT_TYPE_ETH; } else if (!strcmp(buf, "auto\n")) { port_type = MLX4_PORT_TYPE_AUTO; } else { mlx4_err(mdev, "%s is not supported port type\n", buf); err = -EINVAL; goto err_out; } err = __set_port_type(info, port_type); err_out: mutex_unlock(&set_port_type_mutex); return err ? err : count; } enum ibta_mtu { IB_MTU_256 = 1, IB_MTU_512 = 2, IB_MTU_1024 = 3, IB_MTU_2048 = 4, IB_MTU_4096 = 5 }; static inline int int_to_ibta_mtu(int mtu) { switch (mtu) { case 256: return IB_MTU_256; case 512: return IB_MTU_512; case 1024: return IB_MTU_1024; case 2048: return IB_MTU_2048; case 4096: return IB_MTU_4096; default: return -1; } } static inline int ibta_mtu_to_int(enum ibta_mtu mtu) { switch (mtu) { case IB_MTU_256: return 256; case IB_MTU_512: return 512; case IB_MTU_1024: return 1024; case IB_MTU_2048: return 2048; case IB_MTU_4096: return 4096; default: return -1; } } static ssize_t show_port_ib_mtu(struct device *dev, struct device_attribute *attr, char *buf) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_mtu_attr); struct mlx4_dev *mdev = info->dev; - if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) - mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); - sprintf(buf, "%d\n", ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port])); return strlen(buf); } static ssize_t set_port_ib_mtu(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_mtu_attr); struct mlx4_dev *mdev = info->dev; struct mlx4_priv *priv = mlx4_priv(mdev); int err, port, mtu, ibta_mtu = -1; if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) { mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); return -EINVAL; } err = kstrtoint(buf, 0, &mtu); if (!err) ibta_mtu = int_to_ibta_mtu(mtu); if (err || ibta_mtu < 0) { mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf); return -EINVAL; } mdev->caps.port_ib_mtu[info->port] = ibta_mtu; mlx4_stop_sense(mdev); mutex_lock(&priv->port_mutex); mlx4_unregister_device(mdev); for (port = 1; port <= mdev->caps.num_ports; port++) { mlx4_CLOSE_PORT(mdev, port); err = mlx4_SET_PORT(mdev, port, -1); if (err) { mlx4_err(mdev, "Failed to set port %d, aborting\n", port); goto err_set_port; } } err = mlx4_register_device(mdev); err_set_port: mutex_unlock(&priv->port_mutex); mlx4_start_sense(mdev); return err ? err : count; } /* bond for multi-function device */ #define MAX_MF_BOND_ALLOWED_SLAVES 63 static int mlx4_mf_bond(struct mlx4_dev *dev) { int err = 0; int nvfs; struct mlx4_slaves_pport slaves_port1; struct mlx4_slaves_pport slaves_port2; DECLARE_BITMAP(slaves_port_1_2, MLX4_MFUNC_MAX); slaves_port1 = mlx4_phys_to_slaves_pport(dev, 1); slaves_port2 = mlx4_phys_to_slaves_pport(dev, 2); bitmap_and(slaves_port_1_2, slaves_port1.slaves, slaves_port2.slaves, dev->persist->num_vfs + 1); /* only single port vfs are allowed */ if (bitmap_weight(slaves_port_1_2, dev->persist->num_vfs + 1) > 1) { mlx4_warn(dev, "HA mode unsupported for dual ported VFs\n"); return -EINVAL; } /* number of virtual functions is number of total functions minus one * physical function for each port. */ nvfs = bitmap_weight(slaves_port1.slaves, dev->persist->num_vfs + 1) + bitmap_weight(slaves_port2.slaves, dev->persist->num_vfs + 1) - 2; /* limit on maximum allowed VFs */ if (nvfs > MAX_MF_BOND_ALLOWED_SLAVES) { mlx4_warn(dev, "HA mode is not supported for %d VFs (max %d are allowed)\n", nvfs, MAX_MF_BOND_ALLOWED_SLAVES); return -EINVAL; } if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) { mlx4_warn(dev, "HA mode unsupported for NON DMFS steering\n"); return -EINVAL; } err = mlx4_bond_mac_table(dev); if (err) return err; err = mlx4_bond_vlan_table(dev); if (err) goto err1; err = mlx4_bond_fs_rules(dev); if (err) goto err2; return 0; err2: (void)mlx4_unbond_vlan_table(dev); err1: (void)mlx4_unbond_mac_table(dev); return err; } static int mlx4_mf_unbond(struct mlx4_dev *dev) { int ret, ret1; ret = mlx4_unbond_fs_rules(dev); if (ret) mlx4_warn(dev, "multifunction unbond for flow rules failedi (%d)\n", ret); ret1 = mlx4_unbond_mac_table(dev); if (ret1) { mlx4_warn(dev, "multifunction unbond for MAC table failed (%d)\n", ret1); ret = ret1; } ret1 = mlx4_unbond_vlan_table(dev); if (ret1) { mlx4_warn(dev, "multifunction unbond for VLAN table failed (%d)\n", ret1); ret = ret1; } return ret; } int mlx4_bond(struct mlx4_dev *dev) { int ret = 0; struct mlx4_priv *priv = mlx4_priv(dev); mutex_lock(&priv->bond_mutex); if (!mlx4_is_bonded(dev)) { ret = mlx4_do_bond(dev, true); if (ret) mlx4_err(dev, "Failed to bond device: %d\n", ret); if (!ret && mlx4_is_master(dev)) { ret = mlx4_mf_bond(dev); if (ret) { mlx4_err(dev, "bond for multifunction failed\n"); mlx4_do_bond(dev, false); } } } mutex_unlock(&priv->bond_mutex); if (!ret) mlx4_dbg(dev, "Device is bonded\n"); return ret; } EXPORT_SYMBOL_GPL(mlx4_bond); int mlx4_unbond(struct mlx4_dev *dev) { int ret = 0; struct mlx4_priv *priv = mlx4_priv(dev); mutex_lock(&priv->bond_mutex); if (mlx4_is_bonded(dev)) { int ret2 = 0; ret = mlx4_do_bond(dev, false); if (ret) mlx4_err(dev, "Failed to unbond device: %d\n", ret); if (mlx4_is_master(dev)) ret2 = mlx4_mf_unbond(dev); if (ret2) { mlx4_warn(dev, "Failed to unbond device for multifunction (%d)\n", ret2); ret = ret2; } } mutex_unlock(&priv->bond_mutex); if (!ret) mlx4_dbg(dev, "Device is unbonded\n"); return ret; } EXPORT_SYMBOL_GPL(mlx4_unbond); int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p) { u8 port1 = v2p->port1; u8 port2 = v2p->port2; struct mlx4_priv *priv = mlx4_priv(dev); int err; if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP)) return -ENOTSUPP; mutex_lock(&priv->bond_mutex); /* zero means keep current mapping for this port */ if (port1 == 0) port1 = priv->v2p.port1; if (port2 == 0) port2 = priv->v2p.port2; if ((port1 < 1) || (port1 > MLX4_MAX_PORTS) || (port2 < 1) || (port2 > MLX4_MAX_PORTS) || (port1 == 2 && port2 == 1)) { /* besides boundary checks cross mapping makes * no sense and therefore not allowed */ err = -EINVAL; } else if ((port1 == priv->v2p.port1) && (port2 == priv->v2p.port2)) { err = 0; } else { err = mlx4_virt2phy_port_map(dev, port1, port2); if (!err) { mlx4_dbg(dev, "port map changed: [%d][%d]\n", port1, port2); priv->v2p.port1 = port1; priv->v2p.port2 = port2; } else { mlx4_err(dev, "Failed to change port mape: %d\n", err); } } mutex_unlock(&priv->bond_mutex); return err; } EXPORT_SYMBOL_GPL(mlx4_port_map_set); static int mlx4_load_fw(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int err; priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!priv->fw.fw_icm) { mlx4_err(dev, "Couldn't allocate FW area, aborting\n"); return -ENOMEM; } err = mlx4_MAP_FA(dev, priv->fw.fw_icm); if (err) { mlx4_err(dev, "MAP_FA command failed, aborting\n"); goto err_free; } err = mlx4_RUN_FW(dev); if (err) { mlx4_err(dev, "RUN_FW command failed, aborting\n"); goto err_unmap_fa; } return 0; err_unmap_fa: mlx4_UNMAP_FA(dev); err_free: mlx4_free_icm(dev, priv->fw.fw_icm, 0); return err; } static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base, int cmpt_entry_sz) { struct mlx4_priv *priv = mlx4_priv(dev); int err; int num_eqs; err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_QP * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) goto err; err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_SRQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_srqs, dev->caps.reserved_srqs, 0, 0); if (err) goto err_qp; err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_CQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_cqs, dev->caps.reserved_cqs, 0, 0); if (err) goto err_srq; num_eqs = dev->phys_caps.num_phys_eqs; err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_EQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, num_eqs, num_eqs, 0, 0); if (err) goto err_cq; return 0; err_cq: mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); err_srq: mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); err_qp: mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); err: return err; } static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, struct mlx4_init_hca_param *init_hca, u64 icm_size) { struct mlx4_priv *priv = mlx4_priv(dev); u64 aux_pages; int num_eqs; int err; err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages); if (err) { mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n"); return err; } mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n", (unsigned long long) icm_size >> 10, (unsigned long long) aux_pages << 2); priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!priv->fw.aux_icm) { mlx4_err(dev, "Couldn't allocate aux memory, aborting\n"); return -ENOMEM; } err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm); if (err) { mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n"); goto err_free_aux; } err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz); if (err) { mlx4_err(dev, "Failed to map cMPT context memory, aborting\n"); goto err_unmap_aux; } num_eqs = dev->phys_caps.num_phys_eqs; err = mlx4_init_icm_table(dev, &priv->eq_table.table, init_hca->eqc_base, dev_cap->eqc_entry_sz, num_eqs, num_eqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map EQ context memory, aborting\n"); goto err_unmap_cmpt; } /* * Reserved MTT entries must be aligned up to a cacheline * boundary, since the FW will write to them, while the driver * writes to all other MTT entries. (The variable * dev->caps.mtt_entry_sz below is really the MTT segment * size, not the raw entry size) */ dev->caps.reserved_mtts = ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz, dma_get_cache_alignment()) / dev->caps.mtt_entry_sz; err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table, init_hca->mtt_base, dev->caps.mtt_entry_sz, dev->caps.num_mtts, dev->caps.reserved_mtts, 1, 0); if (err) { mlx4_err(dev, "Failed to map MTT context memory, aborting\n"); goto err_unmap_eq; } err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table, init_hca->dmpt_base, dev_cap->dmpt_entry_sz, dev->caps.num_mpts, dev->caps.reserved_mrws, 1, 1); if (err) { mlx4_err(dev, "Failed to map dMPT context memory, aborting\n"); goto err_unmap_mtt; } err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table, init_hca->qpc_base, dev_cap->qpc_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map QP context memory, aborting\n"); goto err_unmap_dmpt; } err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table, init_hca->auxc_base, dev_cap->aux_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map AUXC context memory, aborting\n"); goto err_unmap_qp; } err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table, init_hca->altc_base, dev_cap->altc_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map ALTC context memory, aborting\n"); goto err_unmap_auxc; } err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table, init_hca->rdmarc_base, dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n"); goto err_unmap_altc; } err = mlx4_init_icm_table(dev, &priv->cq_table.table, init_hca->cqc_base, dev_cap->cqc_entry_sz, dev->caps.num_cqs, dev->caps.reserved_cqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map CQ context memory, aborting\n"); goto err_unmap_rdmarc; } err = mlx4_init_icm_table(dev, &priv->srq_table.table, init_hca->srqc_base, dev_cap->srq_entry_sz, dev->caps.num_srqs, dev->caps.reserved_srqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map SRQ context memory, aborting\n"); goto err_unmap_cq; } /* * For flow steering device managed mode it is required to use * mlx4_init_icm_table. For B0 steering mode it's not strictly * required, but for simplicity just map the whole multicast * group table now. The table isn't very big and it's a lot * easier than trying to track ref counts. */ err = mlx4_init_icm_table(dev, &priv->mcg_table.table, init_hca->mc_base, mlx4_get_mgm_entry_size(dev), dev->caps.num_mgms + dev->caps.num_amgms, dev->caps.num_mgms + dev->caps.num_amgms, 0, 0); if (err) { mlx4_err(dev, "Failed to map MCG context memory, aborting\n"); goto err_unmap_srq; } return 0; err_unmap_srq: mlx4_cleanup_icm_table(dev, &priv->srq_table.table); err_unmap_cq: mlx4_cleanup_icm_table(dev, &priv->cq_table.table); err_unmap_rdmarc: mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); err_unmap_altc: mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); err_unmap_auxc: mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); err_unmap_qp: mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); err_unmap_dmpt: mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); err_unmap_mtt: mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); err_unmap_eq: mlx4_cleanup_icm_table(dev, &priv->eq_table.table); err_unmap_cmpt: mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); err_unmap_aux: mlx4_UNMAP_ICM_AUX(dev); err_free_aux: mlx4_free_icm(dev, priv->fw.aux_icm, 0); return err; } static void mlx4_free_icms(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); mlx4_cleanup_icm_table(dev, &priv->mcg_table.table); mlx4_cleanup_icm_table(dev, &priv->srq_table.table); mlx4_cleanup_icm_table(dev, &priv->cq_table.table); mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); mlx4_cleanup_icm_table(dev, &priv->eq_table.table); mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); mlx4_UNMAP_ICM_AUX(dev); mlx4_free_icm(dev, priv->fw.aux_icm, 0); } static void mlx4_slave_exit(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); mutex_lock(&priv->cmd.slave_cmd_mutex); if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) mlx4_warn(dev, "Failed to close slave function\n"); mutex_unlock(&priv->cmd.slave_cmd_mutex); } static int map_bf_area(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); resource_size_t bf_start; resource_size_t bf_len; int err = 0; if (!dev->caps.bf_reg_size) return -ENXIO; bf_start = pci_resource_start(dev->persist->pdev, 2) + (dev->caps.num_uars << PAGE_SHIFT); bf_len = pci_resource_len(dev->persist->pdev, 2) - (dev->caps.num_uars << PAGE_SHIFT); priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len); if (!priv->bf_mapping) err = -ENOMEM; return err; } static void unmap_bf_area(struct mlx4_dev *dev) { if (mlx4_priv(dev)->bf_mapping) io_mapping_free(mlx4_priv(dev)->bf_mapping); } s64 mlx4_read_clock(struct mlx4_dev *dev) { u32 clockhi, clocklo, clockhi1; s64 cycles; int i; struct mlx4_priv *priv = mlx4_priv(dev); if (!priv->clock_mapping) return -ENOTSUPP; for (i = 0; i < 10; i++) { clockhi = swab32(readl(priv->clock_mapping)); clocklo = swab32(readl(priv->clock_mapping + 4)); clockhi1 = swab32(readl(priv->clock_mapping)); if (clockhi == clockhi1) break; } cycles = (u64) clockhi << 32 | (u64) clocklo; return cycles & CORE_CLOCK_MASK; } EXPORT_SYMBOL_GPL(mlx4_read_clock); static int map_internal_clock(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); priv->clock_mapping = ioremap(pci_resource_start(dev->persist->pdev, priv->fw.clock_bar) + priv->fw.clock_offset, MLX4_CLOCK_SIZE); if (!priv->clock_mapping) return -ENOMEM; return 0; } int mlx4_get_internal_clock_params(struct mlx4_dev *dev, struct mlx4_clock_params *params) { struct mlx4_priv *priv = mlx4_priv(dev); if (mlx4_is_slave(dev)) return -ENOTSUPP; if (!params) return -EINVAL; params->bar = priv->fw.clock_bar; params->offset = priv->fw.clock_offset; params->size = MLX4_CLOCK_SIZE; return 0; } EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params); static void unmap_internal_clock(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); if (priv->clock_mapping) iounmap(priv->clock_mapping); } static void mlx4_close_hca(struct mlx4_dev *dev) { sysctl_ctx_free(&dev->hw_ctx); unmap_internal_clock(dev); unmap_bf_area(dev); if (mlx4_is_slave(dev)) mlx4_slave_exit(dev); else { mlx4_CLOSE_HCA(dev, 0); mlx4_free_icms(dev); } } static void mlx4_close_fw(struct mlx4_dev *dev) { if (!mlx4_is_slave(dev)) { mlx4_UNMAP_FA(dev); mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0); } } static int mlx4_comm_check_offline(struct mlx4_dev *dev) { #define COMM_CHAN_OFFLINE_OFFSET 0x09 u32 comm_flags; u32 offline_bit; unsigned long end; struct mlx4_priv *priv = mlx4_priv(dev); end = msecs_to_jiffies(MLX4_COMM_OFFLINE_TIME_OUT) + jiffies; while (time_before(jiffies, end)) { comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm + MLX4_COMM_CHAN_FLAGS)); offline_bit = (comm_flags & (u32)(1 << COMM_CHAN_OFFLINE_OFFSET)); if (!offline_bit) return 0; /* There are cases as part of AER/Reset flow that PF needs * around 100 msec to load. We therefore sleep for 100 msec * to allow other tasks to make use of that CPU during this * time interval. */ msleep(100); } mlx4_err(dev, "Communication channel is offline.\n"); return -EIO; } static void mlx4_reset_vf_support(struct mlx4_dev *dev) { #define COMM_CHAN_RST_OFFSET 0x1e struct mlx4_priv *priv = mlx4_priv(dev); u32 comm_rst; u32 comm_caps; comm_caps = swab32(readl((__iomem char *)priv->mfunc.comm + MLX4_COMM_CHAN_CAPS)); comm_rst = (comm_caps & (u32)(1 << COMM_CHAN_RST_OFFSET)); if (comm_rst) dev->caps.vf_caps |= MLX4_VF_CAP_FLAG_RESET; } static int mlx4_init_slave(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); u64 dma = (u64) priv->mfunc.vhcr_dma; int ret_from_reset = 0; u32 slave_read; u32 cmd_channel_ver; if (atomic_read(&pf_loading)) { mlx4_warn(dev, "PF is not ready - Deferring probe\n"); return -EAGAIN; } mutex_lock(&priv->cmd.slave_cmd_mutex); priv->cmd.max_cmds = 1; if (mlx4_comm_check_offline(dev)) { mlx4_err(dev, "PF is not responsive, skipping initialization\n"); goto err_offline; } mlx4_reset_vf_support(dev); mlx4_warn(dev, "Sending reset\n"); ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME); /* if we are in the middle of flr the slave will try * NUM_OF_RESET_RETRIES times before leaving.*/ if (ret_from_reset) { if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) { mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n"); mutex_unlock(&priv->cmd.slave_cmd_mutex); return -EAGAIN; } else goto err; } /* check the driver version - the slave I/F revision * must match the master's */ slave_read = swab32(readl(&priv->mfunc.comm->slave_read)); cmd_channel_ver = mlx4_comm_get_version(); if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) != MLX4_COMM_GET_IF_REV(slave_read)) { mlx4_err(dev, "slave driver version is not supported by the master\n"); goto err; } mlx4_warn(dev, "Sending vhcr0\n"); if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) goto err; if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) goto err; if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) goto err; if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) goto err; mutex_unlock(&priv->cmd.slave_cmd_mutex); return 0; err: mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, 0); err_offline: mutex_unlock(&priv->cmd.slave_cmd_mutex); return -EIO; } static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev) { int i; for (i = 1; i <= dev->caps.num_ports; i++) { if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH) dev->caps.gid_table_len[i] = mlx4_get_slave_num_gids(dev, 0, i); else dev->caps.gid_table_len[i] = 1; dev->caps.pkey_table_len[i] = dev->phys_caps.pkey_phys_table_len[i] - 1; } } static int choose_log_fs_mgm_entry_size(int qp_per_entry) { int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE; i++) { if (qp_per_entry <= 4 * ((1 << i) / 16 - 2)) break; } return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1; } static const char *dmfs_high_rate_steering_mode_str(int dmfs_high_steer_mode) { switch (dmfs_high_steer_mode) { case MLX4_STEERING_DMFS_A0_DEFAULT: return "default performance"; case MLX4_STEERING_DMFS_A0_DYNAMIC: return "dynamic hybrid mode"; case MLX4_STEERING_DMFS_A0_STATIC: return "performance optimized for limited rule configuration (static)"; case MLX4_STEERING_DMFS_A0_DISABLE: return "disabled performance optimized steering"; case MLX4_STEERING_DMFS_A0_NOT_SUPPORTED: return "performance optimized steering not supported"; default: return "Unrecognized mode"; } } #define MLX4_DMFS_A0_STEERING (1UL << 2) static void choose_steering_mode(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { if (mlx4_log_num_mgm_entry_size <= 0) { if ((-mlx4_log_num_mgm_entry_size) & MLX4_DMFS_A0_STEERING) { if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) mlx4_err(dev, "DMFS high rate mode not supported\n"); else dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_STATIC; } } if (mlx4_log_num_mgm_entry_size <= 0 && dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN && (!mlx4_is_mfunc(dev) || (dev_cap->fs_max_num_qp_per_entry >= (dev->persist->num_vfs + 1))) && choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >= MLX4_MIN_MGM_LOG_ENTRY_SIZE) { dev->oper_log_mgm_entry_size = choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry); dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; dev->caps.fs_log_max_ucast_qp_range_size = dev_cap->fs_log_max_ucast_qp_range_size; } else { if (dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DISABLE; if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER && dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) dev->caps.steering_mode = MLX4_STEERING_MODE_B0; else { dev->caps.steering_mode = MLX4_STEERING_MODE_A0; if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER || dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back to A0 steering mode\n"); } dev->oper_log_mgm_entry_size = mlx4_log_num_mgm_entry_size > 0 ? mlx4_log_num_mgm_entry_size : MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev); } mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size = %d\n", mlx4_steering_mode_str(dev->caps.steering_mode), dev->oper_log_mgm_entry_size, mlx4_log_num_mgm_entry_size); } static void choose_tunnel_offload_mode(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED && dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS) dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN; else dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE; mlx4_dbg(dev, "Tunneling offload mode is: %s\n", (dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) ? "vxlan" : "none"); } static int mlx4_validate_optimized_steering(struct mlx4_dev *dev) { int i; struct mlx4_port_cap port_cap; if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) return -EINVAL; for (i = 1; i <= dev->caps.num_ports; i++) { if (mlx4_dev_port(dev, i, &port_cap)) { mlx4_err(dev, "QUERY_DEV_CAP command failed, can't veify DMFS high rate steering.\n"); } else if ((dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_DEFAULT) && (port_cap.dmfs_optimized_state == !!(dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_DISABLE))) { mlx4_err(dev, "DMFS high rate steer mode differ, driver requested %s but %s in FW.\n", dmfs_high_rate_steering_mode_str( dev->caps.dmfs_high_steer_mode), (port_cap.dmfs_optimized_state ? "enabled" : "disabled")); } } return 0; } static int mlx4_init_fw(struct mlx4_dev *dev) { struct mlx4_mod_stat_cfg mlx4_cfg; int err = 0; if (!mlx4_is_slave(dev)) { err = mlx4_QUERY_FW(dev); if (err) { if (err == -EACCES) mlx4_info(dev, "non-primary physical function, skipping\n"); else mlx4_err(dev, "QUERY_FW command failed, aborting\n"); return err; } err = mlx4_load_fw(dev); if (err) { mlx4_err(dev, "Failed to start FW, aborting\n"); return err; } mlx4_cfg.log_pg_sz_m = 1; mlx4_cfg.log_pg_sz = 0; err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg); if (err) mlx4_warn(dev, "Failed to override log_pg_sz parameter\n"); } return err; } static int mlx4_init_hca(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_adapter adapter; struct mlx4_dev_cap dev_cap = {}; struct mlx4_profile profile; struct mlx4_init_hca_param init_hca; u64 icm_size; struct mlx4_config_dev_params params; int err; if (!mlx4_is_slave(dev)) { err = mlx4_dev_cap(dev, &dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); return err; } choose_steering_mode(dev, &dev_cap); choose_tunnel_offload_mode(dev, &dev_cap); if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC && mlx4_is_master(dev)) dev->caps.function_caps |= MLX4_FUNC_CAP_DMFS_A0_STATIC; err = mlx4_get_phys_port_id(dev); if (err) mlx4_err(dev, "Fail to get physical port id\n"); if (mlx4_is_master(dev)) mlx4_parav_master_pf_caps(dev); if (mlx4_low_memory_profile()) { mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n"); profile = low_mem_profile; } else { profile = default_profile; } if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) profile.num_mcg = MLX4_FS_NUM_MCG; icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca); if ((long long) icm_size < 0) { err = icm_size; return err; } dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1; if (enable_4k_uar) { init_hca.log_uar_sz = ilog2(dev->caps.num_uars) + PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT; init_hca.uar_page_sz = DEFAULT_UAR_PAGE_SHIFT - 12; } else { init_hca.log_uar_sz = ilog2(dev->caps.num_uars); init_hca.uar_page_sz = PAGE_SHIFT - 12; } init_hca.mw_enabled = 0; if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW || dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) init_hca.mw_enabled = INIT_HCA_TPT_MW_ENABLE; err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size); if (err) return err; err = mlx4_INIT_HCA(dev, &init_hca); if (err) { mlx4_err(dev, "INIT_HCA command failed, aborting\n"); goto err_free_icm; } if (dev_cap.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { err = mlx4_query_func(dev, &dev_cap); if (err < 0) { mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n"); goto err_close; } else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) { dev->caps.num_eqs = dev_cap.max_eqs; dev->caps.reserved_eqs = dev_cap.reserved_eqs; dev->caps.reserved_uars = dev_cap.reserved_uars; } } /* * If TS is supported by FW * read HCA frequency by QUERY_HCA command */ if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { memset(&init_hca, 0, sizeof(init_hca)); err = mlx4_QUERY_HCA(dev, &init_hca); if (err) { mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n"); dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; } else { dev->caps.hca_core_clock = init_hca.hca_core_clock; } /* In case we got HCA frequency 0 - disable timestamping * to avoid dividing by zero */ if (!dev->caps.hca_core_clock) { dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; mlx4_err(dev, "HCA frequency is 0 - timestamping is not supported\n"); } else if (map_internal_clock(dev)) { /* * Map internal clock, * in case of failure disable timestamping */ dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n"); } } if (dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) { if (mlx4_validate_optimized_steering(dev)) mlx4_warn(dev, "Optimized steering validation failed\n"); if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_DISABLE) { dev->caps.dmfs_high_rate_qpn_base = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE; } mlx4_dbg(dev, "DMFS high rate steer mode is: %s\n", dmfs_high_rate_steering_mode_str( dev->caps.dmfs_high_steer_mode)); } } else { err = mlx4_init_slave(dev); if (err) { if (err != -EAGAIN) mlx4_err(dev, "Failed to initialize slave\n"); return err; } err = mlx4_slave_cap(dev); if (err) { mlx4_err(dev, "Failed to obtain slave caps\n"); goto err_close; } } if (map_bf_area(dev)) mlx4_dbg(dev, "Failed to map blue flame area\n"); /*Only the master set the ports, all the rest got it from it.*/ if (!mlx4_is_slave(dev)) mlx4_set_port_mask(dev); err = mlx4_QUERY_ADAPTER(dev, &adapter); if (err) { mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n"); goto unmap_bf; } /* Query CONFIG_DEV parameters */ err = mlx4_config_dev_retrieval(dev, ¶ms); if (err && err != -ENOTSUPP) { mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n"); } else if (!err) { dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1; dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2; } priv->eq_table.inta_pin = adapter.inta_pin; memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id); return 0; unmap_bf: unmap_internal_clock(dev); unmap_bf_area(dev); if (mlx4_is_slave(dev)) { kfree(dev->caps.qp0_qkey); kfree(dev->caps.qp0_tunnel); kfree(dev->caps.qp0_proxy); kfree(dev->caps.qp1_tunnel); kfree(dev->caps.qp1_proxy); } err_close: if (mlx4_is_slave(dev)) mlx4_slave_exit(dev); else mlx4_CLOSE_HCA(dev, 0); err_free_icm: if (!mlx4_is_slave(dev)) mlx4_free_icms(dev); return err; } static int mlx4_init_counters_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int nent_pow2; if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) return -ENOENT; if (!dev->caps.max_counters) return -ENOSPC; nent_pow2 = roundup_pow_of_two(dev->caps.max_counters); /* reserve last counter index for sink counter */ return mlx4_bitmap_init(&priv->counters_bitmap, nent_pow2, nent_pow2 - 1, 0, nent_pow2 - dev->caps.max_counters + 1); } static void mlx4_cleanup_counters_table(struct mlx4_dev *dev) { if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) return; if (!dev->caps.max_counters) return; mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap); } static void mlx4_cleanup_default_counters(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int port; for (port = 0; port < dev->caps.num_ports; port++) if (priv->def_counter[port] != -1) mlx4_counter_free(dev, priv->def_counter[port]); } static int mlx4_allocate_default_counters(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int port, err = 0; u32 idx; for (port = 0; port < dev->caps.num_ports; port++) priv->def_counter[port] = -1; for (port = 0; port < dev->caps.num_ports; port++) { err = mlx4_counter_alloc(dev, &idx); if (!err || err == -ENOSPC) { priv->def_counter[port] = idx; } else if (err == -ENOENT) { err = 0; continue; } else if (mlx4_is_slave(dev) && err == -EINVAL) { priv->def_counter[port] = MLX4_SINK_COUNTER_INDEX(dev); mlx4_warn(dev, "can't allocate counter from old PF driver, using index %d\n", MLX4_SINK_COUNTER_INDEX(dev)); err = 0; } else { mlx4_err(dev, "%s: failed to allocate default counter port %d err %d\n", __func__, port + 1, err); mlx4_cleanup_default_counters(dev); return err; } mlx4_dbg(dev, "%s: default counter index %d for port %d\n", __func__, priv->def_counter[port], port + 1); } return err; } int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) { struct mlx4_priv *priv = mlx4_priv(dev); if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) return -ENOENT; *idx = mlx4_bitmap_alloc(&priv->counters_bitmap); if (*idx == -1) { *idx = MLX4_SINK_COUNTER_INDEX(dev); return -ENOSPC; } return 0; } int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) { u64 out_param; int err; if (mlx4_is_mfunc(dev)) { err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER, RES_OP_RESERVE, MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); if (!err) *idx = get_param_l(&out_param); return err; } return __mlx4_counter_alloc(dev, idx); } EXPORT_SYMBOL_GPL(mlx4_counter_alloc); static int __mlx4_clear_if_stat(struct mlx4_dev *dev, u8 counter_index) { struct mlx4_cmd_mailbox *if_stat_mailbox; int err; u32 if_stat_in_mod = (counter_index & 0xff) | MLX4_QUERY_IF_STAT_RESET; if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(if_stat_mailbox)) return PTR_ERR(if_stat_mailbox); err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0, MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev, if_stat_mailbox); return err; } void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx) { if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) return; if (idx == MLX4_SINK_COUNTER_INDEX(dev)) return; __mlx4_clear_if_stat(dev, idx); mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx, MLX4_USE_RR); return; } void mlx4_counter_free(struct mlx4_dev *dev, u32 idx) { u64 in_param = 0; if (mlx4_is_mfunc(dev)) { set_param_l(&in_param, idx); mlx4_cmd(dev, in_param, RES_COUNTER, RES_OP_RESERVE, MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); return; } __mlx4_counter_free(dev, idx); } EXPORT_SYMBOL_GPL(mlx4_counter_free); int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port) { struct mlx4_priv *priv = mlx4_priv(dev); return priv->def_counter[port - 1]; } EXPORT_SYMBOL_GPL(mlx4_get_default_counter_index); void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, int port) { struct mlx4_priv *priv = mlx4_priv(dev); priv->mfunc.master.vf_admin[entry].vport[port].guid = guid; } EXPORT_SYMBOL_GPL(mlx4_set_admin_guid); __be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port) { struct mlx4_priv *priv = mlx4_priv(dev); return priv->mfunc.master.vf_admin[entry].vport[port].guid; } EXPORT_SYMBOL_GPL(mlx4_get_admin_guid); void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port) { struct mlx4_priv *priv = mlx4_priv(dev); __be64 guid; /* hw GUID */ if (entry == 0) return; get_random_bytes((char *)&guid, sizeof(guid)); guid &= ~(cpu_to_be64(1ULL << 56)); guid |= cpu_to_be64(1ULL << 57); priv->mfunc.master.vf_admin[entry].vport[port].guid = guid; } static int mlx4_setup_hca(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int err; int port; __be32 ib_port_default_caps; err = mlx4_init_uar_table(dev); if (err) { mlx4_err(dev, "Failed to initialize user access region table, aborting\n"); return err; } err = mlx4_uar_alloc(dev, &priv->driver_uar); if (err) { mlx4_err(dev, "Failed to allocate driver access region, aborting\n"); goto err_uar_table_free; } priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); if (!priv->kar) { mlx4_err(dev, "Couldn't map kernel access region, aborting\n"); err = -ENOMEM; goto err_uar_free; } err = mlx4_init_pd_table(dev); if (err) { mlx4_err(dev, "Failed to initialize protection domain table, aborting\n"); goto err_kar_unmap; } err = mlx4_init_xrcd_table(dev); if (err) { mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n"); goto err_pd_table_free; } err = mlx4_init_mr_table(dev); if (err) { mlx4_err(dev, "Failed to initialize memory region table, aborting\n"); goto err_xrcd_table_free; } if (!mlx4_is_slave(dev)) { err = mlx4_init_mcg_table(dev); if (err) { mlx4_err(dev, "Failed to initialize multicast group table, aborting\n"); goto err_mr_table_free; } err = mlx4_config_mad_demux(dev); if (err) { mlx4_err(dev, "Failed in config_mad_demux, aborting\n"); goto err_mcg_table_free; } } err = mlx4_init_eq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize event queue table, aborting\n"); goto err_mcg_table_free; } err = mlx4_cmd_use_events(dev); if (err) { mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n"); goto err_eq_table_free; } err = mlx4_NOP(dev); if (err) { if (dev->flags & MLX4_FLAG_MSI_X) { mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n", priv->eq_table.eq[MLX4_EQ_ASYNC].irq); mlx4_warn(dev, "Trying again without MSI-X\n"); } else { mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n", priv->eq_table.eq[MLX4_EQ_ASYNC].irq); mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); } goto err_cmd_poll; } mlx4_dbg(dev, "NOP command IRQ test passed\n"); err = mlx4_init_cq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize completion queue table, aborting\n"); goto err_cmd_poll; } err = mlx4_init_srq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n"); goto err_cq_table_free; } err = mlx4_init_qp_table(dev); if (err) { mlx4_err(dev, "Failed to initialize queue pair table, aborting\n"); goto err_srq_table_free; } if (!mlx4_is_slave(dev)) { err = mlx4_init_counters_table(dev); if (err && err != -ENOENT) { mlx4_err(dev, "Failed to initialize counters table, aborting\n"); goto err_qp_table_free; } } err = mlx4_allocate_default_counters(dev); if (err) { mlx4_err(dev, "Failed to allocate default counters, aborting\n"); goto err_counters_table_free; } if (!mlx4_is_slave(dev)) { for (port = 1; port <= dev->caps.num_ports; port++) { ib_port_default_caps = 0; err = mlx4_get_port_ib_caps(dev, port, &ib_port_default_caps); if (err) mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n", port, err); dev->caps.ib_port_def_cap[port] = ib_port_default_caps; /* initialize per-slave default ib port capabilities */ if (mlx4_is_master(dev)) { int i; for (i = 0; i < dev->num_slaves; i++) { if (i == mlx4_master_func_num(dev)) continue; priv->mfunc.master.slave_state[i].ib_cap_mask[port] = ib_port_default_caps; } } if (mlx4_is_mfunc(dev)) dev->caps.port_ib_mtu[port] = IB_MTU_2048; else dev->caps.port_ib_mtu[port] = IB_MTU_4096; err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ? dev->caps.pkey_table_len[port] : -1); if (err) { mlx4_err(dev, "Failed to set port %d, aborting\n", port); goto err_default_countes_free; } } } return 0; err_default_countes_free: mlx4_cleanup_default_counters(dev); err_counters_table_free: if (!mlx4_is_slave(dev)) mlx4_cleanup_counters_table(dev); err_qp_table_free: mlx4_cleanup_qp_table(dev); err_srq_table_free: mlx4_cleanup_srq_table(dev); err_cq_table_free: mlx4_cleanup_cq_table(dev); err_cmd_poll: mlx4_cmd_use_polling(dev); err_eq_table_free: mlx4_cleanup_eq_table(dev); err_mcg_table_free: if (!mlx4_is_slave(dev)) mlx4_cleanup_mcg_table(dev); err_mr_table_free: mlx4_cleanup_mr_table(dev); err_xrcd_table_free: mlx4_cleanup_xrcd_table(dev); err_pd_table_free: mlx4_cleanup_pd_table(dev); err_kar_unmap: iounmap(priv->kar); err_uar_free: mlx4_uar_free(dev, &priv->driver_uar); err_uar_table_free: mlx4_cleanup_uar_table(dev); return err; } static int mlx4_init_affinity_hint(struct mlx4_dev *dev, int port, int eqn) { int requested_cpu = 0; struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_eq *eq; int off = 0; int i; if (eqn > dev->caps.num_comp_vectors) return -EINVAL; for (i = 1; i < port; i++) off += mlx4_get_eqs_per_port(dev, i); requested_cpu = eqn - off - !!(eqn > MLX4_EQ_ASYNC); /* Meaning EQs are shared, and this call comes from the second port */ if (requested_cpu < 0) return 0; eq = &priv->eq_table.eq[eqn]; eq->affinity_cpu_id = requested_cpu % num_online_cpus(); return 0; } static void mlx4_enable_msi_x(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct msix_entry *entries; int i; int port = 0; if (msi_x) { int nreq = dev->caps.num_ports * num_online_cpus() + 1; nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs, nreq); if (nreq > MAX_MSIX) nreq = MAX_MSIX; entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL); if (!entries) goto no_msi; for (i = 0; i < nreq; ++i) entries[i].entry = i; nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2, nreq); if (nreq < 0 || nreq < MLX4_EQ_ASYNC) { kfree(entries); goto no_msi; } /* 1 is reserved for events (asyncrounous EQ) */ dev->caps.num_comp_vectors = nreq - 1; priv->eq_table.eq[MLX4_EQ_ASYNC].irq = entries[0].vector; bitmap_zero(priv->eq_table.eq[MLX4_EQ_ASYNC].actv_ports.ports, dev->caps.num_ports); for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) { if (i == MLX4_EQ_ASYNC) continue; priv->eq_table.eq[i].irq = entries[i + 1 - !!(i > MLX4_EQ_ASYNC)].vector; if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) { bitmap_fill(priv->eq_table.eq[i].actv_ports.ports, dev->caps.num_ports); /* We don't set affinity hint when there * aren't enough EQs */ } else { set_bit(port, priv->eq_table.eq[i].actv_ports.ports); if (mlx4_init_affinity_hint(dev, port + 1, i)) mlx4_warn(dev, "Couldn't init hint cpumask for EQ %d\n", i); } /* We divide the Eqs evenly between the two ports. * (dev->caps.num_comp_vectors / dev->caps.num_ports) * refers to the number of Eqs per port * (i.e eqs_per_port). Theoretically, we would like to * write something like (i + 1) % eqs_per_port == 0. * However, since there's an asynchronous Eq, we have * to skip over it by comparing this condition to * !!((i + 1) > MLX4_EQ_ASYNC). */ if ((dev->caps.num_comp_vectors > dev->caps.num_ports) && ((i + 1) % (dev->caps.num_comp_vectors / dev->caps.num_ports)) == !!((i + 1) > MLX4_EQ_ASYNC)) /* If dev->caps.num_comp_vectors < dev->caps.num_ports, * everything is shared anyway. */ port++; } dev->flags |= MLX4_FLAG_MSI_X; kfree(entries); return; } no_msi: dev->caps.num_comp_vectors = 1; BUG_ON(MLX4_EQ_ASYNC >= 2); for (i = 0; i < 2; ++i) { priv->eq_table.eq[i].irq = dev->persist->pdev->irq; if (i != MLX4_EQ_ASYNC) { bitmap_fill(priv->eq_table.eq[i].actv_ports.ports, dev->caps.num_ports); } } } static int mlx4_init_port_info(struct mlx4_dev *dev, int port) { struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; int err = 0; info->dev = dev; info->port = port; if (!mlx4_is_slave(dev)) { mlx4_init_mac_table(dev, &info->mac_table); mlx4_init_vlan_table(dev, &info->vlan_table); mlx4_init_roce_gid_table(dev, &info->gid_table); info->base_qpn = mlx4_get_base_qpn(dev, port); } sprintf(info->dev_name, "mlx4_port%d", port); info->port_attr.attr.name = info->dev_name; if (mlx4_is_mfunc(dev)) info->port_attr.attr.mode = S_IRUGO; else { info->port_attr.attr.mode = S_IRUGO | S_IWUSR; info->port_attr.store = set_port_type; } info->port_attr.show = show_port_type; sysfs_attr_init(&info->port_attr.attr); err = device_create_file(&dev->persist->pdev->dev, &info->port_attr); if (err) { mlx4_err(dev, "Failed to create file for port %d\n", port); info->port = -1; } sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port); info->port_mtu_attr.attr.name = info->dev_mtu_name; if (mlx4_is_mfunc(dev)) info->port_mtu_attr.attr.mode = S_IRUGO; else { info->port_mtu_attr.attr.mode = S_IRUGO | S_IWUSR; info->port_mtu_attr.store = set_port_ib_mtu; } info->port_mtu_attr.show = show_port_ib_mtu; sysfs_attr_init(&info->port_mtu_attr.attr); err = device_create_file(&dev->persist->pdev->dev, &info->port_mtu_attr); if (err) { mlx4_err(dev, "Failed to create mtu file for port %d\n", port); device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); info->port = -1; } return err; } static void mlx4_cleanup_port_info(struct mlx4_port_info *info) { if (info->port < 0) return; device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); device_remove_file(&info->dev->persist->pdev->dev, &info->port_mtu_attr); #ifdef CONFIG_RFS_ACCEL free_irq_cpu_rmap(info->rmap); info->rmap = NULL; #endif } static int mlx4_init_steering(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int num_entries = dev->caps.num_ports; int i, j; priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL); if (!priv->steer) return -ENOMEM; for (i = 0; i < num_entries; i++) for (j = 0; j < MLX4_NUM_STEERS; j++) { INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]); INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]); } return 0; } static void mlx4_clear_steering(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_steer_index *entry, *tmp_entry; struct mlx4_promisc_qp *pqp, *tmp_pqp; int num_entries = dev->caps.num_ports; int i, j; for (i = 0; i < num_entries; i++) { for (j = 0; j < MLX4_NUM_STEERS; j++) { list_for_each_entry_safe(pqp, tmp_pqp, &priv->steer[i].promisc_qps[j], list) { list_del(&pqp->list); kfree(pqp); } list_for_each_entry_safe(entry, tmp_entry, &priv->steer[i].steer_entries[j], list) { list_del(&entry->list); list_for_each_entry_safe(pqp, tmp_pqp, &entry->duplicates, list) { list_del(&pqp->list); kfree(pqp); } kfree(entry); } } } kfree(priv->steer); } static int extended_func_num(struct pci_dev *pdev) { return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn); } #define MLX4_OWNER_BASE 0x8069c #define MLX4_OWNER_SIZE 4 static int mlx4_get_ownership(struct mlx4_dev *dev) { void __iomem *owner; u32 ret; if (pci_channel_offline(dev->persist->pdev)) return -EIO; owner = ioremap(pci_resource_start(dev->persist->pdev, 0) + MLX4_OWNER_BASE, MLX4_OWNER_SIZE); if (!owner) { mlx4_err(dev, "Failed to obtain ownership bit\n"); return -ENOMEM; } ret = readl(owner); iounmap(owner); return (int) !!ret; } static void mlx4_free_ownership(struct mlx4_dev *dev) { void __iomem *owner; if (pci_channel_offline(dev->persist->pdev)) return; owner = ioremap(pci_resource_start(dev->persist->pdev, 0) + MLX4_OWNER_BASE, MLX4_OWNER_SIZE); if (!owner) { mlx4_err(dev, "Failed to obtain ownership bit\n"); return; } writel(0, owner); msleep(1000); iounmap(owner); } #define SRIOV_VALID_STATE(flags) (!!((flags) & MLX4_FLAG_SRIOV) ==\ !!((flags) & MLX4_FLAG_MASTER)) static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev, u8 total_vfs, int existing_vfs, int reset_flow) { u64 dev_flags = dev->flags; int err = 0; if (reset_flow) { dev->dev_vfs = kcalloc(total_vfs, sizeof(*dev->dev_vfs), GFP_KERNEL); if (!dev->dev_vfs) goto free_mem; return dev_flags; } atomic_inc(&pf_loading); if (dev->flags & MLX4_FLAG_SRIOV) { if (existing_vfs != total_vfs) { mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n", existing_vfs, total_vfs); total_vfs = existing_vfs; } } dev->dev_vfs = kzalloc(total_vfs * sizeof(*dev->dev_vfs), GFP_KERNEL); if (NULL == dev->dev_vfs) { mlx4_err(dev, "Failed to allocate memory for VFs\n"); goto disable_sriov; } if (!(dev->flags & MLX4_FLAG_SRIOV)) { mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs); err = pci_enable_sriov(pdev, total_vfs); } if (err) { mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n", err); goto disable_sriov; } else { mlx4_warn(dev, "Running in master mode\n"); dev_flags |= MLX4_FLAG_SRIOV | MLX4_FLAG_MASTER; dev_flags &= ~MLX4_FLAG_SLAVE; dev->persist->num_vfs = total_vfs; } return dev_flags; disable_sriov: atomic_dec(&pf_loading); free_mem: dev->persist->num_vfs = 0; kfree(dev->dev_vfs); dev->dev_vfs = NULL; return dev_flags & ~MLX4_FLAG_MASTER; } enum { MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64 = -1, }; static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, int *nvfs) { int requested_vfs = nvfs[0] + nvfs[1] + nvfs[2]; /* Checking for 64 VFs as a limitation of CX2 */ if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_80_VFS) && requested_vfs >= 64) { mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n", requested_vfs); return MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64; } return 0; } static int mlx4_pci_enable_device(struct mlx4_dev *dev) { struct pci_dev *pdev = dev->persist->pdev; int err = 0; mutex_lock(&dev->persist->pci_status_mutex); if (dev->persist->pci_status == MLX4_PCI_STATUS_DISABLED) { err = pci_enable_device(pdev); if (!err) dev->persist->pci_status = MLX4_PCI_STATUS_ENABLED; } mutex_unlock(&dev->persist->pci_status_mutex); return err; } static void mlx4_pci_disable_device(struct mlx4_dev *dev) { struct pci_dev *pdev = dev->persist->pdev; mutex_lock(&dev->persist->pci_status_mutex); if (dev->persist->pci_status == MLX4_PCI_STATUS_ENABLED) { pci_disable_device(pdev); dev->persist->pci_status = MLX4_PCI_STATUS_DISABLED; } mutex_unlock(&dev->persist->pci_status_mutex); } static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data, int total_vfs, int *nvfs, struct mlx4_priv *priv, int reset_flow) { struct mlx4_dev *dev; unsigned sum = 0; int err; int port; int i; struct mlx4_dev_cap *dev_cap = NULL; int existing_vfs = 0; dev = &priv->dev; INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); mutex_init(&priv->port_mutex); mutex_init(&priv->bond_mutex); INIT_LIST_HEAD(&priv->pgdir_list); mutex_init(&priv->pgdir_mutex); spin_lock_init(&priv->cmd.context_lock); INIT_LIST_HEAD(&priv->bf_list); mutex_init(&priv->bf_mutex); dev->rev_id = pdev->revision; dev->numa_node = dev_to_node(&pdev->dev); /* Detect if this device is a virtual function */ if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { mlx4_warn(dev, "Detected virtual function - running in slave mode\n"); dev->flags |= MLX4_FLAG_SLAVE; } else { /* We reset the device and enable SRIOV only for physical * devices. Try to claim ownership on the device; * if already taken, skip -- do not allow multiple PFs */ err = mlx4_get_ownership(dev); if (err) { if (err < 0) return err; else { mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n"); return -EINVAL; } } atomic_set(&priv->opreq_count, 0); INIT_WORK(&priv->opreq_task, mlx4_opreq_action); /* * Now reset the HCA before we touch the PCI capabilities or * attempt a firmware command, since a boot ROM may have left * the HCA in an undefined state. */ err = mlx4_reset(dev); if (err) { mlx4_err(dev, "Failed to reset HCA, aborting\n"); goto err_sriov; } if (total_vfs) { dev->flags = MLX4_FLAG_MASTER; existing_vfs = pci_num_vf(pdev); if (existing_vfs) dev->flags |= MLX4_FLAG_SRIOV; dev->persist->num_vfs = total_vfs; } } /* on load remove any previous indication of internal error, * device is up. */ dev->persist->state = MLX4_DEVICE_STATE_UP; slave_start: err = mlx4_cmd_init(dev); if (err) { mlx4_err(dev, "Failed to init command interface, aborting\n"); goto err_sriov; } /* In slave functions, the communication channel must be initialized * before posting commands. Also, init num_slaves before calling * mlx4_init_hca */ if (mlx4_is_mfunc(dev)) { if (mlx4_is_master(dev)) { dev->num_slaves = MLX4_MAX_NUM_SLAVES; } else { dev->num_slaves = 0; err = mlx4_multi_func_init(dev); if (err) { mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n"); goto err_cmd; } } } err = mlx4_init_fw(dev); if (err) { mlx4_err(dev, "Failed to init fw, aborting.\n"); goto err_mfunc; } if (mlx4_is_master(dev)) { /* when we hit the goto slave_start below, dev_cap already initialized */ if (!dev_cap) { dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL); if (!dev_cap) { err = -ENOMEM; goto err_fw; } err = mlx4_QUERY_DEV_CAP(dev, dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); goto err_fw; } if (mlx4_check_dev_cap(dev, dev_cap, nvfs)) goto err_fw; if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) { u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs, existing_vfs, reset_flow); mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); dev->flags = dev_flags; if (!SRIOV_VALID_STATE(dev->flags)) { mlx4_err(dev, "Invalid SRIOV state\n"); goto err_sriov; } err = mlx4_reset(dev); if (err) { mlx4_err(dev, "Failed to reset HCA, aborting.\n"); goto err_sriov; } goto slave_start; } } else { /* Legacy mode FW requires SRIOV to be enabled before * doing QUERY_DEV_CAP, since max_eq's value is different if * SRIOV is enabled. */ memset(dev_cap, 0, sizeof(*dev_cap)); err = mlx4_QUERY_DEV_CAP(dev, dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); goto err_fw; } if (mlx4_check_dev_cap(dev, dev_cap, nvfs)) goto err_fw; } } err = mlx4_init_hca(dev); if (err) { if (err == -EACCES) { /* Not primary Physical function * Running in slave mode */ mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); /* We're not a PF */ if (dev->flags & MLX4_FLAG_SRIOV) { if (!existing_vfs) pci_disable_sriov(pdev); if (mlx4_is_master(dev) && !reset_flow) atomic_dec(&pf_loading); dev->flags &= ~MLX4_FLAG_SRIOV; } if (!mlx4_is_slave(dev)) mlx4_free_ownership(dev); dev->flags |= MLX4_FLAG_SLAVE; dev->flags &= ~MLX4_FLAG_MASTER; goto slave_start; } else goto err_fw; } if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) { u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs, existing_vfs, reset_flow); if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) { mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_VHCR); dev->flags = dev_flags; err = mlx4_cmd_init(dev); if (err) { /* Only VHCR is cleaned up, so could still * send FW commands */ mlx4_err(dev, "Failed to init VHCR command interface, aborting\n"); goto err_close; } } else { dev->flags = dev_flags; } if (!SRIOV_VALID_STATE(dev->flags)) { mlx4_err(dev, "Invalid SRIOV state\n"); goto err_close; } } /* check if the device is functioning at its maximum possible speed. * No return code for this call, just warn the user in case of PCI * express device capabilities are under-satisfied by the bus. */ if (!mlx4_is_slave(dev)) mlx4_check_pcie_caps(dev); /* In master functions, the communication channel must be initialized * after obtaining its address from fw */ if (mlx4_is_master(dev)) { if (dev->caps.num_ports < 2 && num_vfs_argc > 1) { err = -EINVAL; mlx4_err(dev, "Error: Trying to configure VFs on port 2, but HCA has only %d physical ports\n", dev->caps.num_ports); goto err_close; } memcpy(dev->persist->nvfs, nvfs, sizeof(dev->persist->nvfs)); for (i = 0; i < sizeof(dev->persist->nvfs)/ sizeof(dev->persist->nvfs[0]); i++) { unsigned j; for (j = 0; j < dev->persist->nvfs[i]; ++sum, ++j) { dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1; dev->dev_vfs[sum].n_ports = i < 2 ? 1 : dev->caps.num_ports; } } /* In master functions, the communication channel * must be initialized after obtaining its address from fw */ err = mlx4_multi_func_init(dev); if (err) { mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n"); goto err_close; } } err = mlx4_alloc_eq_table(dev); if (err) goto err_master_mfunc; bitmap_zero(priv->msix_ctl.pool_bm, MAX_MSIX); mutex_init(&priv->msix_ctl.pool_lock); mlx4_enable_msi_x(dev); if ((mlx4_is_mfunc(dev)) && !(dev->flags & MLX4_FLAG_MSI_X)) { err = -ENOSYS; mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n"); goto err_free_eq; } if (!mlx4_is_slave(dev)) { err = mlx4_init_steering(dev); if (err) goto err_disable_msix; } mlx4_init_quotas(dev); err = mlx4_setup_hca(dev); if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) && !mlx4_is_mfunc(dev)) { dev->flags &= ~MLX4_FLAG_MSI_X; dev->caps.num_comp_vectors = 1; pci_disable_msix(pdev); err = mlx4_setup_hca(dev); } if (err) goto err_steer; /* When PF resources are ready arm its comm channel to enable * getting commands */ if (mlx4_is_master(dev)) { err = mlx4_ARM_COMM_CHANNEL(dev); if (err) { mlx4_err(dev, " Failed to arm comm channel eq: %x\n", err); goto err_steer; } } for (port = 1; port <= dev->caps.num_ports; port++) { err = mlx4_init_port_info(dev, port); if (err) goto err_port; } priv->v2p.port1 = 1; priv->v2p.port2 = 2; err = mlx4_register_device(dev); if (err) goto err_port; mlx4_request_modules(dev); mlx4_sense_init(dev); mlx4_start_sense(dev); priv->removed = 0; if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow) atomic_dec(&pf_loading); kfree(dev_cap); return 0; err_port: for (--port; port >= 1; --port) mlx4_cleanup_port_info(&priv->port[port]); mlx4_cleanup_counters_table(dev); mlx4_cleanup_qp_table(dev); mlx4_cleanup_srq_table(dev); mlx4_cleanup_cq_table(dev); mlx4_cmd_use_polling(dev); mlx4_cleanup_eq_table(dev); mlx4_cleanup_mcg_table(dev); mlx4_cleanup_mr_table(dev); mlx4_cleanup_xrcd_table(dev); mlx4_cleanup_pd_table(dev); mlx4_cleanup_uar_table(dev); err_steer: if (!mlx4_is_slave(dev)) mlx4_clear_steering(dev); err_disable_msix: if (dev->flags & MLX4_FLAG_MSI_X) pci_disable_msix(pdev); err_free_eq: mlx4_free_eq_table(dev); err_master_mfunc: if (mlx4_is_master(dev)) { mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); mlx4_multi_func_cleanup(dev); } if (mlx4_is_slave(dev)) { kfree(dev->caps.qp0_qkey); kfree(dev->caps.qp0_tunnel); kfree(dev->caps.qp0_proxy); kfree(dev->caps.qp1_tunnel); kfree(dev->caps.qp1_proxy); } err_close: mlx4_close_hca(dev); err_fw: mlx4_close_fw(dev); err_mfunc: if (mlx4_is_slave(dev)) mlx4_multi_func_cleanup(dev); err_cmd: mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); err_sriov: if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs) { pci_disable_sriov(pdev); dev->flags &= ~MLX4_FLAG_SRIOV; } if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow) atomic_dec(&pf_loading); kfree(priv->dev.dev_vfs); if (!mlx4_is_slave(dev)) mlx4_free_ownership(dev); kfree(dev_cap); return err; } static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data, struct mlx4_priv *priv) { int err; int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; int prb_vf[MLX4_MAX_PORTS + 1] = {0, 0, 0}; const int param_map[MLX4_MAX_PORTS + 1][MLX4_MAX_PORTS + 1] = { {2, 0, 0}, {0, 1, 2}, {0, 1, 2} }; unsigned total_vfs = 0; unsigned int i; pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev)); err = mlx4_pci_enable_device(&priv->dev); if (err) { dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); return err; } /* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS * per port, we must limit the number of VFs to 63 (since their are * 128 MACs) */ for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && i < num_vfs_argc; total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) { nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i]; if (nvfs[i] < 0) { dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n"); err = -EINVAL; goto err_disable_pdev; } } for (i = 0; i < sizeof(prb_vf)/sizeof(prb_vf[0]) && i < probe_vfs_argc; i++) { prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i]; if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) { dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n"); err = -EINVAL; goto err_disable_pdev; } } if (total_vfs > MLX4_MAX_NUM_VF) { dev_err(&pdev->dev, "Requested more VF's (%d) than allowed by hw (%d)\n", total_vfs, MLX4_MAX_NUM_VF); err = -EINVAL; goto err_disable_pdev; } for (i = 0; i < MLX4_MAX_PORTS; i++) { if (nvfs[i] + nvfs[2] > MLX4_MAX_NUM_VF_P_PORT) { dev_err(&pdev->dev, "Requested more VF's (%d) for port (%d) than allowed by driver (%d)\n", nvfs[i] + nvfs[2], i + 1, MLX4_MAX_NUM_VF_P_PORT); err = -EINVAL; goto err_disable_pdev; } } /* Check for BARs. */ if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) && !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\n", pci_dev_data, (long)pci_resource_flags(pdev, 0)); err = -ENODEV; goto err_disable_pdev; } if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "Missing UAR, aborting\n"); err = -ENODEV; goto err_disable_pdev; } err = pci_request_regions(pdev, DRV_NAME); if (err) { dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); goto err_disable_pdev; } pci_set_master(pdev); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); goto err_release_regions; } } err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, aborting\n"); goto err_release_regions; } } /* Allow large DMA segments, up to the firmware limit of 1 GB */ dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024); /* Detect if this device is a virtual function */ if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { /* When acting as pf, we normally skip vfs unless explicitly * requested to probe them. */ if (total_vfs) { unsigned vfs_offset = 0; for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && vfs_offset + nvfs[i] < extended_func_num(pdev); vfs_offset += nvfs[i], i++) ; if (i == sizeof(nvfs)/sizeof(nvfs[0])) { err = -ENODEV; goto err_release_regions; } if ((extended_func_num(pdev) - vfs_offset) > prb_vf[i]) { dev_warn(&pdev->dev, "Skipping virtual function:%d\n", extended_func_num(pdev)); err = -ENODEV; goto err_release_regions; } } } err = mlx4_catas_init(&priv->dev); if (err) goto err_release_regions; err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 0); if (err) goto err_catas; return 0; err_catas: mlx4_catas_end(&priv->dev); err_release_regions: pci_release_regions(pdev); err_disable_pdev: mlx4_pci_disable_device(&priv->dev); pci_set_drvdata(pdev, NULL); return err; } static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { struct sysctl_ctx_list *ctx; struct sysctl_oid *node; struct sysctl_oid_list *node_list; struct mlx4_priv *priv; struct mlx4_dev *dev; int ret; printk_once(KERN_INFO "%s", mlx4_version); priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; dev = &priv->dev; dev->persist = kzalloc(sizeof(*dev->persist), GFP_KERNEL); if (!dev->persist) { kfree(priv); return -ENOMEM; } dev->persist->pdev = pdev; dev->persist->dev = dev; pci_set_drvdata(pdev, dev->persist); priv->pci_dev_data = id->driver_data; mutex_init(&dev->persist->device_state_mutex); mutex_init(&dev->persist->interface_state_mutex); mutex_init(&dev->persist->pci_status_mutex); ret = __mlx4_init_one(pdev, id->driver_data, priv); if (ret) { kfree(dev->persist); kfree(priv); return ret; } else { device_set_desc(pdev->dev.bsddev, mlx4_description); pci_save_state(pdev->dev.bsddev); } snprintf(dev->fw_str, sizeof(dev->fw_str), "%d.%d.%d", (int) (dev->caps.fw_ver >> 32), (int) (dev->caps.fw_ver >> 16) & 0xffff, (int) (dev->caps.fw_ver & 0xffff)); ctx = &dev->hw_ctx; sysctl_ctx_init(ctx); node = SYSCTL_ADD_NODE(ctx,SYSCTL_CHILDREN(pdev->dev.kobj.oidp), OID_AUTO, "hw" , CTLFLAG_RD, 0, "mlx4 dev hw information"); if (node != NULL) { node_list = SYSCTL_CHILDREN(node); SYSCTL_ADD_STRING(ctx, node_list, OID_AUTO, "fw_version", CTLFLAG_RD, dev->fw_str, 0, "Device firmware version"); SYSCTL_ADD_STRING(ctx, node_list, OID_AUTO, "board_id", CTLFLAG_RD, dev->board_id, 0, "Device board identifier"); } return ret; } static void mlx4_clean_dev(struct mlx4_dev *dev) { struct mlx4_dev_persistent *persist = dev->persist; struct mlx4_priv *priv = mlx4_priv(dev); unsigned long flags = (dev->flags & RESET_PERSIST_MASK_FLAGS); memset(priv, 0, sizeof(*priv)); priv->dev.persist = persist; priv->dev.flags = flags; } static void mlx4_unload_one(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int pci_dev_data; int p, i; if (priv->removed) return; /* saving current ports type for further use */ for (i = 0; i < dev->caps.num_ports; i++) { dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1]; dev->persist->curr_port_poss_type[i] = dev->caps. possible_type[i + 1]; } pci_dev_data = priv->pci_dev_data; mlx4_stop_sense(dev); mlx4_unregister_device(dev); for (p = 1; p <= dev->caps.num_ports; p++) { mlx4_cleanup_port_info(&priv->port[p]); mlx4_CLOSE_PORT(dev, p); } if (mlx4_is_master(dev)) mlx4_free_resource_tracker(dev, RES_TR_FREE_SLAVES_ONLY); mlx4_cleanup_default_counters(dev); if (!mlx4_is_slave(dev)) mlx4_cleanup_counters_table(dev); mlx4_cleanup_qp_table(dev); mlx4_cleanup_srq_table(dev); mlx4_cleanup_cq_table(dev); mlx4_cmd_use_polling(dev); mlx4_cleanup_eq_table(dev); mlx4_cleanup_mcg_table(dev); mlx4_cleanup_mr_table(dev); mlx4_cleanup_xrcd_table(dev); mlx4_cleanup_pd_table(dev); if (mlx4_is_master(dev)) mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); iounmap(priv->kar); mlx4_uar_free(dev, &priv->driver_uar); mlx4_cleanup_uar_table(dev); if (!mlx4_is_slave(dev)) mlx4_clear_steering(dev); mlx4_free_eq_table(dev); if (mlx4_is_master(dev)) mlx4_multi_func_cleanup(dev); mlx4_close_hca(dev); mlx4_close_fw(dev); if (mlx4_is_slave(dev)) mlx4_multi_func_cleanup(dev); mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); if (dev->flags & MLX4_FLAG_MSI_X) pci_disable_msix(pdev); if (!mlx4_is_slave(dev)) mlx4_free_ownership(dev); kfree(dev->caps.qp0_qkey); kfree(dev->caps.qp0_tunnel); kfree(dev->caps.qp0_proxy); kfree(dev->caps.qp1_tunnel); kfree(dev->caps.qp1_proxy); kfree(dev->dev_vfs); mlx4_clean_dev(dev); priv->pci_dev_data = pci_dev_data; priv->removed = 1; } static void mlx4_remove_one(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int active_vfs = 0; mutex_lock(&persist->interface_state_mutex); persist->interface_state |= MLX4_INTERFACE_STATE_DELETION; mutex_unlock(&persist->interface_state_mutex); /* * Clear the device description to avoid use after free, * because the bsddev is not destroyed when this module is * unloaded: */ device_set_desc(pdev->dev.bsddev, NULL); /* Disabling SR-IOV is not allowed while there are active vf's */ if (mlx4_is_master(dev) && dev->flags & MLX4_FLAG_SRIOV) { active_vfs = mlx4_how_many_lives_vf(dev); if (active_vfs) { pr_warn("Removing PF when there are active VF's !!\n"); pr_warn("Will not disable SR-IOV.\n"); } } /* device marked to be under deletion running now without the lock * letting other tasks to be terminated */ if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); else mlx4_info(dev, "%s: interface is down\n", __func__); mlx4_catas_end(dev); if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) { mlx4_warn(dev, "Disabling SR-IOV\n"); pci_disable_sriov(pdev); } pci_release_regions(pdev); pci_disable_device(pdev); kfree(dev->persist); kfree(priv); pci_set_drvdata(pdev, NULL); } static int restore_current_port_types(struct mlx4_dev *dev, enum mlx4_port_type *types, enum mlx4_port_type *poss_types) { struct mlx4_priv *priv = mlx4_priv(dev); int err, i; mlx4_stop_sense(dev); mutex_lock(&priv->port_mutex); for (i = 0; i < dev->caps.num_ports; i++) dev->caps.possible_type[i + 1] = poss_types[i]; err = mlx4_change_port_types(dev, types); mutex_unlock(&priv->port_mutex); mlx4_start_sense(dev); return err; } int mlx4_restart_one(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; int pci_dev_data, err, total_vfs; pci_dev_data = priv->pci_dev_data; total_vfs = dev->persist->num_vfs; memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); mlx4_unload_one(pdev); err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 1); if (err) { mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n", __func__, pci_name(pdev), err); return err; } err = restore_current_port_types(dev, dev->persist->curr_port_type, dev->persist->curr_port_poss_type); if (err) mlx4_err(dev, "could not restore original port types (%d)\n", err); return err; } static DEFINE_PCI_DEVICE_TABLE(mlx4_pci_table) = { /* MT25408 "Hermon" SDR */ { PCI_VDEVICE(MELLANOX, 0x6340), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" DDR */ { PCI_VDEVICE(MELLANOX, 0x634a), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" QDR */ { PCI_VDEVICE(MELLANOX, 0x6354), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" DDR PCIe gen2 */ { PCI_VDEVICE(MELLANOX, 0x6732), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" QDR PCIe gen2 */ { PCI_VDEVICE(MELLANOX, 0x673c), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" EN 10GigE */ { PCI_VDEVICE(MELLANOX, 0x6368), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ { PCI_VDEVICE(MELLANOX, 0x6750), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */ { PCI_VDEVICE(MELLANOX, 0x6372), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ { PCI_VDEVICE(MELLANOX, 0x675a), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT26468 ConnectX EN 10GigE PCIe gen2*/ { PCI_VDEVICE(MELLANOX, 0x6764), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */ { PCI_VDEVICE(MELLANOX, 0x6746), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT26478 ConnectX2 40GigE PCIe gen2 */ { PCI_VDEVICE(MELLANOX, 0x676e), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, /* MT25400 Family [ConnectX-2 Virtual Function] */ { PCI_VDEVICE(MELLANOX, 0x1002), .driver_data = MLX4_PCI_DEV_IS_VF }, /* MT27500 Family [ConnectX-3] */ { PCI_VDEVICE(MELLANOX, 0x1003) }, /* MT27500 Family [ConnectX-3 Virtual Function] */ { PCI_VDEVICE(MELLANOX, 0x1004), .driver_data = MLX4_PCI_DEV_IS_VF }, { PCI_VDEVICE(MELLANOX, 0x1005) }, /* MT27510 Family */ { PCI_VDEVICE(MELLANOX, 0x1006) }, /* MT27511 Family */ { PCI_VDEVICE(MELLANOX, 0x1007) }, /* MT27520 Family */ { PCI_VDEVICE(MELLANOX, 0x1008) }, /* MT27521 Family */ { PCI_VDEVICE(MELLANOX, 0x1009) }, /* MT27530 Family */ { PCI_VDEVICE(MELLANOX, 0x100a) }, /* MT27531 Family */ { PCI_VDEVICE(MELLANOX, 0x100b) }, /* MT27540 Family */ { PCI_VDEVICE(MELLANOX, 0x100c) }, /* MT27541 Family */ { PCI_VDEVICE(MELLANOX, 0x100d) }, /* MT27550 Family */ { PCI_VDEVICE(MELLANOX, 0x100e) }, /* MT27551 Family */ { PCI_VDEVICE(MELLANOX, 0x100f) }, /* MT27560 Family */ { PCI_VDEVICE(MELLANOX, 0x1010) }, /* MT27561 Family */ { 0, } }; MODULE_DEVICE_TABLE(pci, mlx4_pci_table); static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n"); mlx4_enter_error_state(persist); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); if (state == pci_channel_io_perm_failure) return PCI_ERS_RESULT_DISCONNECT; mlx4_pci_disable_device(persist->dev); return PCI_ERS_RESULT_NEED_RESET; } static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; int err; mlx4_err(dev, "mlx4_pci_slot_reset was called\n"); err = mlx4_pci_enable_device(dev); if (err) { mlx4_err(dev, "Can not re-enable device, err=%d\n", err); return PCI_ERS_RESULT_DISCONNECT; } pci_set_master(pdev); return PCI_ERS_RESULT_RECOVERED; } static void mlx4_pci_resume(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; int total_vfs; int err; mlx4_err(dev, "%s was called\n", __func__); total_vfs = dev->persist->num_vfs; memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); mutex_lock(&persist->interface_state_mutex); if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) { err = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs, nvfs, priv, 1); if (err) { mlx4_err(dev, "%s: mlx4_load_one failed, err=%d\n", __func__, err); goto end; } err = restore_current_port_types(dev, dev->persist-> curr_port_type, dev->persist-> curr_port_poss_type); if (err) mlx4_err(dev, "could not restore original port types (%d)\n", err); } end: mutex_unlock(&persist->interface_state_mutex); } static void mlx4_shutdown(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); mlx4_info(persist->dev, "mlx4_shutdown was called\n"); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); } static const struct pci_error_handlers mlx4_err_handler = { .error_detected = mlx4_pci_err_detected, .slot_reset = mlx4_pci_slot_reset, .resume = mlx4_pci_resume, }; static struct pci_driver mlx4_driver = { .name = DRV_NAME, .id_table = mlx4_pci_table, .probe = mlx4_init_one, .shutdown = mlx4_shutdown, .remove = mlx4_remove_one, .err_handler = &mlx4_err_handler, }; static int __init mlx4_verify_params(void) { if ((log_num_mac < 0) || (log_num_mac > 7)) { pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac); return -1; } if (log_num_vlan != 0) pr_warn("mlx4_core: log_num_vlan - obsolete module param, using %d\n", MLX4_LOG_NUM_VLANS); if (use_prio != 0) pr_warn("mlx4_core: use_prio - obsolete module param, ignored\n"); if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 7)) { pr_warn("mlx4_core: bad log_mtts_per_seg: %d\n", log_mtts_per_seg); return -1; } /* Check if module param for ports type has legal combination */ if (port_type_array[0] == false && port_type_array[1] == true) { pr_warn("Module parameter configuration ETH/IB is not supported. Switching to default configuration IB/IB\n"); port_type_array[0] = true; } if (mlx4_log_num_mgm_entry_size < -7 || (mlx4_log_num_mgm_entry_size > 0 && (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE || mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE))) { pr_warn("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not in legal range (-7..0 or %d..%d)\n", mlx4_log_num_mgm_entry_size, MLX4_MIN_MGM_LOG_ENTRY_SIZE, MLX4_MAX_MGM_LOG_ENTRY_SIZE); return -1; } return 0; } static int __init mlx4_init(void) { int ret; if (mlx4_verify_params()) return -EINVAL; mlx4_wq = create_singlethread_workqueue("mlx4"); if (!mlx4_wq) return -ENOMEM; ret = pci_register_driver(&mlx4_driver); if (ret < 0) destroy_workqueue(mlx4_wq); return ret < 0 ? ret : 0; } static void __exit mlx4_cleanup(void) { pci_unregister_driver(&mlx4_driver); destroy_workqueue(mlx4_wq); } module_init(mlx4_init); module_exit(mlx4_cleanup); static int mlx4_evhand(module_t mod, int event, void *arg) { return (0); } static moduledata_t mlx4_mod = { .name = "mlx4", .evhand = mlx4_evhand, }; MODULE_VERSION(mlx4, 1); DECLARE_MODULE(mlx4, mlx4_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); MODULE_DEPEND(mlx4, linuxkpi, 1, 1, 1); Index: projects/import-googletest-1.8.1/sys/dev/pms/RefTisa/tisa/sassata/sas/ini/itdio.c =================================================================== --- projects/import-googletest-1.8.1/sys/dev/pms/RefTisa/tisa/sassata/sas/ini/itdio.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/dev/pms/RefTisa/tisa/sassata/sas/ini/itdio.c (revision 345026) @@ -1,1893 +1,1895 @@ /******************************************************************************* *Copyright (c) 2014 PMC-Sierra, Inc. All rights reserved. * *Redistribution and use in source and binary forms, with or without modification, are permitted provided *that the following conditions are met: *1. Redistributions of source code must retain the above copyright notice, this list of conditions and the *following disclaimer. *2. Redistributions in binary form must reproduce the above copyright notice, *this list of conditions and the following disclaimer in the documentation and/or other materials provided *with the distribution. * *THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED *WARRANTIES,INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS *FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT *NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR *BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT *LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS *SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE ********************************************************************************/ /*******************************************************************************/ /** \file * * * This file contains initiator IO related functions in TD layer * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef FDS_SM #include #include #include #endif #ifdef FDS_DM #include #include #include #endif #include #include #include #ifdef INITIATOR_DRIVER #include #include #include #endif #ifdef TARGET_DRIVER #include #include #include #endif #include #include /***************************************************************************** *! \brief tiINIIOStart * * Purpose: This routine is called to initiate a new SCSI request. * * \param tiRoot: Pointer to initiator driver/port instance. * \param tiIORequest: Pointer to the I/O request context for this I/O. * \param tiDeviceHandle: Pointer to device handle for this I/O. * \param tiScsiRequest: Pointer to the SCSI-3 I/O request and SGL list. * \param tiRequestBody: Pointer to the OS Specific module allocated storage * to be used by the TD layer for executing this I/O. * \param interruptContext: The interrupt context within which this function * is called. * \return: * * tiSuccess: I/O request successfully initiated. * tiBusy: No resources available, try again later. * tiIONoDevice: Invalid device handle. * tiError: Other errors that prevent the I/O request to be started. * * *****************************************************************************/ osGLOBAL bit32 tiINIIOStart( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiScsiInitiatorRequest_t *tiScsiRequest, void *tiRequestBody, bit32 interruptContext ) { tdsaRoot_t *tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; tdsaContext_t *tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; itdsaIni_t *Initiator = (itdsaIni_t *)tdsaAllShared->itdsaIni; tdsaDeviceData_t *oneDeviceData; agsaRoot_t *agRoot = agNULL; agsaIORequest_t *agIORequest = agNULL; agsaDevHandle_t *agDevHandle = agNULL; bit32 agRequestType; agsaSASRequestBody_t *agSASRequestBody = agNULL; bit32 tiStatus = tiError; bit32 saStatus = AGSA_RC_FAILURE; tdIORequestBody_t *tdIORequestBody; agsaSSPInitiatorRequest_t *agSSPInitiatorRequest; #ifdef REMOVED /* only for debugging */ bit32 i; #endif #ifdef SATA_ENABLE #ifndef FDS_SM satIOContext_t *satIOContext; #endif #endif #ifdef FDS_SM smRoot_t *smRoot = &(tdsaAllShared->smRoot); smIORequest_t *smIORequest; smDeviceHandle_t *smDeviceHandle; smScsiInitiatorRequest_t *smSCSIRequest; #endif TDSA_INP_ENTER(tiRoot); TI_DBG6(("tiINIIOStart: start\n")); TI_DBG6(("tiINIIOStart:: ******* tdsaRoot %p tdsaAllShared %p \n", tdsaRoot,tdsaAllShared)); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; TI_DBG6(("tiINIIOStart: onedevicedata %p\n", oneDeviceData)); if(oneDeviceData == agNULL) { TI_DBG1(("tiINIIOStart: tiDeviceHandle=%p DeviceData is NULL\n", tiDeviceHandle )); tiStatus = tiIONoDevice; goto ext; } /* for hotplug */ if (oneDeviceData->valid != agTRUE || oneDeviceData->registered != agTRUE || oneDeviceData->tdPortContext == agNULL ) { TI_DBG1(("tiINIIOStart: tiDeviceHandle=%p did %d DeviceData was removed\n", tiDeviceHandle, oneDeviceData->id)); TI_DBG6(("tiINIIOStart: device AddrHi 0x%08x AddrLo 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi, oneDeviceData->SASAddressID.sasAddressLo)); // for debugging tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging1Completed; TI_DBG6(("tiINIIOStart: IOCompletionFunc %p\n", tdIORequestBody->IOCompletionFunc)); tiStatus = tiIONoDevice; goto ext; } #if 1 if (tiIORequest->osData == agNULL) { TI_DBG1(("tiINIIOStart: tiIORequest->osData is NULL, wrong\n")); } #endif /* starting IO with SAS device */ if (oneDeviceData->DeviceType == TD_SAS_DEVICE) { TI_DBG6(("tiINIIOStart: calling saSSPStart\n")); agRoot = oneDeviceData->agRoot; agDevHandle = oneDeviceData->agDevHandle; /* OS layer has tdlayer data structure pointer in tdIORequestBody_t tdIOReqBody; in ccb_t in agtiapi.h */ tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t)); /* let's initialize tdIOrequestBody */ /* initialize callback */ tdIORequestBody->IOCompletionFunc = itdssIOCompleted; /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* initialize expDataLength */ tdIORequestBody->IOType.InitiatorRegIO.expDataLength = tiScsiRequest->scsiCmnd.expDataLength; tdIORequestBody->IOType.InitiatorRegIO.sglVirtualAddr = tiScsiRequest->sglVirtualAddr; /* initializes "agsaSgl_t agSgl" of "agsaDifSSPInitiatorRequest_t" */ tiStatus = itdssIOPrepareSGL( tiRoot, tdIORequestBody, &tiScsiRequest->agSgl1, tiScsiRequest->sglVirtualAddr ); if (tiStatus != tiSuccess) { TI_DBG1(("tiINIIOStart: can't get SGL\n")); goto ext; } /* initialize agIORequest */ agIORequest = &(tdIORequestBody->agIORequest); agIORequest->osData = (void *) tdIORequestBody; agIORequest->sdkData = agNULL; /* LL takes care of this */ /* initialize tdIORequestBody_t tdIORequestBody -> agSASRequestBody */ agSASRequestBody = &(tdIORequestBody->transport.SAS.agSASRequestBody); agSSPInitiatorRequest = &(agSASRequestBody->sspInitiatorReq); agSSPInitiatorRequest->flag = 0; /* copy cdb bytes */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.cdb, tiScsiRequest->scsiCmnd.cdb, 16); /* copy lun field */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.lun, tiScsiRequest->scsiCmnd.lun.lun, 8); /* setting the data length */ agSSPInitiatorRequest->dataLength = tiScsiRequest->scsiCmnd.expDataLength; TI_DBG6(("tiINIIOStart: tiScsiRequest->scsiCmnd.expDataLength %d\n", tiScsiRequest->scsiCmnd.expDataLength)); agSSPInitiatorRequest->firstBurstSize = 0; /* process taskattribute */ if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_SIMPLE) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_SIMPLE; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_ORDERED) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_ORDERED; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_HEAD_OF_QUEUE) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_HEAD_OF_QUEUE; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_ACA) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_ACA; } if (tiScsiRequest->dataDirection == tiDirectionIn) { agRequestType = AGSA_SSP_INIT_READ; TI_DBG6(("tiINIIOStart: READ\n")); } else if (tiScsiRequest->dataDirection == tiDirectionOut) { agRequestType = AGSA_SSP_INIT_WRITE; TI_DBG6(("tiINIIOStart: WRITE\n")); } else { agRequestType = AGSA_REQ_TYPE_UNKNOWN; TI_DBG1(("tiINIIOStart: unknown data direction\n")); } tdIORequestBody->agRequestType = agRequestType; TI_DBG6(("tiINIIOStart: device AddrHi 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi)); TI_DBG6(("tiINIIOStart: device AddrLo 0x%08x\n", oneDeviceData->SASAddressID.sasAddressLo)); /* for debugging */ if (tdIORequestBody->IOCompletionFunc == agNULL) { TI_DBG1(("tiINIIOStart: Error!!!! IOCompletionFunc is NULL\n")); } saStatus = saSSPStart(agRoot, agIORequest, tdsaRotateQnumber(tiRoot, oneDeviceData), agDevHandle, agRequestType, agSASRequestBody, agNULL, &ossaSSPCompleted); tdIORequestBody->ioStarted = agTRUE; tdIORequestBody->ioCompleted = agFALSE; tdIORequestBody->reTries = 0; if (saStatus == AGSA_RC_SUCCESS) { Initiator->NumIOsActive++; tiStatus = tiSuccess; } else { tdIORequestBody->ioStarted = agFALSE; tdIORequestBody->ioCompleted = agTRUE; if (saStatus == AGSA_RC_BUSY) { TI_DBG4(("tiINIIOStart: saSSPStart busy\n")); tiStatus = tiBusy; } else { tiStatus = tiError; } goto ext; } } #ifdef FDS_SM else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { TI_DBG5(("tiINIIOStart: calling satIOStart\n")); TI_DBG5(("tiINIIOStart: onedevicedata did %d\n", oneDeviceData->id)); tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t)); /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; tdIORequestBody->superIOFlag = agFALSE; tiIORequest->tdData = tdIORequestBody; tdIORequestBody->tiIORequest = tiIORequest; smIORequest = (smIORequest_t *)&(tdIORequestBody->smIORequest); smIORequest->tdData = tdIORequestBody; smDeviceHandle = (smDeviceHandle_t *)&(oneDeviceData->smDeviceHandle); smDeviceHandle->tdData = oneDeviceData; smSCSIRequest = (smScsiInitiatorRequest_t *)&(tdIORequestBody->SM.smSCSIRequest); osti_memcpy(smSCSIRequest, tiScsiRequest, sizeof(smScsiInitiatorRequest_t)); tiStatus = smIOStart(smRoot, smIORequest, smDeviceHandle, smSCSIRequest, interruptContext); /* osGLOBAL bit32 smIOStart( smRoot_t *smRoot, smIORequest_t *smIORequest, smDeviceHandle_t *smDeviceHandle, smScsiInitiatorRequest_t *smSCSIRequest, bit32 interruptContext ) */ } #else else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { TI_DBG5(("tiINIIOStart: calling satIOStart\n")); TI_DBG5(("tiINIIOStart: onedevicedata did %d\n", oneDeviceData->id)); #ifdef SATA_ENABLE tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t)); /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging2Completed; satIOContext = &(tdIORequestBody->transport.SATA.satIOContext); /* * Need to initialize all the fields within satIOContext except * reqType and satCompleteCB which will be set in sat.c depending on cmd. */ tdIORequestBody->transport.SATA.tiSenseData.senseData = agNULL; tdIORequestBody->transport.SATA.tiSenseData.senseLen = 0; satIOContext->pSatDevData = &oneDeviceData->satDevData; satIOContext->pFis = &tdIORequestBody->transport.SATA.agSATARequestBody.fis.fisRegHostToDev; satIOContext->pScsiCmnd = &tiScsiRequest->scsiCmnd; satIOContext->pSense = &tdIORequestBody->transport.SATA.sensePayload; satIOContext->pTiSenseData = &tdIORequestBody->transport.SATA.tiSenseData; satIOContext->pTiSenseData->senseData = satIOContext->pSense; /* satIOContext->pSense = (scsiRspSense_t *)satIOContext->pTiSenseData->senseData; */ satIOContext->tiRequestBody = tiRequestBody; satIOContext->interruptContext = interruptContext; satIOContext->ptiDeviceHandle = tiDeviceHandle; satIOContext->tiScsiXchg = tiScsiRequest; satIOContext->satIntIoContext = agNULL; satIOContext->satOrgIOContext = agNULL; /* satIOContext->tiIORequest = tiIORequest; */ /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* followings are used only for internal IO */ satIOContext->currentLBA = 0; satIOContext->OrgTL = 0; TI_DBG5(("tiINIIOStart: pSatDevData=%p\n", satIOContext->pSatDevData )); tiStatus = satIOStart( tiRoot, tiIORequest, tiDeviceHandle, tiScsiRequest, satIOContext); goto ext; #endif } #endif /* else of FDS_SM */ else { tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging3Completed; TI_DBG1(("tiINIIOStart: wrong unspported Device %d\n", oneDeviceData->DeviceType)); /* error. unsupported IO */ } ext: TDSA_INP_LEAVE(tiRoot); return tiStatus; } #ifdef FAST_IO_TEST osGLOBAL bit32 tiINIFastIOSend(void *ioh) { bit32 saStatus, tiStatus; saStatus = saFastSSPSend(ioh); if (saStatus == AGSA_RC_SUCCESS) tiStatus = tiSuccess; else tiStatus = tiError; return tiStatus; } osGLOBAL bit32 tiINIFastIOCancel(void *ioh) { bit32 saStatus, tiStatus; saStatus = saFastSSPCancel(ioh); if (saStatus == AGSA_RC_SUCCESS) tiStatus = tiSuccess; else tiStatus = tiError; return tiStatus; } osGLOBAL void* tiINIFastIOPrepare( tiRoot_t *tiRoot, void *ioHandle, agsaFastCommand_t *fc) { tdsaDeviceData_t *oneDeviceData; tiDeviceHandle_t *tiDeviceHandle = fc->devHandle; bit32 taskAttribute = fc->taskAttribute; void *ioh = ioHandle; TDSA_INP_ENTER(tiRoot); TI_DBG6(("tiINIFastIOPrepare: enter\n")); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; if(oneDeviceData == agNULL) { TI_DBG1(("tiINIFastIOPrepare: tiDeviceHandle=%p DeviceData is NULL\n", tiDeviceHandle)); ioHandle = 0; TD_ASSERT((0), ""); goto ext; } TI_DBG6(("tiINIFastIOPrepare: onedevicedata %p\n", oneDeviceData)); /* starting IO with SAS device */ if (oneDeviceData->DeviceType != TD_SAS_DEVICE) { TI_DBG1(("tiINISuperIOSend: wrong Device %d\n", oneDeviceData->DeviceType)); /* error: unsupported IO */ ioHandle = 0; TD_ASSERT((0), ""); goto ext; } fc->agRoot = oneDeviceData->agRoot; TD_ASSERT((NULL != fc->agRoot), ""); fc->devHandle = oneDeviceData->agDevHandle; TD_ASSERT((NULL != fc->devHandle), ""); fc->safb->oneDeviceData = oneDeviceData; /* process taskattribute */ switch (taskAttribute) { case TASK_SIMPLE: fc->taskAttribute = TD_TASK_SIMPLE; break; case TASK_ORDERED: fc->taskAttribute = TD_TASK_ORDERED; break; case TASK_HEAD_OF_QUEUE: fc->taskAttribute = TD_TASK_HEAD_OF_QUEUE; break; case TASK_ACA: fc->taskAttribute = TD_TASK_ACA; break; /* compile out for "iniload" */ } TI_DBG3(("tiINIFastIOPrepare: data direction: %x\n", fc->agRequestType)); TI_DBG6(("tiINIFastIOPrepare: device AddrHi/Lo 0x%08x / 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi, oneDeviceData->SASAddressID.sasAddressLo)); fc->queueNum = tdsaRotateQnumber(tiRoot, oneDeviceData); ioHandle = saFastSSPPrepare(ioHandle, fc, ossaFastSSPCompleted, fc->safb); if (!ioHandle) { TI_DBG1(("tiINIFastIOPrepare: saSuperSSPSend error\n")); TD_ASSERT((0), ""); //goto ext; } ext: if (ioh && !ioHandle) { saFastSSPCancel(ioh); } TI_DBG6(("tiINIFastIOPrepare: leave\n")); TDSA_INP_LEAVE(tiRoot); return ioHandle; } /* tiINIFastIOPrepare */ #endif /***************************************************************************** * * tiINIIOStartDif * * Purpose: This routine is called to initiate a new SCSI request with * DIF enable. * * Parameters: * tiRoot: Pointer to initiator driver/port instance. * tiIORequest: Pointer to the I/O request context for this I/O. * tiDeviceHandle: Pointer to device handle for this I/O. * tiScsiRequest: Pointer to the SCSI-3 I/O request and SGL list. * tiRequestBody: Pointer to the OS Specific module allocated storage * to be used by the TD layer for executing this I/O. * interruptContext: The interrupt context within which this function * is called. * difOption: DIF option. * * Return: * * tiSuccess: I/O request successfully initiated. * tiBusy: No resources available, try again later. * tiIONoDevice: Invalid device handle. * tiError: Other errors that prevent the I/O request to be started. * * *****************************************************************************/ osGLOBAL bit32 tiINIIOStartDif( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiScsiInitiatorRequest_t *tiScsiRequest, void *tiRequestBody, bit32 interruptContext, tiDif_t *difOption ) { /* This function was never used by SAS/SATA. Use tiINISuperIOStart() instead. */ return tiBusy; } /***************************************************************************** * * tiINISuperIOStart * * Purpose: This routine is called to initiate a new SCSI request. * * Parameters: * tiRoot: Pointer to initiator driver/port instance. * tiIORequest: Pointer to the I/O request context for this I/O. * tiDeviceHandle: Pointer to device handle for this I/O. * tiScsiRequest: Pointer to the SCSI-3 I/O request and SGL list. * tiRequestBody: Pointer to the OS Specific module allocated storage * to be used by the TD layer for executing this I/O. * interruptContext: The interrupt context within which this function * is called. * Return: * * tiSuccess: I/O request successfully initiated. * tiBusy: No resources available, try again later. * tiIONoDevice: Invalid device handle. * tiError: Other errors that prevent the I/O request to be started. * * *****************************************************************************/ osGLOBAL bit32 tiINISuperIOStart( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiSuperScsiInitiatorRequest_t *tiScsiRequest, void *tiRequestBody, bit32 interruptContext ) { tdsaRoot_t *tdsaRoot = agNULL; tdsaContext_t *tdsaAllShared = agNULL; itdsaIni_t *Initiator = agNULL; tdsaDeviceData_t *oneDeviceData = agNULL; tdIORequestBody_t *tdIORequestBody = agNULL; agsaSSPInitiatorRequest_t *agSSPInitiatorRequest = agNULL; agsaRoot_t *agRoot = agNULL; agsaIORequest_t *agIORequest = agNULL; agsaDevHandle_t *agDevHandle = agNULL; agsaSASRequestBody_t *agSASRequestBody = agNULL; bit32 tiStatus = tiError; bit32 saStatus = AGSA_RC_FAILURE; bit32 adjusted_length = 0; bit32 agRequestType = 0; agBOOLEAN needPlusDataLenAdjustment = agFALSE; agBOOLEAN needMinusDataLenAdjustment = agFALSE; #ifdef SATA_ENABLE #ifndef FDS_SM satIOContext_t *satIOContext; #endif #endif #ifdef FDS_SM smRoot_t *smRoot; smIORequest_t *smIORequest; smDeviceHandle_t *smDeviceHandle; smSuperScsiInitiatorRequest_t *smSuperSCSIRequest; #endif #ifdef CCBUILD_INDIRECT_CDB agsaSSPInitiatorRequestIndirect_t *agSSPInitiatorIndRequest = agNULL; #endif TD_ASSERT(tiRoot , "tiRoot"); TD_ASSERT(tiIORequest, "tiIORequest"); TD_ASSERT(tiDeviceHandle, "tiDeviceHandle"); TD_ASSERT(tiRequestBody, "tiRequestBody"); TD_ASSERT(tiRoot->tdData, "tiRoot->tdData"); TD_ASSERT(tiDeviceHandle, "tiDeviceHandle"); tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; TD_ASSERT(tdsaRoot, "tdsaRoot"); tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; TD_ASSERT(tdsaAllShared, "tdsaAllShared"); Initiator = (itdsaIni_t *)tdsaAllShared->itdsaIni; TD_ASSERT(Initiator, "Initiator"); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; TD_ASSERT(oneDeviceData, "oneDeviceData"); #ifdef FDS_SM smRoot = &(tdsaAllShared->smRoot); TD_ASSERT(smRoot , "smRoot"); #endif TI_DBG6(("tiINISuperIOStart: start\n")); TI_DBG6(("tiINISuperIOStart:: ******* tdsaRoot %p tdsaAllShared %p \n", tdsaRoot,tdsaAllShared)); TI_DBG6(("tiINISuperIOStart: onedevicedata %p\n", oneDeviceData)); if (oneDeviceData == agNULL) { TI_DBG1(("tiINISuperIOStart: tiDeviceHandle=%p DeviceData is NULL\n", tiDeviceHandle )); return tiIONoDevice; } /* for hotplug */ if (oneDeviceData->valid != agTRUE || oneDeviceData->registered != agTRUE || oneDeviceData->tdPortContext == agNULL ) { TI_DBG1(("tiINISuperIOStart: tiDeviceHandle=%p did %d DeviceData was removed\n", tiDeviceHandle, oneDeviceData->id)); TI_DBG6(("tiINISuperIOStart: device AddrHi 0x%08x AddrLo 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi, oneDeviceData->SASAddressID.sasAddressLo)); // for debugging tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging1Completed; TI_DBG6(("tiINISuperIOStart: IOCompletionFunc %p\n", tdIORequestBody->IOCompletionFunc)); return tiIONoDevice; } #ifdef DBG if (tiIORequest->osData == agNULL) { TI_DBG1(("tiINISuperIOStart: tiIORequest->osData is NULL, wrong\n")); return tiError; } #endif /* starting IO with SAS device */ if (oneDeviceData->DeviceType == TD_SAS_DEVICE) { TI_DBG3(("tiINISuperIOStart: calling saSSPStart\n")); agRoot = oneDeviceData->agRoot; agDevHandle = oneDeviceData->agDevHandle; /* OS layer has tdlayer data structure pointer in tdIORequestBody_t tdIOReqBody; in ccb_t in agtiapi.h */ tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ /*the tdIORequestBody has been initialized in HwBuildIo routine */ /*osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t));*/ /* let's initialize tdIOrequestBody */ /* initialize callback */ tdIORequestBody->IOCompletionFunc = itdssIOCompleted; /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* initialize expDataLength */ tdIORequestBody->IOType.InitiatorRegIO.expDataLength = tiScsiRequest->scsiCmnd.expDataLength; tdIORequestBody->IOType.InitiatorRegIO.sglVirtualAddr = tiScsiRequest->sglVirtualAddr; /* initialize agIORequest */ agIORequest = &(tdIORequestBody->agIORequest); agIORequest->osData = (void *) tdIORequestBody; /* initialize tdIORequestBody_t tdIORequestBody -> agSASRequestBody */ agSASRequestBody = &(tdIORequestBody->transport.SAS.agSASRequestBody); agSSPInitiatorRequest = &(agSASRequestBody->sspInitiatorReq); agSSPInitiatorRequest->flag = 0; if (tiScsiRequest->flags & TI_SCSI_INITIATOR_ENCRYPT) { TI_DBG3(("tiINISuperIOStart: TI_SCSI_INITIATOR_ENCRYPT\n")); /* Copy all of the relevant encrypt information */ agSSPInitiatorRequest->flag |= AGSA_SAS_ENABLE_ENCRYPTION; TD_ASSERT( sizeof(tiEncrypt_t) == sizeof(agsaEncrypt_t) , "sizeof(tiEncrypt_t) == sizeof(agsaEncrypt_t)"); osti_memcpy(&agSSPInitiatorRequest->encrypt, &tiScsiRequest->Encrypt, sizeof(agsaEncrypt_t)); } if ((tiScsiRequest->flags & TI_SCSI_INITIATOR_DIF) && (tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_READ_10 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_WRITE_10 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_WRITE_6 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_READ_6 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_READ_12 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_WRITE_12 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_WRITE_16 || tiScsiRequest->scsiCmnd.cdb[0] == SCSIOPC_READ_16 )) { TI_DBG3(("tiINISuperIOStart: TI_SCSI_INITIATOR_DIF\n")); /* Copy all of the relevant DIF information */ agSSPInitiatorRequest->flag |= AGSA_SAS_ENABLE_DIF; osti_memcpy(&agSSPInitiatorRequest->dif, &tiScsiRequest->Dif, sizeof(agsaDif_t)); /* Check if need to adjust dataLength. */ switch (tiScsiRequest->dataDirection) { case tiDirectionOut: /* Write/Outbound */ break; case tiDirectionIn: /* Read/Inbound */ if ((agSSPInitiatorRequest->dif.flags & DIF_ACTION_FLAG_MASK) == DIF_INSERT) { needPlusDataLenAdjustment = agTRUE; } break; } /* Set SGL data len XXX This code needs to support more sector sizes */ /* Length adjustment for PCIe DMA only not SAS */ if (needPlusDataLenAdjustment == agTRUE) { adjusted_length = tiScsiRequest->scsiCmnd.expDataLength; adjusted_length += (adjusted_length/512) * 8; agSSPInitiatorRequest->dataLength = adjusted_length; } else if (needMinusDataLenAdjustment == agTRUE) { adjusted_length = tiScsiRequest->scsiCmnd.expDataLength; adjusted_length -= (adjusted_length/520) * 8; agSSPInitiatorRequest->dataLength = adjusted_length; } else { /* setting the data length */ agSSPInitiatorRequest->dataLength = tiScsiRequest->scsiCmnd.expDataLength; } /* initializes "agsaSgl_t agSgl" of "agsaDifSSPInitiatorRequest_t" */ tiStatus = itdssIOPrepareSGL( tiRoot, tdIORequestBody, &tiScsiRequest->agSgl1, tiScsiRequest->sglVirtualAddr ); TI_DBG2(("tiINISuperIOStart:TI_SCSI_INITIATOR_DIF needMinusDataLenAdjustment %d needPlusDataLenAdjustment %d difAction %X\n", needMinusDataLenAdjustment, needPlusDataLenAdjustment, agSSPInitiatorRequest->dif.flags & DIF_ACTION_FLAG_MASK)); } else { /* setting the data length */ agSSPInitiatorRequest->dataLength = tiScsiRequest->scsiCmnd.expDataLength; /* initializes "agsaSgl_t agSgl" of "agsaSSPInitiatorRequest_t" */ tiStatus = itdssIOPrepareSGL( tiRoot, tdIORequestBody, &tiScsiRequest->agSgl1, tiScsiRequest->sglVirtualAddr ); } if (tiStatus != tiSuccess) { TI_DBG1(("tiINISuperIOStart: can't get SGL\n")); return tiStatus; } TI_DBG6(("tiINISuperIOStart: tiScsiRequest->scsiCmnd.expDataLength %d\n", tiScsiRequest->scsiCmnd.expDataLength)); /* process taskattribute */ if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_SIMPLE) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_SIMPLE; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_ORDERED) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_ORDERED; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_HEAD_OF_QUEUE) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_HEAD_OF_QUEUE; } else if (tiScsiRequest->scsiCmnd.taskAttribute == TASK_ACA) { agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute = (bit8) agSSPInitiatorRequest->sspCmdIU.efb_tp_taskAttribute | TD_TASK_ACA; } /* copy cdb bytes */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.cdb, tiScsiRequest->scsiCmnd.cdb, 16); /* copy lun field */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.lun, tiScsiRequest->scsiCmnd.lun.lun, 8); #ifdef CCBUILD_INDIRECT_CDB /* check the Indirect CDB flag */ if (tiScsiRequest->flags & TI_SCSI_INITIATOR_INDIRECT_CDB) { /* Indirect CDB */ if (tiScsiRequest->dataDirection == tiDirectionIn) { agRequestType = AGSA_SSP_INIT_READ_INDIRECT; TI_DBG6(("tiINISuperIOStart: Indirect READ\n")); } else if (tiScsiRequest->dataDirection == tiDirectionOut) { agRequestType = AGSA_SSP_INIT_WRITE_INDIRECT; TI_DBG6(("tiINISuperIOStart: Indirect WRITE\n")); } else { agRequestType = AGSA_REQ_TYPE_UNKNOWN; TI_DBG1(("tiINISuperIOStart: unknown data direction\n")); } agSSPInitiatorIndRequest = &(agSASRequestBody->sspInitiatorReqIndirect); /* copy the constructed SSPIU info to indirect SSPIU buffer */ osti_memcpy(tiScsiRequest->IndCDBBuffer, &agSSPInitiatorRequest->sspCmdIU, sizeof(agsaSSPCmdInfoUnit_t)); /* initialize the indirect CDB buffer address and length */ agSSPInitiatorIndRequest->sspInitiatorReqAddrLower32 = tiScsiRequest->IndCDBLowAddr; agSSPInitiatorIndRequest->sspInitiatorReqAddrUpper32 = tiScsiRequest->IndCDBHighAddr; agSSPInitiatorIndRequest->sspInitiatorReqLen = sizeof(agsaSSPCmdInfoUnit_t); } else #endif //CCBUILD_INDIRECT_CDB { /* Direct CDB */ if (tiScsiRequest->dataDirection == tiDirectionIn) { agRequestType = AGSA_SSP_INIT_READ; TI_DBG6(("tiINISuperIOStart: READ\n")); } else if (tiScsiRequest->dataDirection == tiDirectionOut) { agRequestType = AGSA_SSP_INIT_WRITE; TI_DBG6(("tiINISuperIOStart: WRITE\n")); } else { agRequestType = AGSA_REQ_TYPE_UNKNOWN; TI_DBG1(("tiINISuperIOStart: unknown data direction\n")); } } tdIORequestBody->agRequestType = agRequestType; TI_DBG6(("tiINISuperIOStart: device AddrHi 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi)); TI_DBG6(("tiINISuperIOStart: device AddrLo 0x%08x\n", oneDeviceData->SASAddressID.sasAddressLo)); #ifdef DBG /* for debugging */ if (tdIORequestBody->IOCompletionFunc == agNULL) { TI_DBG1(("tiINISuperIOStart: Error!!!! IOCompletionFunc is NULL\n")); return tiError; } #endif saStatus = saSSPStart(agRoot, agIORequest, tdsaRotateQnumber(tiRoot, oneDeviceData), agDevHandle, agRequestType, agSASRequestBody, agNULL, &ossaSSPCompleted); if (saStatus == AGSA_RC_SUCCESS) { Initiator->NumIOsActive++; tdIORequestBody->ioStarted = agTRUE; tdIORequestBody->ioCompleted = agFALSE; tiStatus = tiSuccess; } else { tdIORequestBody->ioStarted = agFALSE; tdIORequestBody->ioCompleted = agTRUE; if (saStatus == AGSA_RC_BUSY) { TI_DBG4(("tiINISuperIOStart: saSSPStart busy\n")); tiStatus = tiBusy; } else { tiStatus = tiError; } return tiStatus; } } #ifdef FDS_SM else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { TI_DBG5(("tiINISuperIOStart: calling satIOStart\n")); TI_DBG5(("tiINISuperIOStart: onedevicedata did %d\n", oneDeviceData->id)); TI_DBG5(("tiINISuperIOStart: SATA sasAddressHi 0x%08x\n", oneDeviceData->SASAddressID.sasAddressHi)); TI_DBG5(("tiINISuperIOStart: SATA sasAddressLo 0x%08x\n", oneDeviceData->SASAddressID.sasAddressLo)); tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ /* the tdIORequestBody has been initialized by Storport in SRB Extension */ /*osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t));*/ /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; tdIORequestBody->superIOFlag = agTRUE; tiIORequest->tdData = tdIORequestBody; tdIORequestBody->tiIORequest = tiIORequest; smIORequest = (smIORequest_t *)&(tdIORequestBody->smIORequest); smIORequest->tdData = tdIORequestBody; smIORequest->smData = &tdIORequestBody->smIORequestBody; smDeviceHandle = (smDeviceHandle_t *)&(oneDeviceData->smDeviceHandle); smDeviceHandle->tdData = oneDeviceData; smSuperSCSIRequest = (smSuperScsiInitiatorRequest_t *)&(tdIORequestBody->SM.smSuperSCSIRequest); osti_memcpy(smSuperSCSIRequest, tiScsiRequest, sizeof(smSuperScsiInitiatorRequest_t)); tiStatus = smSuperIOStart(smRoot, smIORequest, smDeviceHandle, smSuperSCSIRequest, oneDeviceData->SASAddressID.sasAddressHi, oneDeviceData->SASAddressID.sasAddressLo, interruptContext); } #else else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { TI_DBG5(("tiINISuperIOStart: calling satIOStart\n")); TI_DBG5(("tiINISuperIOStart: onedevicedata did %d\n", oneDeviceData->id)); #ifdef SATA_ENABLE tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* initialize */ osti_memset(tdIORequestBody, 0, sizeof(tdIORequestBody_t)); /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging2Completed; satIOContext = &(tdIORequestBody->transport.SATA.satIOContext); /* * Need to initialize all the fields within satIOContext except * reqType and satCompleteCB which will be set in sat.c depending on cmd. */ tdIORequestBody->transport.SATA.tiSenseData.senseData = agNULL; tdIORequestBody->transport.SATA.tiSenseData.senseLen = 0; satIOContext->pSatDevData = &oneDeviceData->satDevData; satIOContext->pFis = &tdIORequestBody->transport.SATA.agSATARequestBody.fis.fisRegHostToDev; satIOContext->pScsiCmnd = &tiScsiRequest->scsiCmnd; satIOContext->pSense = &tdIORequestBody->transport.SATA.sensePayload; satIOContext->pTiSenseData = &tdIORequestBody->transport.SATA.tiSenseData; satIOContext->pTiSenseData->senseData = satIOContext->pSense; /* satIOContext->pSense = (scsiRspSense_t *)satIOContext->pTiSenseData->senseData; */ satIOContext->tiRequestBody = tiRequestBody; satIOContext->interruptContext = interruptContext; satIOContext->ptiDeviceHandle = tiDeviceHandle; /* This code uses a kludge for the tiScsiXchg. Many subroutines in the SATA code require a tiScsiInitiatorRequest. Since it would be a lot of work to replicate those functions for a tiSuperScsiInitiatorRequest, we will use a short cut. The standard pointer will be passed, but the superIOFlag marks the real type of the structure. */ satIOContext->tiScsiXchg = tiScsiRequest; satIOContext->superIOFlag = agTRUE; satIOContext->satIntIoContext = agNULL; satIOContext->satOrgIOContext = agNULL; /* satIOContext->tiIORequest = tiIORequest; */ /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* followings are used only for internal IO */ satIOContext->currentLBA = 0; satIOContext->OrgTL = 0; TI_DBG5(("tiINISuperIOStart: pSatDevData=%p\n", satIOContext->pSatDevData )); tiStatus = satIOStart( tiRoot, tiIORequest, tiDeviceHandle, satIOContext->tiScsiXchg, satIOContext); return tiStatus; #endif } #endif /* else of FDS_SM */ else { tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; tdIORequestBody->IOCompletionFunc = itdssIOForDebugging3Completed; TI_DBG1(("tiINISuperIOStart: wrong unspported Device %d\n", oneDeviceData->DeviceType)); /* error. unsupported IO */ } return tiStatus; } osGLOBAL bit32 tiINISMPStart( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiSMPFrame_t *tiSMPFrame, void *tiSMPBody, bit32 interruptContext ) { tdsaDeviceData_t *oneDeviceData; agsaIORequest_t *agIORequest = agNULL; tdIORequestBody_t *tdSMPRequestBody = agNULL; agsaRoot_t *agRoot = agNULL; agsaDevHandle_t *agDevHandle = agNULL; agsaSASRequestBody_t *agRequestBody = agNULL; agsaSMPFrame_t *agSMPFrame = agNULL; bit32 agRequestType; bit32 tiStatus = tiError; bit32 saStatus = AGSA_RC_FAILURE; bit32 queueNum; TDSA_INP_ENTER(tiRoot); TI_DBG6(("tiINISMPStart: start\n")); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; TI_DBG6(("tiINISMPStart: onedevicedata %p\n", oneDeviceData)); TI_DBG6(("tiINISMPStart: tiDeviceHandle %p\n", tiDeviceHandle)); if (oneDeviceData == agNULL) { TI_DBG1(("tiINISMPStart: tiDeviceHandle=%p Expander DeviceData is NULL\n", tiDeviceHandle )); return tiError; } if (tiIORequest->osData == agNULL) { TI_DBG1(("tiINISMPStart: tiIORequest->osData is NULL, wrong\n")); return tiError; } agRoot = oneDeviceData->agRoot; agDevHandle = oneDeviceData->agDevHandle; tdSMPRequestBody = (tdIORequestBody_t *)tiSMPBody; tdSMPRequestBody->tiIORequest = tiIORequest; tiIORequest->tdData = tdSMPRequestBody; agIORequest = &(tdSMPRequestBody->agIORequest); agIORequest->osData = (void *) tdSMPRequestBody; agRequestBody = &(tdSMPRequestBody->transport.SAS.agSASRequestBody); agSMPFrame = &(agRequestBody->smpFrame); if (!DEVICE_IS_SMP_TARGET(oneDeviceData)) { TI_DBG1(("tiINISMPStart: Target Device is not SMP device\n")); return tiError; } if (tiSMPFrame->flag == 0) // define DIRECT SMP at td layer? { TI_DBG6(("tiINISMPStart: Direct SMP\n")); agSMPFrame->outFrameBuf = tiSMPFrame->outFrameBuf; agSMPFrame->outFrameLen = tiSMPFrame->outFrameLen; tdhexdump("tiINISMPStart agSMPFrame", (bit8 *)agSMPFrame->outFrameBuf, agSMPFrame->outFrameLen); agSMPFrame->expectedRespLen = tiSMPFrame->expectedRespLen; agSMPFrame->inFrameLen = 0; agSMPFrame->flag = tiSMPFrame->flag; agRequestType = AGSA_SMP_INIT_REQ; queueNum = 0; saStatus = saSMPStart(agRoot, agIORequest, queueNum, agDevHandle, agRequestType, agRequestBody, &ossaSMPCAMCompleted ); if (saStatus == AGSA_RC_SUCCESS) { tiStatus = tiSuccess; } else { if (saStatus == AGSA_RC_BUSY) { TI_DBG1(("tiINISMPStart: saSSPStart busy\n")); tiStatus = tiBusy; } else { TI_DBG1(("tiINISMPStart: saSSPStart error\n")); tiStatus = tiError; } return tiStatus; } } else { TI_DBG1(("tiINISMPStart: Indirect SMP! Not supported yet\n")); tiStatus = tiError; } return tiStatus; } #ifdef TD_INT_COALESCE osGLOBAL bit32 tiINIIOStartIntCoalesce( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiScsiInitiatorRequest_t *tiScsiRequest, void *tiRequestBody, bit32 interruptContext, tiIntCoalesceContext_t *tiIntCoalesceCxt ) { tdsaRoot_t *tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; tdsaContext_t *tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; itdsaIni_t *Initiator = (itdsaIni_t *)tdsaAllShared->itdsaIni; tdsaDeviceData_t *oneDeviceData; agsaRoot_t *agRoot = agNULL; agsaIORequest_t *agIORequest = agNULL; agsaDevHandle_t *agDevHandle = agNULL; bit32 agRequestType; agsaSASRequestBody_t *agSASRequestBody = agNULL; bit32 tiStatus = tiError; bit32 saStatus = AGSA_RC_FAILURE; tdIORequestBody_t *tdIORequestBody; agsaSSPInitiatorRequest_t *agSSPInitiatorRequest; tdsaIntCoalesceContext_t *tdsaIntCoalCxt; agsaIntCoalesceContext_t *agIntCoalCxt; TI_DBG1(("tiINIIOStartIntCoalesce: start\n")); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; TI_DBG6(("tiINIIOStartIntCoalesce: onedevicedata %p\n", oneDeviceData)); if(oneDeviceData == agNULL) { TI_DBG1(("tiINIIOStartIntCoalesce: tiDeviceHandle=%p DeviceData is NULL\n", tiDeviceHandle )); return tiIONoDevice; } /* starting IO with SAS device */ if (oneDeviceData->DeviceType == TD_SAS_DEVICE) { TI_DBG6(("tiINIIOStartIntCoalesce: calling saSSPStart\n")); agRoot = oneDeviceData->agRoot; agDevHandle = oneDeviceData->agDevHandle; /* OS layer has tdlayer data structure pointer in tdIORequestBody_t tdIOReqBody; in ccb_t in agtiapi.h */ tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* let's initialize tdIOrequestBody */ /* initialize callback */ tdIORequestBody->IOCompletionFunc = itdssIOCompleted; /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* initialize expDataLength */ tdIORequestBody->IOType.InitiatorRegIO.expDataLength = tiScsiRequest->scsiCmnd.expDataLength; /* initializes "agsaSgl_t agSgl" of "agsaDifSSPInitiatorRequest_t" */ tiStatus = itdssIOPrepareSGL( tiRoot, tdIORequestBody, &tiScsiRequest->agSgl1, tiScsiRequest->sglVirtualAddr ); if (tiStatus != tiSuccess) { TI_DBG1(("tiINIIOStartIntCoalesce: can't get SGL\n")); return tiStatus; } /* initialize agIORequest */ agIORequest = &(tdIORequestBody->agIORequest); agIORequest->osData = (void *) tdIORequestBody; agIORequest->sdkData = agNULL; /* LL takes care of this */ /* initialize tdIORequestBody_t tdIORequestBody -> agSASRequestBody */ agSASRequestBody = &(tdIORequestBody->transport.SAS.agSASRequestBody); agSSPInitiatorRequest = &(agSASRequestBody->sspInitiatorReq); /* copy cdb bytes */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.cdb, tiScsiRequest->scsiCmnd.cdb, 16); /* copy lun field */ osti_memcpy(agSSPInitiatorRequest->sspCmdIU.lun, tiScsiRequest->scsiCmnd.lun.lun, 8); /* setting the data length */ agSSPInitiatorRequest->dataLength = tiScsiRequest->scsiCmnd.expDataLength; TI_DBG6(("tiINIIOStartIntCoalesce: tiScsiRequest->scsiCmnd.expDataLength %d\n", tiScsiRequest->scsiCmnd.expDataLength)); agSSPInitiatorRequest->firstBurstSize = 0; if (tiScsiRequest->dataDirection == tiDirectionIn) { agRequestType = AGSA_SSP_INIT_READ; TI_DBG6(("tiINIIOStartIntCoalesce: READ\n")); } else if (tiScsiRequest->dataDirection == tiDirectionOut) { agRequestType = AGSA_SSP_INIT_WRITE; TI_DBG6(("tiINIIOStartIntCoalesce: WRITE\n")); } else { agRequestType = AGSA_REQ_TYPE_UNKNOWN; TI_DBG1(("tiINIIOStartIntCoalesce: unknown data direction\n")); } tdIORequestBody->agRequestType = agRequestType; tdsaIntCoalCxt = (tdsaIntCoalesceContext_t *)tiIntCoalesceCxt->tdData; agIntCoalCxt = &(tdsaIntCoalCxt->agIntCoalCxt); #ifdef LL_INT_COALESCE saStatus = saSSPStartIntCoalesce(agRoot, agIORequest, agIntCoalCxt, agDevHandle, agRequestType, agSASRequestBody, &ossaSSPCompleted); #endif tdIORequestBody->ioStarted = agTRUE; tdIORequestBody->ioCompleted = agFALSE; if (saStatus == AGSA_RC_SUCCESS) { Initiator->NumIOsActive++; tiStatus = tiSuccess; } else { TI_DBG1(("tiINIIOStartIntCoalesce: saSSPStart failed\n")); tdIORequestBody->ioStarted = agFALSE; tdIORequestBody->ioCompleted = agTRUE; if (saStatus == AGSA_RC_BUSY) { tiStatus = tiBusy; } else { tiStatus = tiError; } return tiStatus; } } else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { /* satIOStart() -> saSATAStartIntCoalesce() */ TI_DBG1(("tiINIIOStartIntCoalesce: SATA not supported yet\n")); return tiStatus; } else { TI_DBG1(("tiINIIOStartIntCoalesce: wrong unspported Device %d\n", oneDeviceData->DeviceType)); /* error. unsupported IO */ } return tiStatus; } osGLOBAL bit32 tiINIIOStartIntCoalesceDif( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, tiScsiInitiatorRequest_t *tiScsiRequest, void *tiRequestBody, bit32 interruptContext, tiIntCoalesceContext_t *tiIntCoalesceCxt, tiDif_t *difOption ) { tdsaRoot_t *tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; tdsaContext_t *tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; itdsaIni_t *Initiator = (itdsaIni_t *)tdsaAllShared->itdsaIni; tdsaDeviceData_t *oneDeviceData; agsaRoot_t *agRoot = agNULL; agsaIORequest_t *agIORequest = agNULL; agsaDevHandle_t *agDevHandle = agNULL; bit32 agRequestType; agsaDifSSPRequestBody_t *agEdcSSPRequestBody = agNULL; bit32 tiStatus = tiError; bit32 saStatus = AGSA_RC_FAILURE; tdIORequestBody_t *tdIORequestBody; agsaDifSSPInitiatorRequest_t *agEdcSSPInitiatorRequest; agsaDif_t *agEdc; bit32 agUpdateMask = 0; bit32 agVerifyMask = 0; tdsaIntCoalesceContext_t *tdsaIntCoalCxt; agsaIntCoalesceContext_t *agIntCoalCxt; TI_DBG1(("tiINIIOStartIntCoalesceDif: start\n")); oneDeviceData = (tdsaDeviceData_t *)tiDeviceHandle->tdData; TI_DBG6(("tiINIIOStartIntCoalesceDif: onedevicedata %p\n", oneDeviceData)); if(oneDeviceData == agNULL) { TI_DBG1(("tiINIIOStartIntCoalesceDif: tiDeviceHandle=%p DeviceData is NULL\n", tiDeviceHandle )); return tiIONoDevice; } /* starting IO with SAS device */ if (oneDeviceData->DeviceType == TD_SAS_DEVICE) { TI_DBG6(("tiINIIOStartIntCoalesceDif: calling saSSPStart\n")); agRoot = oneDeviceData->agRoot; agDevHandle = oneDeviceData->agDevHandle; /* OS layer has tdlayer data structure pointer in tdIORequestBody_t tdIOReqBody; in ccb_t in agtiapi.h */ tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; /* let's initialize tdIOrequestBody */ /* initialize callback */ tdIORequestBody->IOCompletionFunc = itdssIOCompleted; /* initialize tiDevhandle */ tdIORequestBody->tiDevHandle = tiDeviceHandle; /* initialize tiIORequest */ tdIORequestBody->tiIORequest = tiIORequest; /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; /* initialize expDataLength */ tdIORequestBody->IOType.InitiatorRegIO.expDataLength = tiScsiRequest->scsiCmnd.expDataLength; /* initializes "agsaSgl_t agSgl" of "agsaDifSSPInitiatorRequest_t" */ tiStatus = itdssIOPrepareSGL( tiRoot, tdIORequestBody, &tiScsiRequest->agSgl1, tiScsiRequest->sglVirtualAddr ); if (tiStatus != tiSuccess) { TI_DBG1(("tiINIIOStartIntCoalesceDif: can't get SGL\n")); return tiStatus; } /* initialize agIORequest */ agIORequest = &(tdIORequestBody->agIORequest); agIORequest->osData = (void *) tdIORequestBody; agIORequest->sdkData = agNULL; /* LL takes care of this */ /* initialize tdIORequestBody_t tdIORequestBody -> agSASRequestBody */ agEdcSSPRequestBody = &(tdIORequestBody->transport.SAS.agEdcSSPRequestBody); agEdcSSPInitiatorRequest = &(agEdcSSPRequestBody->edcSSPInitiatorReq); /* copy cdb bytes */ osti_memcpy(agEdcSSPInitiatorRequest->sspCmdIU.cdb, tiScsiRequest->scsiCmnd.cdb, 16); /* copy lun field */ osti_memcpy(agEdcSSPInitiatorRequest->sspCmdIU.lun, tiScsiRequest->scsiCmnd.lun.lun, 8); /* setting the data length */ agEdcSSPInitiatorRequest->dataLength = tiScsiRequest->scsiCmnd.expDataLength; TI_DBG6(("tiINIIOStartIntCoalesceDif: tiScsiRequest->scsiCmnd.expDataLength %d\n", tiScsiRequest->scsiCmnd.expDataLength)); agEdcSSPInitiatorRequest->firstBurstSize = 0; if (tiScsiRequest->dataDirection == tiDirectionIn) { agRequestType = AGSA_SSP_INIT_READ; TI_DBG1(("tiINIIOStartIntCoalesceDif: READ difAction %X\n",difOption->difAction)); } else if (tiScsiRequest->dataDirection == tiDirectionOut) { agRequestType = AGSA_SSP_INIT_WRITE; TI_DBG1(("tiINIIOStartIntCoalesceDif: WRITE difAction %X\n",difOption->difAction)); } else { agRequestType = AGSA_REQ_TYPE_UNKNOWN; TI_DBG1(("tiINIIOStartIntCoalesceDif: unknown data direction\n")); } tdIORequestBody->agRequestType = agRequestType; /* process interrupt coalesce context */ tdsaIntCoalCxt = (tdsaIntCoalesceContext_t *)tiIntCoalesceCxt->tdData; agIntCoalCxt = &(tdsaIntCoalCxt->agIntCoalCxt); /* process DIF */ agEdc = &(agEdcSSPInitiatorRequest->edc); osti_memset(agEdc, 0, sizeof(agsaDif_t)); /* setting edcFlag */ if (difOption->enableBlockCount) { /* enables block count; bit5 */ agEdc->edcFlag = agEdc->edcFlag | 0x20; /* 0010 0000 */ } if (difOption->enableCrc) { /* enables CRC verification; bit6 */ agEdc->edcFlag = agEdc->edcFlag | 0x40; /* 0100 0000 */ } if (difOption->enableIOSeed) { } if (difOption->difAction == DIF_INSERT) { /* bit 0 - 2; 000 */ agEdc->edcFlag = agEdc->edcFlag & 0xFFFFFFF8; } else if (difOption->difAction == DIF_VERIFY_FORWARD) { /* bit 0 - 2; 001 */ agEdc->edcFlag = agEdc->edcFlag | 0x01; } else if (difOption->difAction == DIF_VERIFY_DELETE) { /* bit 0 - 2; 010 */ agEdc->edcFlag = agEdc->edcFlag | 0x02; } else { /* DIF_VERIFY_REPLACE */ /* bit 0 - 2; 011 */ agEdc->edcFlag = agEdc->edcFlag | 0x04; } /* set Update Mask; bit 16-21 */ agUpdateMask = (difOption->tagUpdateMask) & 0x3F; /* 0011 1111 */ agUpdateMask = agUpdateMask << 16; agEdc->edcFlag = agEdc->edcFlag | agUpdateMask; /* set Verify Mask bit 24-29 */ agVerifyMask = (difOption->tagVerifyMask) & 0x3F; /* 0011 1111 */ agVerifyMask = agVerifyMask << 24; agEdc->edcFlag = agEdc->edcFlag | agVerifyMask; agEdc->appTag = difOption->udtArray[0]; agEdc->appTag = (agEdc->appTag << 8) | difOption->udtArray[1]; agEdc->lbaReferenceTag = difOption->udtArray[2]; agEdc->lbaReferenceTag = (agEdc->lbaReferenceTag << 8) | difOption->udtArray[3]; agEdc->lbaReferenceTag = (agEdc->lbaReferenceTag << 8) | difOption->udtArray[4]; agEdc->lbaReferenceTag = (agEdc->lbaReferenceTag << 8) | difOption->udtArray[5]; /* currently TISA supports only 512 logical block size */ agEdc->lbSize = 512; #ifdef LL_INT_COALESCE saStatus = saSSPStartIntCoalesceEdc(agRoot, agIORequest, agIntCoalCxt, agDevHandle, agRequestType, agEdcSSPRequestBody, &ossaSSPCompleted); #endif tdIORequestBody->ioStarted = agTRUE; tdIORequestBody->ioCompleted = agFALSE; if (saStatus == AGSA_RC_SUCCESS) { Initiator->NumIOsActive++; tiStatus = tiSuccess; } else { TI_DBG1(("tiINIIOStartIntCoalesceDif: saSSPStart failed\n")); tdIORequestBody->ioStarted = agFALSE; tdIORequestBody->ioCompleted = agTRUE; if (saStatus == AGSA_RC_BUSY) { tiStatus = tiBusy; } else { tiStatus = tiError; } return tiStatus; } } else if (oneDeviceData->DeviceType == TD_SATA_DEVICE) { /* satIOStart() -> saSATAStartIntCoalesceEdc() */ TI_DBG1(("tiINIIOStartIntCoalesceDif: SATA not supported yet\n")); return tiStatus; } else { TI_DBG1(("tiINIIOStartIntCoalesceDif: wrong unspported Device %d\n", oneDeviceData->DeviceType)); /* error. unsupported IO */ } return tiStatus; } osGLOBAL bit32 tiINIIntCoalesceInit( tiRoot_t *tiRoot, tiIntCoalesceContext_t *tiIntCoalesceCxt, bit32 count ) { tdsaRoot_t *tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; tdsaContext_t *tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; agsaRoot_t *agRoot = agNULL; tdsaIntCoalesceContext_t *tdsaIntCoalCxtHead = (tdsaIntCoalesceContext_t *)tdsaAllShared->IntCoalesce; tdsaIntCoalesceContext_t *tdsaIntCoalCxt; agsaIntCoalesceContext_t *agIntCoalCxt; tdList_t *tdsaIntCoalCxtList = agNULL; bit32 tiStatus = tiError; TI_DBG1(("tiINIIntCoalesceInit: start\n")); tdsaSingleThreadedEnter(tiRoot, TD_INTCOAL_LOCK); if (TDLIST_NOT_EMPTY(&(tdsaIntCoalCxtHead->FreeLink))) { TDLIST_DEQUEUE_FROM_HEAD(&tdsaIntCoalCxtList, &(tdsaIntCoalCxtHead->FreeLink)); tdsaSingleThreadedLeave(tiRoot, TD_INTCOAL_LOCK); tdsaIntCoalCxt = TDLIST_OBJECT_BASE(tdsaIntCoalesceContext_t, FreeLink, tdsaIntCoalCxtList); TI_DBG1(("tiINIIntCoalesceInit: id %d\n", tdsaIntCoalCxt->id)); agRoot = &(tdsaAllShared->agRootNonInt); agIntCoalCxt = &(tdsaIntCoalCxt->agIntCoalCxt); tdsaIntCoalCxt->tiIntCoalesceCxt = tiIntCoalesceCxt; tiIntCoalesceCxt->tdData = tdsaIntCoalCxt; agIntCoalCxt->osData = tdsaIntCoalCxt; tdsaSingleThreadedEnter(tiRoot, TD_INTCOAL_LOCK); TDLIST_ENQUEUE_AT_TAIL(&(tdsaIntCoalCxt->MainLink), &(tdsaIntCoalCxtHead->MainLink)); tdsaSingleThreadedLeave(tiRoot, TD_INTCOAL_LOCK); /* note: currently asynchronously call is assumed. In other words, "ossaIntCoalesceInitCB()" -> "ostiInitiatorCoalesceInitCB()" are used */ #ifdef LL_INT_COALESCE tiStatus = saIntCoalesceInit(agRoot, agIntCoalCxt, count); #endif TI_DBG6(("tiINIIntCoalesceInit: status %d\n", tiStatus)); return tiStatus; } else { tdsaSingleThreadedLeave(tiRoot, TD_INTCOAL_LOCK); TI_DBG1(("tiINIIntCoalesceInit: no more interrupt coalesce context; return fail\n")); return tiStatus; } } #endif /* TD_INT_COALESCE */ /***************************************************************************** *! \brief itdssIOPrepareSGL * * Purpose: This function is called to translate TISA SGL information to the * LL layer SGL. * * \param tiRoot: Pointer to initiator driver/port instance. * \param IORequestBody: TD layer request body for the I/O. * \param tiSgl1: First TISA SGL info. * \param sglVirtualAddr: The virtual address of the first element in * tiSgl1 when tiSgl1 is used with the type tiSglList. * * \return: * * tiSuccess: SGL initialized successfully. * tiError: Failed to initialize SGL. * * *****************************************************************************/ osGLOBAL FORCEINLINE bit32 itdssIOPrepareSGL( tiRoot_t *tiRoot, tdIORequestBody_t *tdIORequestBody, tiSgl_t *tiSgl1, void *sglVirtualAddr ) { agsaSgl_t *agSgl; TI_DBG6(("itdssIOPrepareSGL: start\n")); agSgl = &(tdIORequestBody->transport.SAS.agSASRequestBody.sspInitiatorReq.agSgl); agSgl->len = 0; if (tiSgl1 == agNULL) { TI_DBG1(("itdssIOPrepareSGL: Error tiSgl1 is NULL\n")); return tiError; } if (tdIORequestBody->IOType.InitiatorRegIO.expDataLength == 0) { TI_DBG6(("itdssIOPrepareSGL: expDataLength is 0\n")); agSgl->sgUpper = 0; agSgl->sgLower = 0; agSgl->len = 0; CLEAR_ESGL_EXTEND(agSgl->extReserved); return tiSuccess; } agSgl->sgUpper = tiSgl1->upper; agSgl->sgLower = tiSgl1->lower; agSgl->len = tiSgl1->len; agSgl->extReserved = tiSgl1->type; return tiSuccess; } osGLOBAL bit32 tiNumOfLunIOCTLreq( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, tiDeviceHandle_t *tiDeviceHandle, void *tiRequestBody, tiIOCTLPayload_t *agIOCTLPayload, void *agParam1, void *agParam2 ) { tdsaRoot_t *tdsaRoot = (tdsaRoot_t *) tiRoot->tdData; tdsaContext_t *tdsaAllShared = (tdsaContext_t *)&tdsaRoot->tdsaAllShared; agsaRoot_t *agRoot = &(tdsaAllShared->agRootInt); void *respBuffer = agNULL; void *osMemHandle = agNULL; bit32 ostiMemoryStatus = 0; tdsaDeviceData_t *oneDeviceData = agNULL; agsaSSPInitiatorRequest_t *agSSPFrame = agNULL; bit32 status = IOCTL_CALL_SUCCESS; bit32 agRequestType = 0; agsaDevHandle_t *agDevHandle = agNULL; agsaIORequest_t *agIORequest = agNULL; tdIORequestBody_t *tdIORequestBody = agNULL; agsaSASRequestBody_t *agSASRequestBody = agNULL; do { if((tiIORequest == agNULL) || (tiRequestBody == agNULL)) { status = IOCTL_CALL_FAIL; break; } tdIORequestBody = (tdIORequestBody_t *)tiRequestBody; tdIORequestBody->tiIORequest = tiIORequest; /* save context if we need to abort later */ tiIORequest->tdData = tdIORequestBody; agIORequest = &(tdIORequestBody->agIORequest); agIORequest->osData = (void *) tdIORequestBody; agSASRequestBody = &(tdIORequestBody->transport.SAS.agSASRequestBody); agSSPFrame = &(agSASRequestBody->sspInitiatorReq); ostiMemoryStatus = ostiAllocMemory( tiRoot, &osMemHandle, (void **)&respBuffer, &(agSSPFrame->agSgl.sgUpper), &(agSSPFrame->agSgl.sgLower), 8, REPORT_LUN_LEN, agFALSE); if((ostiMemoryStatus != tiSuccess) && (respBuffer == agNULL )) { status = IOCTL_CALL_FAIL; break; } osti_memset((void *)respBuffer, 0, REPORT_LUN_LEN); // use FW control place in shared structure to keep the neccesary information tdsaAllShared->tdFWControlEx.virtAddr = respBuffer; tdsaAllShared->tdFWControlEx.len = REPORT_LUN_LEN; tdsaAllShared->tdFWControlEx.param1 = agParam1; tdsaAllShared->tdFWControlEx.param2 = agParam2; tdsaAllShared->tdFWControlEx.payload = agIOCTLPayload; tdsaAllShared->tdFWControlEx.inProgress = 1; agRequestType = AGSA_SSP_INIT_READ; status = IOCTL_CALL_PENDING; oneDeviceData = (tdsaDeviceData_t *)(tiDeviceHandle->tdData); agDevHandle = oneDeviceData->agDevHandle; agSSPFrame->sspCmdIU.cdb[0] = REPORT_LUN_OPCODE; agSSPFrame->sspCmdIU.cdb[1] = 0x0; agSSPFrame->sspCmdIU.cdb[2] = 0x0; agSSPFrame->sspCmdIU.cdb[3] = 0x0; agSSPFrame->sspCmdIU.cdb[4] = 0x0; agSSPFrame->sspCmdIU.cdb[5] = 0x0; agSSPFrame->sspCmdIU.cdb[6] = 0x0; agSSPFrame->sspCmdIU.cdb[7] = 0x0; agSSPFrame->sspCmdIU.cdb[8] = 0x0; agSSPFrame->sspCmdIU.cdb[9] = REPORT_LUN_LEN; agSSPFrame->sspCmdIU.cdb[10] = 0x0; agSSPFrame->sspCmdIU.cdb[11] = 0x0; agSSPFrame->dataLength = REPORT_LUN_LEN; agSSPFrame->agSgl.len = sizeof(agsaSSPCmdInfoUnit_t); - + agSSPFrame->agSgl.extReserved = 0; + CLEAR_ESGL_EXTEND(agSSPFrame->agSgl.extReserved); + status = saSSPStart(agRoot, agIORequest, 0, agDevHandle, agRequestType,agSASRequestBody,agNULL, &ossaSSPIoctlCompleted); if(status != AGSA_RC_SUCCESS) { ostiFreeMemory(tiRoot, tdsaAllShared->tdFWControlEx.virtAddr, tdsaAllShared->tdFWControlEx.len); tdsaAllShared->tdFWControlEx.payload = NULL; tdsaAllShared->tdFWControlEx.inProgress = 0; status = IOCTL_CALL_FAIL; } }while(0); return status; } Index: projects/import-googletest-1.8.1/sys/fs/nandfs/nandfs_vnops.c =================================================================== --- projects/import-googletest-1.8.1/sys/fs/nandfs/nandfs_vnops.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/fs/nandfs/nandfs_vnops.c (revision 345026) @@ -1,2457 +1,2454 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2012 Semihalf * Copyright (c) 2008, 2009 Reinoud Zandijk * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * From: NetBSD: nilfs_vnops.c,v 1.2 2009/08/26 03:40:48 elad */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern uma_zone_t nandfs_node_zone; static void nandfs_read_filebuf(struct nandfs_node *, struct buf *); static void nandfs_itimes_locked(struct vnode *); static int nandfs_truncate(struct vnode *, uint64_t); static vop_pathconf_t nandfs_pathconf; #define UPDATE_CLOSE 0 #define UPDATE_WAIT 0 static int nandfs_inactive(struct vop_inactive_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); int error = 0; DPRINTF(VNCALL, ("%s: vp:%p node:%p\n", __func__, vp, node)); if (node == NULL) { DPRINTF(NODE, ("%s: inactive NULL node\n", __func__)); return (0); } if (node->nn_inode.i_mode != 0 && !(node->nn_inode.i_links_count)) { nandfs_truncate(vp, 0); error = nandfs_node_destroy(node); if (error) nandfs_error("%s: destroy node: %p\n", __func__, node); node->nn_flags = 0; vrecycle(vp); } return (error); } static int nandfs_reclaim(struct vop_reclaim_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *nandfs_node = VTON(vp); struct nandfs_device *fsdev = nandfs_node->nn_nandfsdev; uint64_t ino = nandfs_node->nn_ino; DPRINTF(VNCALL, ("%s: vp:%p node:%p\n", __func__, vp, nandfs_node)); /* Invalidate all entries to a particular vnode. */ cache_purge(vp); /* Destroy the vm object and flush associated pages. */ vnode_destroy_vobject(vp); /* Remove from vfs hash if not system vnode */ if (!NANDFS_SYS_NODE(nandfs_node->nn_ino)) vfs_hash_remove(vp); /* Dispose all node knowledge */ nandfs_dispose_node(&nandfs_node); if (!NANDFS_SYS_NODE(ino)) NANDFS_WRITEUNLOCK(fsdev); return (0); } static int nandfs_read(struct vop_read_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); struct nandfs_device *nandfsdev = node->nn_nandfsdev; struct uio *uio = ap->a_uio; struct buf *bp; uint64_t size; uint32_t blocksize; off_t bytesinfile; ssize_t toread, off; daddr_t lbn; ssize_t resid; int error = 0; if (uio->uio_resid == 0) return (0); size = node->nn_inode.i_size; if (uio->uio_offset >= size) return (0); blocksize = nandfsdev->nd_blocksize; bytesinfile = size - uio->uio_offset; resid = omin(uio->uio_resid, bytesinfile); while (resid) { lbn = uio->uio_offset / blocksize; off = uio->uio_offset & (blocksize - 1); toread = omin(resid, blocksize - off); DPRINTF(READ, ("nandfs_read bn: 0x%jx toread: 0x%zx (0x%x)\n", (uintmax_t)lbn, toread, blocksize)); error = nandfs_bread(node, lbn, NOCRED, 0, &bp); if (error) { brelse(bp); break; } error = uiomove(bp->b_data + off, toread, uio); if (error) { brelse(bp); break; } brelse(bp); resid -= toread; } return (error); } static int nandfs_write(struct vop_write_args *ap) { struct nandfs_device *fsdev; struct nandfs_node *node; struct vnode *vp; struct uio *uio; struct buf *bp; uint64_t file_size, vblk; uint32_t blocksize; ssize_t towrite, off; daddr_t lbn; ssize_t resid; int error, ioflag, modified; vp = ap->a_vp; uio = ap->a_uio; ioflag = ap->a_ioflag; node = VTON(vp); fsdev = node->nn_nandfsdev; if (nandfs_fs_full(fsdev)) return (ENOSPC); DPRINTF(WRITE, ("nandfs_write called %#zx at %#jx\n", uio->uio_resid, (uintmax_t)uio->uio_offset)); if (uio->uio_offset < 0) return (EINVAL); if (uio->uio_resid == 0) return (0); blocksize = fsdev->nd_blocksize; file_size = node->nn_inode.i_size; switch (vp->v_type) { case VREG: if (ioflag & IO_APPEND) uio->uio_offset = file_size; break; case VDIR: return (EISDIR); case VLNK: break; default: panic("%s: bad file type vp: %p", __func__, vp); } /* If explicitly asked to append, uio_offset can be wrong? */ if (ioflag & IO_APPEND) uio->uio_offset = file_size; resid = uio->uio_resid; modified = error = 0; while (uio->uio_resid) { lbn = uio->uio_offset / blocksize; off = uio->uio_offset & (blocksize - 1); towrite = omin(uio->uio_resid, blocksize - off); DPRINTF(WRITE, ("%s: lbn: 0x%jd toread: 0x%zx (0x%x)\n", __func__, (uintmax_t)lbn, towrite, blocksize)); error = nandfs_bmap_lookup(node, lbn, &vblk); if (error) break; DPRINTF(WRITE, ("%s: lbn: 0x%jd toread: 0x%zx (0x%x) " "vblk=%jx\n", __func__, (uintmax_t)lbn, towrite, blocksize, vblk)); if (vblk != 0) error = nandfs_bread(node, lbn, NOCRED, 0, &bp); else error = nandfs_bcreate(node, lbn, NOCRED, 0, &bp); DPRINTF(WRITE, ("%s: vp %p bread bp %p lbn %#jx\n", __func__, vp, bp, (uintmax_t)lbn)); if (error) { if (bp) brelse(bp); break; } error = uiomove((char *)bp->b_data + off, (int)towrite, uio); if (error) break; error = nandfs_dirty_buf(bp, 0); if (error) break; modified++; } /* XXX proper handling when only part of file was properly written */ if (modified) { if (resid > uio->uio_resid && ap->a_cred && ap->a_cred->cr_uid != 0) node->nn_inode.i_mode &= ~(ISUID | ISGID); if (file_size < uio->uio_offset + uio->uio_resid) { node->nn_inode.i_size = uio->uio_offset + uio->uio_resid; node->nn_flags |= IN_CHANGE | IN_UPDATE; vnode_pager_setsize(vp, uio->uio_offset + uio->uio_resid); nandfs_itimes(vp); } } DPRINTF(WRITE, ("%s: return:%d\n", __func__, error)); return (error); } static int nandfs_lookup(struct vop_cachedlookup_args *ap) { struct vnode *dvp, **vpp; struct componentname *cnp; struct ucred *cred; struct thread *td; struct nandfs_node *dir_node, *node; struct nandfsmount *nmp; uint64_t ino, off; const char *name; int namelen, nameiop, islastcn, mounted_ro; int error, found; DPRINTF(VNCALL, ("%s\n", __func__)); dvp = ap->a_dvp; vpp = ap->a_vpp; *vpp = NULL; cnp = ap->a_cnp; cred = cnp->cn_cred; td = cnp->cn_thread; dir_node = VTON(dvp); nmp = dir_node->nn_nmp; /* Simplify/clarification flags */ nameiop = cnp->cn_nameiop; islastcn = cnp->cn_flags & ISLASTCN; mounted_ro = dvp->v_mount->mnt_flag & MNT_RDONLY; /* * If requesting a modify on the last path element on a read-only * filingsystem, reject lookup; */ if (islastcn && mounted_ro && (nameiop == DELETE || nameiop == RENAME)) return (EROFS); if (dir_node->nn_inode.i_links_count == 0) return (ENOENT); /* * Obviously, the file is not (anymore) in the namecache, we have to * search for it. There are three basic cases: '.', '..' and others. * * Following the guidelines of VOP_LOOKUP manpage and tmpfs. */ error = 0; if ((cnp->cn_namelen == 1) && (cnp->cn_nameptr[0] == '.')) { DPRINTF(LOOKUP, ("\tlookup '.'\n")); /* Special case 1 '.' */ VREF(dvp); *vpp = dvp; /* Done */ } else if (cnp->cn_flags & ISDOTDOT) { /* Special case 2 '..' */ DPRINTF(LOOKUP, ("\tlookup '..'\n")); /* Get our node */ name = ".."; namelen = 2; error = nandfs_lookup_name_in_dir(dvp, name, namelen, &ino, &found, &off); if (error) goto out; if (!found) error = ENOENT; /* First unlock parent */ VOP_UNLOCK(dvp, 0); if (error == 0) { DPRINTF(LOOKUP, ("\tfound '..'\n")); /* Try to create/reuse the node */ error = nandfs_get_node(nmp, ino, &node); if (!error) { DPRINTF(LOOKUP, ("\tnode retrieved/created OK\n")); *vpp = NTOV(node); } } /* Try to relock parent */ vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); } else { DPRINTF(LOOKUP, ("\tlookup file\n")); /* All other files */ /* Look up filename in the directory returning its inode */ name = cnp->cn_nameptr; namelen = cnp->cn_namelen; error = nandfs_lookup_name_in_dir(dvp, name, namelen, &ino, &found, &off); if (error) goto out; if (!found) { DPRINTF(LOOKUP, ("\tNOT found\n")); /* * UGH, didn't find name. If we're creating or * renaming on the last name this is OK and we ought * to return EJUSTRETURN if its allowed to be created. */ error = ENOENT; if ((nameiop == CREATE || nameiop == RENAME) && islastcn) { error = VOP_ACCESS(dvp, VWRITE, cred, td); if (!error) { /* keep the component name */ cnp->cn_flags |= SAVENAME; error = EJUSTRETURN; } } /* Done */ } else { if (ino == NANDFS_WHT_INO) cnp->cn_flags |= ISWHITEOUT; if ((cnp->cn_flags & ISWHITEOUT) && (nameiop == LOOKUP)) return (ENOENT); if ((nameiop == DELETE) && islastcn) { if ((cnp->cn_flags & ISWHITEOUT) && (cnp->cn_flags & DOWHITEOUT)) { cnp->cn_flags |= SAVENAME; dir_node->nn_diroff = off; return (EJUSTRETURN); } error = VOP_ACCESS(dvp, VWRITE, cred, cnp->cn_thread); if (error) return (error); /* Try to create/reuse the node */ error = nandfs_get_node(nmp, ino, &node); if (!error) { *vpp = NTOV(node); node->nn_diroff = off; } if ((dir_node->nn_inode.i_mode & ISVTX) && cred->cr_uid != 0 && cred->cr_uid != dir_node->nn_inode.i_uid && node->nn_inode.i_uid != cred->cr_uid) { vput(*vpp); *vpp = NULL; return (EPERM); } } else if ((nameiop == RENAME) && islastcn) { error = VOP_ACCESS(dvp, VWRITE, cred, cnp->cn_thread); if (error) return (error); /* Try to create/reuse the node */ error = nandfs_get_node(nmp, ino, &node); if (!error) { *vpp = NTOV(node); node->nn_diroff = off; } } else { /* Try to create/reuse the node */ error = nandfs_get_node(nmp, ino, &node); if (!error) { *vpp = NTOV(node); node->nn_diroff = off; } } } } out: /* * Store result in the cache if requested. If we are creating a file, * the file might not be found and thus putting it into the namecache * might be seen as negative caching. */ if ((cnp->cn_flags & MAKEENTRY) != 0) cache_enter(dvp, *vpp, cnp); return (error); } static int nandfs_getattr(struct vop_getattr_args *ap) { struct vnode *vp = ap->a_vp; struct vattr *vap = ap->a_vap; struct nandfs_node *node = VTON(vp); struct nandfs_inode *inode = &node->nn_inode; DPRINTF(VNCALL, ("%s: vp: %p\n", __func__, vp)); nandfs_itimes(vp); /* Basic info */ VATTR_NULL(vap); vap->va_atime.tv_sec = inode->i_mtime; vap->va_atime.tv_nsec = inode->i_mtime_nsec; vap->va_mtime.tv_sec = inode->i_mtime; vap->va_mtime.tv_nsec = inode->i_mtime_nsec; vap->va_ctime.tv_sec = inode->i_ctime; vap->va_ctime.tv_nsec = inode->i_ctime_nsec; vap->va_type = IFTOVT(inode->i_mode); vap->va_mode = inode->i_mode & ~S_IFMT; vap->va_nlink = inode->i_links_count; vap->va_uid = inode->i_uid; vap->va_gid = inode->i_gid; vap->va_rdev = inode->i_special; vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; vap->va_fileid = node->nn_ino; vap->va_size = inode->i_size; vap->va_blocksize = node->nn_nandfsdev->nd_blocksize; vap->va_gen = 0; vap->va_flags = inode->i_flags; vap->va_bytes = inode->i_blocks * vap->va_blocksize; vap->va_filerev = 0; vap->va_vaflags = 0; return (0); } static int nandfs_vtruncbuf(struct vnode *vp, uint64_t nblks) { struct nandfs_device *nffsdev; struct bufobj *bo; struct buf *bp, *nbp; bo = &vp->v_bufobj; nffsdev = VTON(vp)->nn_nandfsdev; ASSERT_VOP_LOCKED(vp, "nandfs_truncate"); restart: BO_LOCK(bo); restart_locked: TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) { if (bp->b_lblkno < nblks) continue; if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) goto restart_locked; bremfree(bp); bp->b_flags |= (B_INVAL | B_RELBUF); bp->b_flags &= ~(B_ASYNC | B_MANAGED); BO_UNLOCK(bo); brelse(bp); BO_LOCK(bo); } TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (bp->b_lblkno < nblks) continue; if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo)) == ENOLCK) goto restart; bp->b_flags |= (B_INVAL | B_RELBUF); bp->b_flags &= ~(B_ASYNC | B_MANAGED); brelse(bp); nandfs_dirty_bufs_decrement(nffsdev); BO_LOCK(bo); } BO_UNLOCK(bo); return (0); } static int nandfs_truncate(struct vnode *vp, uint64_t newsize) { struct nandfs_device *nffsdev; struct nandfs_node *node; struct nandfs_inode *inode; struct buf *bp = NULL; uint64_t oblks, nblks, vblk, size, rest; int error; node = VTON(vp); nffsdev = node->nn_nandfsdev; inode = &node->nn_inode; /* Calculate end of file */ size = inode->i_size; if (newsize == size) { node->nn_flags |= IN_CHANGE | IN_UPDATE; nandfs_itimes(vp); return (0); } if (newsize > size) { inode->i_size = newsize; vnode_pager_setsize(vp, newsize); node->nn_flags |= IN_CHANGE | IN_UPDATE; nandfs_itimes(vp); return (0); } nblks = howmany(newsize, nffsdev->nd_blocksize); oblks = howmany(size, nffsdev->nd_blocksize); rest = newsize % nffsdev->nd_blocksize; if (rest) { error = nandfs_bmap_lookup(node, nblks - 1, &vblk); if (error) return (error); if (vblk != 0) error = nandfs_bread(node, nblks - 1, NOCRED, 0, &bp); else error = nandfs_bcreate(node, nblks - 1, NOCRED, 0, &bp); if (error) { if (bp) brelse(bp); return (error); } bzero((char *)bp->b_data + rest, (u_int)(nffsdev->nd_blocksize - rest)); error = nandfs_dirty_buf(bp, 0); if (error) return (error); } DPRINTF(VNCALL, ("%s: vp %p oblks %jx nblks %jx\n", __func__, vp, oblks, nblks)); error = nandfs_bmap_truncate_mapping(node, oblks - 1, nblks - 1); if (error) { if (bp) nandfs_undirty_buf(bp); return (error); } error = nandfs_vtruncbuf(vp, nblks); if (error) { if (bp) nandfs_undirty_buf(bp); return (error); } inode->i_size = newsize; vnode_pager_setsize(vp, newsize); node->nn_flags |= IN_CHANGE | IN_UPDATE; nandfs_itimes(vp); return (error); } static void nandfs_itimes_locked(struct vnode *vp) { struct nandfs_node *node; struct nandfs_inode *inode; struct timespec ts; ASSERT_VI_LOCKED(vp, __func__); node = VTON(vp); inode = &node->nn_inode; if ((node->nn_flags & (IN_ACCESS | IN_CHANGE | IN_UPDATE)) == 0) return; if (((vp->v_mount->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND)) == 0) || (node->nn_flags & (IN_CHANGE | IN_UPDATE))) node->nn_flags |= IN_MODIFIED; vfs_timestamp(&ts); if (node->nn_flags & IN_UPDATE) { inode->i_mtime = ts.tv_sec; inode->i_mtime_nsec = ts.tv_nsec; } if (node->nn_flags & IN_CHANGE) { inode->i_ctime = ts.tv_sec; inode->i_ctime_nsec = ts.tv_nsec; } node->nn_flags &= ~(IN_ACCESS | IN_CHANGE | IN_UPDATE); } void nandfs_itimes(struct vnode *vp) { VI_LOCK(vp); nandfs_itimes_locked(vp); VI_UNLOCK(vp); } static int nandfs_chmod(struct vnode *vp, int mode, struct ucred *cred, struct thread *td) { struct nandfs_node *node = VTON(vp); struct nandfs_inode *inode = &node->nn_inode; uint16_t nmode; int error = 0; DPRINTF(VNCALL, ("%s: vp %p, mode %x, cred %p, td %p\n", __func__, vp, mode, cred, td)); /* * To modify the permissions on a file, must possess VADMIN * for that file. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) return (error); /* * Privileged processes may set the sticky bit on non-directories, * as well as set the setgid bit on a file with a group that the * process is not a member of. Both of these are allowed in * jail(8). */ if (vp->v_type != VDIR && (mode & S_ISTXT)) { if (priv_check_cred(cred, PRIV_VFS_STICKYFILE)) return (EFTYPE); } if (!groupmember(inode->i_gid, cred) && (mode & ISGID)) { error = priv_check_cred(cred, PRIV_VFS_SETGID); if (error) return (error); } /* * Deny setting setuid if we are not the file owner. */ if ((mode & ISUID) && inode->i_uid != cred->cr_uid) { error = priv_check_cred(cred, PRIV_VFS_ADMIN); if (error) return (error); } nmode = inode->i_mode; nmode &= ~ALLPERMS; nmode |= (mode & ALLPERMS); inode->i_mode = nmode; node->nn_flags |= IN_CHANGE; DPRINTF(VNCALL, ("%s: to mode %x\n", __func__, nmode)); return (error); } static int nandfs_chown(struct vnode *vp, uid_t uid, gid_t gid, struct ucred *cred, struct thread *td) { struct nandfs_node *node = VTON(vp); struct nandfs_inode *inode = &node->nn_inode; uid_t ouid; gid_t ogid; int error = 0; if (uid == (uid_t)VNOVAL) uid = inode->i_uid; if (gid == (gid_t)VNOVAL) gid = inode->i_gid; /* * To modify the ownership of a file, must possess VADMIN for that * file. */ if ((error = VOP_ACCESSX(vp, VWRITE_OWNER, cred, td))) return (error); /* * To change the owner of a file, or change the group of a file to a * group of which we are not a member, the caller must have * privilege. */ if (((uid != inode->i_uid && uid != cred->cr_uid) || (gid != inode->i_gid && !groupmember(gid, cred))) && (error = priv_check_cred(cred, PRIV_VFS_CHOWN))) return (error); ogid = inode->i_gid; ouid = inode->i_uid; inode->i_gid = gid; inode->i_uid = uid; node->nn_flags |= IN_CHANGE; if ((inode->i_mode & (ISUID | ISGID)) && (ouid != uid || ogid != gid)) { if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID)) inode->i_mode &= ~(ISUID | ISGID); } DPRINTF(VNCALL, ("%s: vp %p, cred %p, td %p - ret OK\n", __func__, vp, cred, td)); return (0); } static int nandfs_setattr(struct vop_setattr_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); struct nandfs_inode *inode = &node->nn_inode; struct vattr *vap = ap->a_vap; struct ucred *cred = ap->a_cred; struct thread *td = curthread; uint32_t flags; int error = 0; if ((vap->va_type != VNON) || (vap->va_nlink != VNOVAL) || (vap->va_fsid != VNOVAL) || (vap->va_fileid != VNOVAL) || (vap->va_blocksize != VNOVAL) || (vap->va_rdev != VNOVAL) || (vap->va_bytes != VNOVAL) || (vap->va_gen != VNOVAL)) { DPRINTF(VNCALL, ("%s: unsettable attribute\n", __func__)); return (EINVAL); } if (vap->va_flags != VNOVAL) { DPRINTF(VNCALL, ("%s: vp:%p td:%p flags:%lx\n", __func__, vp, td, vap->va_flags)); if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); /* * Callers may only modify the file flags on objects they * have VADMIN rights for. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) return (error); /* * Unprivileged processes are not permitted to unset system * flags, or modify flags if any system flags are set. * Privileged non-jail processes may not modify system flags * if securelevel > 0 and any existing system flags are set. * Privileged jail processes behave like privileged non-jail * processes if the PR_ALLOW_CHFLAGS permission bit is set; * otherwise, they behave like unprivileged processes. */ flags = inode->i_flags; if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS)) { if (flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) { error = securelevel_gt(cred, 0); if (error) return (error); } /* Snapshot flag cannot be set or cleared */ if (((vap->va_flags & SF_SNAPSHOT) != 0 && (flags & SF_SNAPSHOT) == 0) || ((vap->va_flags & SF_SNAPSHOT) == 0 && (flags & SF_SNAPSHOT) != 0)) return (EPERM); inode->i_flags = vap->va_flags; } else { if (flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) || (vap->va_flags & UF_SETTABLE) != vap->va_flags) return (EPERM); flags &= SF_SETTABLE; flags |= (vap->va_flags & UF_SETTABLE); inode->i_flags = flags; } node->nn_flags |= IN_CHANGE; if (vap->va_flags & (IMMUTABLE | APPEND)) return (0); } if (inode->i_flags & (IMMUTABLE | APPEND)) return (EPERM); if (vap->va_size != (u_quad_t)VNOVAL) { DPRINTF(VNCALL, ("%s: vp:%p td:%p size:%jx\n", __func__, vp, td, (uintmax_t)vap->va_size)); switch (vp->v_type) { case VDIR: return (EISDIR); case VLNK: case VREG: if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if ((inode->i_flags & SF_SNAPSHOT) != 0) return (EPERM); break; default: return (0); } if (vap->va_size > node->nn_nandfsdev->nd_maxfilesize) return (EFBIG); KASSERT((vp->v_type == VREG), ("Set size %d", vp->v_type)); nandfs_truncate(vp, vap->va_size); node->nn_flags |= IN_CHANGE; return (0); } if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); DPRINTF(VNCALL, ("%s: vp:%p td:%p uid/gid %x/%x\n", __func__, vp, td, vap->va_uid, vap->va_gid)); error = nandfs_chown(vp, vap->va_uid, vap->va_gid, cred, td); if (error) return (error); } if (vap->va_mode != (mode_t)VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); DPRINTF(VNCALL, ("%s: vp:%p td:%p mode %x\n", __func__, vp, td, vap->va_mode)); error = nandfs_chmod(vp, (int)vap->va_mode, cred, td); if (error) return (error); } if (vap->va_atime.tv_sec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || vap->va_birthtime.tv_sec != VNOVAL) { DPRINTF(VNCALL, ("%s: vp:%p td:%p time a/m/b %jx/%jx/%jx\n", __func__, vp, td, (uintmax_t)vap->va_atime.tv_sec, (uintmax_t)vap->va_mtime.tv_sec, (uintmax_t)vap->va_birthtime.tv_sec)); if (vap->va_atime.tv_sec != VNOVAL) node->nn_flags |= IN_ACCESS; if (vap->va_mtime.tv_sec != VNOVAL) node->nn_flags |= IN_CHANGE | IN_UPDATE; if (vap->va_birthtime.tv_sec != VNOVAL) node->nn_flags |= IN_MODIFIED; nandfs_itimes(vp); return (0); } return (0); } static int nandfs_open(struct vop_open_args *ap) { struct nandfs_node *node = VTON(ap->a_vp); uint64_t filesize; DPRINTF(VNCALL, ("nandfs_open called ap->a_mode %x\n", ap->a_mode)); if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) return (EOPNOTSUPP); if ((node->nn_inode.i_flags & APPEND) && (ap->a_mode & (FWRITE | O_APPEND)) == FWRITE) return (EPERM); filesize = node->nn_inode.i_size; vnode_create_vobject(ap->a_vp, filesize, ap->a_td); return (0); } static int nandfs_close(struct vop_close_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); DPRINTF(VNCALL, ("%s: vp %p node %p\n", __func__, vp, node)); mtx_lock(&vp->v_interlock); if (vp->v_usecount > 1) nandfs_itimes_locked(vp); mtx_unlock(&vp->v_interlock); return (0); } static int nandfs_check_possible(struct vnode *vp, struct vattr *vap, mode_t mode) { /* Check if we are allowed to write */ switch (vap->va_type) { case VDIR: case VLNK: case VREG: /* * Normal nodes: check if we're on a read-only mounted * filingsystem and bomb out if we're trying to write. */ if ((mode & VMODIFY_PERMS) && (vp->v_mount->mnt_flag & MNT_RDONLY)) return (EROFS); break; case VBLK: case VCHR: case VSOCK: case VFIFO: /* * Special nodes: even on read-only mounted filingsystems * these are allowed to be written to if permissions allow. */ break; default: /* No idea what this is */ return (EINVAL); } /* No one may write immutable files */ if ((mode & VWRITE) && (VTON(vp)->nn_inode.i_flags & IMMUTABLE)) return (EPERM); return (0); } static int nandfs_check_permitted(struct vnode *vp, struct vattr *vap, mode_t mode, struct ucred *cred) { return (vaccess(vp->v_type, vap->va_mode, vap->va_uid, vap->va_gid, mode, cred, NULL)); } static int nandfs_advlock(struct vop_advlock_args *ap) { struct nandfs_node *nvp; quad_t size; nvp = VTON(ap->a_vp); size = nvp->nn_inode.i_size; return (lf_advlock(ap, &(nvp->nn_lockf), size)); } static int nandfs_access(struct vop_access_args *ap) { struct vnode *vp = ap->a_vp; accmode_t accmode = ap->a_accmode; struct ucred *cred = ap->a_cred; struct vattr vap; int error; DPRINTF(VNCALL, ("%s: vp:%p mode: %x\n", __func__, vp, accmode)); error = VOP_GETATTR(vp, &vap, NULL); if (error) return (error); error = nandfs_check_possible(vp, &vap, accmode); if (error) return (error); error = nandfs_check_permitted(vp, &vap, accmode, cred); return (error); } static int nandfs_print(struct vop_print_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *nvp = VTON(vp); printf("\tvp=%p, nandfs_node=%p\n", vp, nvp); printf("nandfs inode %#jx\n", (uintmax_t)nvp->nn_ino); printf("flags = 0x%b\n", (u_int)nvp->nn_flags, PRINT_NODE_FLAGS); return (0); } static void nandfs_read_filebuf(struct nandfs_node *node, struct buf *bp) { struct nandfs_device *nandfsdev = node->nn_nandfsdev; struct buf *nbp; nandfs_daddr_t vblk, pblk; nandfs_lbn_t from; uint32_t blocksize; int error = 0; int blk2dev = nandfsdev->nd_blocksize / DEV_BSIZE; /* * Translate all the block sectors into a series of buffers to read * asynchronously from the nandfs device. Note that this lookup may * induce readin's too. */ blocksize = nandfsdev->nd_blocksize; if (bp->b_bcount / blocksize != 1) panic("invalid b_count in bp %p\n", bp); from = bp->b_blkno; DPRINTF(READ, ("\tread in from inode %#jx blkno %#jx" " count %#lx\n", (uintmax_t)node->nn_ino, from, bp->b_bcount)); /* Get virtual block numbers for the vnode's buffer span */ error = nandfs_bmap_lookup(node, from, &vblk); if (error) { bp->b_error = EINVAL; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return; } /* Translate virtual block numbers to physical block numbers */ error = nandfs_vtop(node, vblk, &pblk); if (error) { bp->b_error = EINVAL; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return; } /* Issue translated blocks */ bp->b_resid = bp->b_bcount; /* Note virtual block 0 marks not mapped */ if (vblk == 0) { vfs_bio_clrbuf(bp); bufdone(bp); return; } nbp = bp; nbp->b_blkno = pblk * blk2dev; bp->b_iooffset = dbtob(nbp->b_blkno); MPASS(bp->b_iooffset >= 0); BO_STRATEGY(&nandfsdev->nd_devvp->v_bufobj, nbp); nandfs_vblk_set(bp, vblk); DPRINTF(READ, ("read_filebuf : ino %#jx blk %#jx -> " "%#jx -> %#jx [bp %p]\n", (uintmax_t)node->nn_ino, (uintmax_t)(from), (uintmax_t)vblk, (uintmax_t)pblk, nbp)); } static void nandfs_write_filebuf(struct nandfs_node *node, struct buf *bp) { struct nandfs_device *nandfsdev = node->nn_nandfsdev; bp->b_iooffset = dbtob(bp->b_blkno); MPASS(bp->b_iooffset >= 0); BO_STRATEGY(&nandfsdev->nd_devvp->v_bufobj, bp); } static int nandfs_strategy(struct vop_strategy_args *ap) { struct vnode *vp = ap->a_vp; struct buf *bp = ap->a_bp; struct nandfs_node *node = VTON(vp); /* check if we ought to be here */ KASSERT((vp->v_type != VBLK && vp->v_type != VCHR), ("nandfs_strategy on type %d", vp->v_type)); /* Translate if needed and pass on */ if (bp->b_iocmd == BIO_READ) { nandfs_read_filebuf(node, bp); return (0); } /* Send to segment collector */ nandfs_write_filebuf(node, bp); return (0); } static int nandfs_readdir(struct vop_readdir_args *ap) { struct uio *uio = ap->a_uio; struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); struct nandfs_dir_entry *ndirent; struct dirent dirent; struct buf *bp; uint64_t file_size, diroffset, transoffset, blkoff; uint64_t blocknr; uint32_t blocksize = node->nn_nandfsdev->nd_blocksize; uint8_t *pos, name_len; int error; DPRINTF(READDIR, ("nandfs_readdir called\n")); if (vp->v_type != VDIR) return (ENOTDIR); file_size = node->nn_inode.i_size; DPRINTF(READDIR, ("nandfs_readdir filesize %jd resid %zd\n", (uintmax_t)file_size, uio->uio_resid )); /* We are called just as long as we keep on pushing data in */ error = 0; if ((uio->uio_offset < file_size) && (uio->uio_resid >= sizeof(struct dirent))) { diroffset = uio->uio_offset; transoffset = diroffset; blocknr = diroffset / blocksize; blkoff = diroffset % blocksize; error = nandfs_bread(node, blocknr, NOCRED, 0, &bp); if (error) { brelse(bp); return (EIO); } while (diroffset < file_size) { DPRINTF(READDIR, ("readdir : offset = %"PRIu64"\n", diroffset)); if (blkoff >= blocksize) { blkoff = 0; blocknr++; brelse(bp); error = nandfs_bread(node, blocknr, NOCRED, 0, &bp); if (error) { brelse(bp); return (EIO); } } /* Read in one dirent */ pos = (uint8_t *)bp->b_data + blkoff; ndirent = (struct nandfs_dir_entry *)pos; name_len = ndirent->name_len; memset(&dirent, 0, sizeof(dirent)); dirent.d_fileno = ndirent->inode; if (dirent.d_fileno) { dirent.d_type = ndirent->file_type; dirent.d_namlen = name_len; strncpy(dirent.d_name, ndirent->name, name_len); dirent.d_reclen = GENERIC_DIRSIZ(&dirent); /* NOTE: d_off is the offset of the *next* entry. */ dirent.d_off = diroffset + ndirent->rec_len; dirent_terminate(&dirent); DPRINTF(READDIR, ("copying `%*.*s`\n", name_len, name_len, dirent.d_name)); } /* * If there isn't enough space in the uio to return a * whole dirent, break off read */ if (uio->uio_resid < GENERIC_DIRSIZ(&dirent)) break; /* Transfer */ if (dirent.d_fileno) uiomove(&dirent, dirent.d_reclen, uio); /* Advance */ diroffset += ndirent->rec_len; blkoff += ndirent->rec_len; /* Remember the last entry we transferred */ transoffset = diroffset; } brelse(bp); /* Pass on last transferred offset */ uio->uio_offset = transoffset; } if (ap->a_eofflag) *ap->a_eofflag = (uio->uio_offset >= file_size); return (error); } static int nandfs_dirempty(struct vnode *dvp, uint64_t parentino, struct ucred *cred) { struct nandfs_node *dnode = VTON(dvp); struct nandfs_dir_entry *dirent; uint64_t file_size = dnode->nn_inode.i_size; uint64_t blockcount = dnode->nn_inode.i_blocks; uint64_t blocknr; uint32_t blocksize = dnode->nn_nandfsdev->nd_blocksize; uint32_t limit; uint32_t off; uint8_t *pos; struct buf *bp; int error; DPRINTF(LOOKUP, ("%s: dvp %p parentino %#jx cred %p\n", __func__, dvp, (uintmax_t)parentino, cred)); KASSERT((file_size != 0), ("nandfs_dirempty for NULL dir %p", dvp)); blocknr = 0; while (blocknr < blockcount) { error = nandfs_bread(dnode, blocknr, NOCRED, 0, &bp); if (error) { brelse(bp); return (0); } pos = (uint8_t *)bp->b_data; off = 0; if (blocknr == (blockcount - 1)) limit = file_size % blocksize; else limit = blocksize; while (off < limit) { dirent = (struct nandfs_dir_entry *)(pos + off); off += dirent->rec_len; if (dirent->inode == 0) continue; switch (dirent->name_len) { case 0: break; case 1: if (dirent->name[0] != '.') goto notempty; KASSERT(dirent->inode == dnode->nn_ino, (".'s inode does not match dir")); break; case 2: if (dirent->name[0] != '.' && dirent->name[1] != '.') goto notempty; KASSERT(dirent->inode == parentino, ("..'s inode does not match parent")); break; default: goto notempty; } } brelse(bp); blocknr++; } return (1); notempty: brelse(bp); return (0); } static int nandfs_link(struct vop_link_args *ap) { struct vnode *tdvp = ap->a_tdvp; struct vnode *vp = ap->a_vp; struct componentname *cnp = ap->a_cnp; struct nandfs_node *node = VTON(vp); struct nandfs_inode *inode = &node->nn_inode; int error; if (inode->i_links_count >= NANDFS_LINK_MAX) return (EMLINK); if (inode->i_flags & (IMMUTABLE | APPEND)) return (EPERM); /* Update link count */ inode->i_links_count++; /* Add dir entry */ error = nandfs_add_dirent(tdvp, node->nn_ino, cnp->cn_nameptr, cnp->cn_namelen, IFTODT(inode->i_mode)); if (error) { inode->i_links_count--; } node->nn_flags |= IN_CHANGE; nandfs_itimes(vp); DPRINTF(VNCALL, ("%s: tdvp %p vp %p cnp %p\n", __func__, tdvp, vp, cnp)); return (0); } static int nandfs_create(struct vop_create_args *ap) { struct vnode *dvp = ap->a_dvp; struct vnode **vpp = ap->a_vpp; struct componentname *cnp = ap->a_cnp; uint16_t mode = MAKEIMODE(ap->a_vap->va_type, ap->a_vap->va_mode); struct nandfs_node *dir_node = VTON(dvp); struct nandfsmount *nmp = dir_node->nn_nmp; struct nandfs_node *node; int error; DPRINTF(VNCALL, ("%s: dvp %p\n", __func__, dvp)); if (nandfs_fs_full(dir_node->nn_nandfsdev)) return (ENOSPC); /* Create new vnode/inode */ error = nandfs_node_create(nmp, &node, mode); if (error) return (error); node->nn_inode.i_gid = dir_node->nn_inode.i_gid; node->nn_inode.i_uid = cnp->cn_cred->cr_uid; /* Add new dir entry */ error = nandfs_add_dirent(dvp, node->nn_ino, cnp->cn_nameptr, cnp->cn_namelen, IFTODT(mode)); if (error) { if (nandfs_node_destroy(node)) { nandfs_error("%s: error destroying node %p\n", __func__, node); } return (error); } *vpp = NTOV(node); if ((cnp->cn_flags & MAKEENTRY) != 0) cache_enter(dvp, *vpp, cnp); DPRINTF(VNCALL, ("created file vp %p nandnode %p ino %jx\n", *vpp, node, (uintmax_t)node->nn_ino)); return (0); } static int nandfs_remove(struct vop_remove_args *ap) { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct nandfs_node *node = VTON(vp); struct nandfs_node *dnode = VTON(dvp); struct componentname *cnp = ap->a_cnp; DPRINTF(VNCALL, ("%s: dvp %p vp %p nandnode %p ino %#jx link %d\n", __func__, dvp, vp, node, (uintmax_t)node->nn_ino, node->nn_inode.i_links_count)); if (vp->v_type == VDIR) return (EISDIR); /* Files marked as immutable or append-only cannot be deleted. */ if ((node->nn_inode.i_flags & (IMMUTABLE | APPEND | NOUNLINK)) || (dnode->nn_inode.i_flags & APPEND)) return (EPERM); nandfs_remove_dirent(dvp, node, cnp); node->nn_inode.i_links_count--; node->nn_flags |= IN_CHANGE; return (0); } /* * Check if source directory is in the path of the target directory. * Target is supplied locked, source is unlocked. * The target is always vput before returning. */ static int nandfs_checkpath(struct nandfs_node *src, struct nandfs_node *dest, struct ucred *cred) { struct vnode *vp; int error, rootino; struct nandfs_dir_entry dirent; vp = NTOV(dest); if (src->nn_ino == dest->nn_ino) { error = EEXIST; goto out; } rootino = NANDFS_ROOT_INO; error = 0; if (dest->nn_ino == rootino) goto out; for (;;) { if (vp->v_type != VDIR) { error = ENOTDIR; break; } error = vn_rdwr(UIO_READ, vp, (caddr_t)&dirent, NANDFS_DIR_REC_LEN(2), (off_t)0, UIO_SYSSPACE, IO_NODELOCKED | IO_NOMACCHECK, cred, NOCRED, NULL, NULL); if (error != 0) break; if (dirent.name_len != 2 || dirent.name[0] != '.' || dirent.name[1] != '.') { error = ENOTDIR; break; } if (dirent.inode == src->nn_ino) { error = EINVAL; break; } if (dirent.inode == rootino) break; vput(vp); if ((error = VFS_VGET(vp->v_mount, dirent.inode, LK_EXCLUSIVE, &vp)) != 0) { vp = NULL; break; } } out: if (error == ENOTDIR) printf("checkpath: .. not a directory\n"); if (vp != NULL) vput(vp); return (error); } static int nandfs_rename(struct vop_rename_args *ap) { struct vnode *tvp = ap->a_tvp; struct vnode *tdvp = ap->a_tdvp; struct vnode *fvp = ap->a_fvp; struct vnode *fdvp = ap->a_fdvp; struct componentname *tcnp = ap->a_tcnp; struct componentname *fcnp = ap->a_fcnp; int doingdirectory = 0, oldparent = 0, newparent = 0; int error = 0; struct nandfs_node *fdnode, *fnode, *fnode1; struct nandfs_node *tdnode = VTON(tdvp); struct nandfs_node *tnode; uint32_t tdflags, fflags, fdflags; uint16_t mode; DPRINTF(VNCALL, ("%s: fdvp:%p fvp:%p tdvp:%p tdp:%p\n", __func__, fdvp, fvp, tdvp, tvp)); /* * Check for cross-device rename. */ if ((fvp->v_mount != tdvp->v_mount) || (tvp && (fvp->v_mount != tvp->v_mount))) { error = EXDEV; abortit: if (tdvp == tvp) vrele(tdvp); else vput(tdvp); if (tvp) vput(tvp); vrele(fdvp); vrele(fvp); return (error); } tdflags = tdnode->nn_inode.i_flags; if (tvp && ((VTON(tvp)->nn_inode.i_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (tdflags & APPEND))) { error = EPERM; goto abortit; } /* * Renaming a file to itself has no effect. The upper layers should * not call us in that case. Temporarily just warn if they do. */ if (fvp == tvp) { printf("nandfs_rename: fvp == tvp (can't happen)\n"); error = 0; goto abortit; } if ((error = vn_lock(fvp, LK_EXCLUSIVE)) != 0) goto abortit; fdnode = VTON(fdvp); fnode = VTON(fvp); if (fnode->nn_inode.i_links_count >= NANDFS_LINK_MAX) { VOP_UNLOCK(fvp, 0); error = EMLINK; goto abortit; } fflags = fnode->nn_inode.i_flags; fdflags = fdnode->nn_inode.i_flags; if ((fflags & (NOUNLINK | IMMUTABLE | APPEND)) || (fdflags & APPEND)) { VOP_UNLOCK(fvp, 0); error = EPERM; goto abortit; } mode = fnode->nn_inode.i_mode; if ((mode & S_IFMT) == S_IFDIR) { /* * Avoid ".", "..", and aliases of "." for obvious reasons. */ if ((fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.') || (fdvp == fvp) || ((fcnp->cn_flags | tcnp->cn_flags) & ISDOTDOT) || (fnode->nn_flags & IN_RENAME)) { VOP_UNLOCK(fvp, 0); error = EINVAL; goto abortit; } fnode->nn_flags |= IN_RENAME; doingdirectory = 1; DPRINTF(VNCALL, ("%s: doingdirectory dvp %p\n", __func__, tdvp)); oldparent = fdnode->nn_ino; } vrele(fdvp); tnode = NULL; if (tvp) tnode = VTON(tvp); /* * Bump link count on fvp while we are moving stuff around. If we * crash before completing the work, the link count may be wrong * but correctable. */ fnode->nn_inode.i_links_count++; /* Check for in path moving XXX */ error = VOP_ACCESS(fvp, VWRITE, tcnp->cn_cred, tcnp->cn_thread); VOP_UNLOCK(fvp, 0); if (oldparent != tdnode->nn_ino) newparent = tdnode->nn_ino; if (doingdirectory && newparent) { if (error) /* write access check above */ goto bad; if (tnode != NULL) vput(tvp); error = nandfs_checkpath(fnode, tdnode, tcnp->cn_cred); if (error) goto out; VREF(tdvp); error = relookup(tdvp, &tvp, tcnp); if (error) goto out; vrele(tdvp); tdnode = VTON(tdvp); tnode = NULL; if (tvp) tnode = VTON(tvp); } /* * If the target doesn't exist, link the target to the source and * unlink the source. Otherwise, rewrite the target directory to * reference the source and remove the original entry. */ if (tvp == NULL) { /* * Account for ".." in new directory. */ if (doingdirectory && fdvp != tdvp) tdnode->nn_inode.i_links_count++; DPRINTF(VNCALL, ("%s: new entry in dvp:%p\n", __func__, tdvp)); /* * Add name in new directory. */ error = nandfs_add_dirent(tdvp, fnode->nn_ino, tcnp->cn_nameptr, tcnp->cn_namelen, IFTODT(fnode->nn_inode.i_mode)); if (error) { if (doingdirectory && fdvp != tdvp) tdnode->nn_inode.i_links_count--; goto bad; } vput(tdvp); } else { /* * If the parent directory is "sticky", then the user must * own the parent directory, or the destination of the rename, * otherwise the destination may not be changed (except by * root). This implements append-only directories. */ if ((tdnode->nn_inode.i_mode & S_ISTXT) && tcnp->cn_cred->cr_uid != 0 && tcnp->cn_cred->cr_uid != tdnode->nn_inode.i_uid && tnode->nn_inode.i_uid != tcnp->cn_cred->cr_uid) { error = EPERM; goto bad; } /* * Target must be empty if a directory and have no links * to it. Also, ensure source and target are compatible * (both directories, or both not directories). */ mode = tnode->nn_inode.i_mode; if ((mode & S_IFMT) == S_IFDIR) { if (!nandfs_dirempty(tvp, tdnode->nn_ino, tcnp->cn_cred)) { error = ENOTEMPTY; goto bad; } if (!doingdirectory) { error = ENOTDIR; goto bad; } /* * Update name cache since directory is going away. */ cache_purge(tdvp); } else if (doingdirectory) { error = EISDIR; goto bad; } DPRINTF(VNCALL, ("%s: update entry dvp:%p\n", __func__, tdvp)); /* * Change name tcnp in tdvp to point at fvp. */ error = nandfs_update_dirent(tdvp, fnode, tnode); if (error) goto bad; if (doingdirectory && !newparent) tdnode->nn_inode.i_links_count--; vput(tdvp); tnode->nn_inode.i_links_count--; vput(tvp); tnode = NULL; } /* * Unlink the source. */ fcnp->cn_flags &= ~MODMASK; fcnp->cn_flags |= LOCKPARENT | LOCKLEAF; VREF(fdvp); error = relookup(fdvp, &fvp, fcnp); if (error == 0) vrele(fdvp); if (fvp != NULL) { fnode1 = VTON(fvp); fdnode = VTON(fdvp); } else { /* * From name has disappeared. */ if (doingdirectory) panic("nandfs_rename: lost dir entry"); vrele(ap->a_fvp); return (0); } DPRINTF(VNCALL, ("%s: unlink source fnode:%p\n", __func__, fnode)); /* * Ensure that the directory entry still exists and has not * changed while the new name has been entered. If the source is * a file then the entry may have been unlinked or renamed. In * either case there is no further work to be done. If the source * is a directory then it cannot have been rmdir'ed; its link * count of three would cause a rmdir to fail with ENOTEMPTY. * The IN_RENAME flag ensures that it cannot be moved by another * rename. */ if (fnode != fnode1) { if (doingdirectory) panic("nandfs: lost dir entry"); } else { /* * If the source is a directory with a * new parent, the link count of the old * parent directory must be decremented * and ".." set to point to the new parent. */ if (doingdirectory && newparent) { DPRINTF(VNCALL, ("%s: new parent %#jx -> %#jx\n", __func__, (uintmax_t) oldparent, (uintmax_t) newparent)); error = nandfs_update_parent_dir(fvp, newparent); if (!error) { fdnode->nn_inode.i_links_count--; fdnode->nn_flags |= IN_CHANGE; } } error = nandfs_remove_dirent(fdvp, fnode, fcnp); if (!error) { fnode->nn_inode.i_links_count--; fnode->nn_flags |= IN_CHANGE; } fnode->nn_flags &= ~IN_RENAME; } if (fdnode) vput(fdvp); if (fnode) vput(fvp); vrele(ap->a_fvp); return (error); bad: DPRINTF(VNCALL, ("%s: error:%d\n", __func__, error)); if (tnode) vput(NTOV(tnode)); vput(NTOV(tdnode)); out: if (doingdirectory) fnode->nn_flags &= ~IN_RENAME; if (vn_lock(fvp, LK_EXCLUSIVE) == 0) { fnode->nn_inode.i_links_count--; fnode->nn_flags |= IN_CHANGE; fnode->nn_flags &= ~IN_RENAME; vput(fvp); } else vrele(fvp); return (error); } static int nandfs_mkdir(struct vop_mkdir_args *ap) { struct vnode *dvp = ap->a_dvp; struct vnode **vpp = ap->a_vpp; struct componentname *cnp = ap->a_cnp; struct nandfs_node *dir_node = VTON(dvp); struct nandfs_inode *dir_inode = &dir_node->nn_inode; struct nandfs_node *node; struct nandfsmount *nmp = dir_node->nn_nmp; uint16_t mode = MAKEIMODE(ap->a_vap->va_type, ap->a_vap->va_mode); int error; DPRINTF(VNCALL, ("%s: dvp %p\n", __func__, dvp)); if (nandfs_fs_full(dir_node->nn_nandfsdev)) return (ENOSPC); if (dir_inode->i_links_count >= NANDFS_LINK_MAX) return (EMLINK); error = nandfs_node_create(nmp, &node, mode); if (error) return (error); node->nn_inode.i_gid = dir_node->nn_inode.i_gid; node->nn_inode.i_uid = cnp->cn_cred->cr_uid; *vpp = NTOV(node); error = nandfs_add_dirent(dvp, node->nn_ino, cnp->cn_nameptr, cnp->cn_namelen, IFTODT(mode)); if (error) { vput(*vpp); return (error); } dir_node->nn_inode.i_links_count++; dir_node->nn_flags |= IN_CHANGE; error = nandfs_init_dir(NTOV(node), node->nn_ino, dir_node->nn_ino); if (error) { vput(NTOV(node)); return (error); } DPRINTF(VNCALL, ("created dir vp %p nandnode %p ino %jx\n", *vpp, node, (uintmax_t)node->nn_ino)); return (0); } static int nandfs_mknod(struct vop_mknod_args *ap) { struct vnode *dvp = ap->a_dvp; struct vnode **vpp = ap->a_vpp; struct vattr *vap = ap->a_vap; uint16_t mode = MAKEIMODE(vap->va_type, vap->va_mode); struct componentname *cnp = ap->a_cnp; struct nandfs_node *dir_node = VTON(dvp); struct nandfsmount *nmp = dir_node->nn_nmp; struct nandfs_node *node; int error; if (nandfs_fs_full(dir_node->nn_nandfsdev)) return (ENOSPC); error = nandfs_node_create(nmp, &node, mode); if (error) return (error); node->nn_inode.i_gid = dir_node->nn_inode.i_gid; node->nn_inode.i_uid = cnp->cn_cred->cr_uid; if (vap->va_rdev != VNOVAL) node->nn_inode.i_special = vap->va_rdev; *vpp = NTOV(node); if (nandfs_add_dirent(dvp, node->nn_ino, cnp->cn_nameptr, cnp->cn_namelen, IFTODT(mode))) { vput(*vpp); return (ENOTDIR); } node->nn_flags |= IN_ACCESS | IN_CHANGE | IN_UPDATE; return (0); } static int nandfs_symlink(struct vop_symlink_args *ap) { struct vnode **vpp = ap->a_vpp; struct vnode *dvp = ap->a_dvp; uint16_t mode = MAKEIMODE(ap->a_vap->va_type, ap->a_vap->va_mode); struct componentname *cnp = ap->a_cnp; struct nandfs_node *dir_node = VTON(dvp); struct nandfsmount *nmp = dir_node->nn_nmp; struct nandfs_node *node; int len, error; if (nandfs_fs_full(dir_node->nn_nandfsdev)) return (ENOSPC); error = nandfs_node_create(nmp, &node, S_IFLNK | mode); if (error) return (error); node->nn_inode.i_gid = dir_node->nn_inode.i_gid; node->nn_inode.i_uid = cnp->cn_cred->cr_uid; *vpp = NTOV(node); if (nandfs_add_dirent(dvp, node->nn_ino, cnp->cn_nameptr, cnp->cn_namelen, IFTODT(mode))) { vput(*vpp); return (ENOTDIR); } len = strlen(ap->a_target); error = vn_rdwr(UIO_WRITE, *vpp, __DECONST(void *, ap->a_target), len, (off_t)0, UIO_SYSSPACE, IO_NODELOCKED | IO_NOMACCHECK, cnp->cn_cred, NOCRED, NULL, NULL); if (error) vput(*vpp); return (error); } static int nandfs_readlink(struct vop_readlink_args *ap) { struct vnode *vp = ap->a_vp; return (VOP_READ(vp, ap->a_uio, 0, ap->a_cred)); } static int nandfs_rmdir(struct vop_rmdir_args *ap) { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct nandfs_node *node, *dnode; uint32_t dflag, flag; int error = 0; node = VTON(vp); dnode = VTON(dvp); /* Files marked as immutable or append-only cannot be deleted. */ if ((node->nn_inode.i_flags & (IMMUTABLE | APPEND | NOUNLINK)) || (dnode->nn_inode.i_flags & APPEND)) return (EPERM); DPRINTF(VNCALL, ("%s: dvp %p vp %p nandnode %p ino %#jx\n", __func__, dvp, vp, node, (uintmax_t)node->nn_ino)); if (node->nn_inode.i_links_count < 2) return (EINVAL); if (!nandfs_dirempty(vp, dnode->nn_ino, cnp->cn_cred)) return (ENOTEMPTY); /* Files marked as immutable or append-only cannot be deleted. */ dflag = dnode->nn_inode.i_flags; flag = node->nn_inode.i_flags; if ((dflag & APPEND) || (flag & (NOUNLINK | IMMUTABLE | APPEND))) { return (EPERM); } if (vp->v_mountedhere != 0) return (EINVAL); nandfs_remove_dirent(dvp, node, cnp); dnode->nn_inode.i_links_count -= 1; dnode->nn_flags |= IN_CHANGE; cache_purge(dvp); error = nandfs_truncate(vp, (uint64_t)0); if (error) return (error); node->nn_inode.i_links_count -= 2; node->nn_flags |= IN_CHANGE; cache_purge(vp); return (error); } static int nandfs_fsync(struct vop_fsync_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); int locked; DPRINTF(VNCALL, ("%s: vp %p nandnode %p ino %#jx\n", __func__, vp, node, (uintmax_t)node->nn_ino)); /* * Start syncing vnode only if inode was modified or * there are some dirty buffers */ if (VTON(vp)->nn_flags & IN_MODIFIED || vp->v_bufobj.bo_dirty.bv_cnt) { locked = VOP_ISLOCKED(vp); VOP_UNLOCK(vp, 0); nandfs_wakeup_wait_sync(node->nn_nandfsdev, SYNCER_FSYNC); VOP_LOCK(vp, locked | LK_RETRY); } return (0); } static int nandfs_bmap(struct vop_bmap_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *nnode = VTON(vp); struct nandfs_device *nandfsdev = nnode->nn_nandfsdev; nandfs_daddr_t l2vmap, v2pmap; int error; int blk2dev = nandfsdev->nd_blocksize / DEV_BSIZE; DPRINTF(VNCALL, ("%s: vp %p nandnode %p ino %#jx\n", __func__, vp, nnode, (uintmax_t)nnode->nn_ino)); if (ap->a_bop != NULL) *ap->a_bop = &nandfsdev->nd_devvp->v_bufobj; if (ap->a_bnp == NULL) return (0); if (ap->a_runp != NULL) *ap->a_runp = 0; if (ap->a_runb != NULL) *ap->a_runb = 0; /* * Translate all the block sectors into a series of buffers to read * asynchronously from the nandfs device. Note that this lookup may * induce readin's too. */ /* Get virtual block numbers for the vnode's buffer span */ error = nandfs_bmap_lookup(nnode, ap->a_bn, &l2vmap); if (error) return (-1); /* Translate virtual block numbers to physical block numbers */ error = nandfs_vtop(nnode, l2vmap, &v2pmap); if (error) return (-1); /* Note virtual block 0 marks not mapped */ if (l2vmap == 0) *ap->a_bnp = -1; else *ap->a_bnp = v2pmap * blk2dev; /* in DEV_BSIZE */ DPRINTF(VNCALL, ("%s: vp %p nandnode %p ino %#jx lblk %jx -> blk %jx\n", __func__, vp, nnode, (uintmax_t)nnode->nn_ino, (uintmax_t)ap->a_bn, (uintmax_t)*ap->a_bnp )); return (0); } static void nandfs_force_syncer(struct nandfsmount *nmp) { nmp->nm_flags |= NANDFS_FORCE_SYNCER; nandfs_wakeup_wait_sync(nmp->nm_nandfsdev, SYNCER_FFORCE); } static int nandfs_ioctl(struct vop_ioctl_args *ap) { struct vnode *vp = ap->a_vp; u_long command = ap->a_command; caddr_t data = ap->a_data; struct nandfs_node *node = VTON(vp); struct nandfs_device *nandfsdev = node->nn_nandfsdev; struct nandfsmount *nmp = node->nn_nmp; uint64_t *tab, *cno; struct nandfs_seg_stat *nss; struct nandfs_cpmode *ncpm; struct nandfs_argv *nargv; struct nandfs_cpstat *ncp; int error; DPRINTF(VNCALL, ("%s: %x\n", __func__, (uint32_t)command)); error = priv_check(ap->a_td, PRIV_VFS_MOUNT); if (error) return (error); if (nmp->nm_ronly) { switch (command) { case NANDFS_IOCTL_GET_FSINFO: case NANDFS_IOCTL_GET_SUSTAT: case NANDFS_IOCTL_GET_CPINFO: case NANDFS_IOCTL_GET_CPSTAT: case NANDFS_IOCTL_GET_SUINFO: case NANDFS_IOCTL_GET_VINFO: case NANDFS_IOCTL_GET_BDESCS: break; default: return (EROFS); } } switch (command) { case NANDFS_IOCTL_GET_FSINFO: error = nandfs_get_fsinfo(nmp, (struct nandfs_fsinfo *)data); break; case NANDFS_IOCTL_GET_SUSTAT: nss = (struct nandfs_seg_stat *)data; error = nandfs_get_seg_stat(nandfsdev, nss); break; case NANDFS_IOCTL_CHANGE_CPMODE: ncpm = (struct nandfs_cpmode *)data; error = nandfs_chng_cpmode(nandfsdev->nd_cp_node, ncpm); nandfs_force_syncer(nmp); break; case NANDFS_IOCTL_GET_CPINFO: nargv = (struct nandfs_argv *)data; error = nandfs_get_cpinfo_ioctl(nandfsdev->nd_cp_node, nargv); break; case NANDFS_IOCTL_DELETE_CP: tab = (uint64_t *)data; error = nandfs_delete_cp(nandfsdev->nd_cp_node, tab[0], tab[1]); nandfs_force_syncer(nmp); break; case NANDFS_IOCTL_GET_CPSTAT: ncp = (struct nandfs_cpstat *)data; error = nandfs_get_cpstat(nandfsdev->nd_cp_node, ncp); break; case NANDFS_IOCTL_GET_SUINFO: nargv = (struct nandfs_argv *)data; error = nandfs_get_segment_info_ioctl(nandfsdev, nargv); break; case NANDFS_IOCTL_GET_VINFO: nargv = (struct nandfs_argv *)data; error = nandfs_get_dat_vinfo_ioctl(nandfsdev, nargv); break; case NANDFS_IOCTL_GET_BDESCS: nargv = (struct nandfs_argv *)data; error = nandfs_get_dat_bdescs_ioctl(nandfsdev, nargv); break; case NANDFS_IOCTL_SYNC: cno = (uint64_t *)data; nandfs_force_syncer(nmp); *cno = nandfsdev->nd_last_cno; error = 0; break; case NANDFS_IOCTL_MAKE_SNAP: cno = (uint64_t *)data; error = nandfs_make_snap(nandfsdev, cno); nandfs_force_syncer(nmp); break; case NANDFS_IOCTL_DELETE_SNAP: cno = (uint64_t *)data; error = nandfs_delete_snap(nandfsdev, *cno); nandfs_force_syncer(nmp); break; default: error = ENOTTY; break; } return (error); } /* * Whiteout vnode call */ static int nandfs_whiteout(struct vop_whiteout_args *ap) { struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; int error = 0; switch (ap->a_flags) { case LOOKUP: return (0); case CREATE: /* Create a new directory whiteout */ #ifdef INVARIANTS if ((cnp->cn_flags & SAVENAME) == 0) panic("nandfs_whiteout: missing name"); #endif error = nandfs_add_dirent(dvp, NANDFS_WHT_INO, cnp->cn_nameptr, cnp->cn_namelen, DT_WHT); break; case DELETE: /* Remove an existing directory whiteout */ cnp->cn_flags &= ~DOWHITEOUT; error = nandfs_remove_dirent(dvp, NULL, cnp); break; default: panic("nandf_whiteout: unknown op: %d", ap->a_flags); } return (error); } static int nandfs_pathconf(struct vop_pathconf_args *ap) { int error; error = 0; switch (ap->a_name) { case _PC_LINK_MAX: *ap->a_retval = NANDFS_LINK_MAX; break; case _PC_NAME_MAX: *ap->a_retval = NANDFS_NAME_LEN; break; case _PC_PIPE_BUF: if (ap->a_vp->v_type == VDIR || ap->a_vp->v_type == VFIFO) *ap->a_retval = PIPE_BUF; else error = EINVAL; break; case _PC_CHOWN_RESTRICTED: *ap->a_retval = 1; break; case _PC_NO_TRUNC: *ap->a_retval = 1; break; - case _PC_ACL_EXTENDED: - *ap->a_retval = 0; - break; case _PC_ALLOC_SIZE_MIN: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_bsize; break; case _PC_FILESIZEBITS: *ap->a_retval = 64; break; case _PC_REC_INCR_XFER_SIZE: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize; break; case _PC_REC_MAX_XFER_SIZE: *ap->a_retval = -1; /* means ``unlimited'' */ break; case _PC_REC_MIN_XFER_SIZE: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize; break; default: error = vop_stdpathconf(ap); break; } return (error); } static int nandfs_vnlock1(struct vop_lock1_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); int error, vi_locked; /* * XXX can vnode go away while we are sleeping? */ vi_locked = mtx_owned(&vp->v_interlock); if (vi_locked) VI_UNLOCK(vp); error = NANDFS_WRITELOCKFLAGS(node->nn_nandfsdev, ap->a_flags & LK_NOWAIT); if (vi_locked && !error) VI_LOCK(vp); if (error) return (error); error = vop_stdlock(ap); if (error) { NANDFS_WRITEUNLOCK(node->nn_nandfsdev); return (error); } return (0); } static int nandfs_vnunlock(struct vop_unlock_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); int error; error = vop_stdunlock(ap); if (error) return (error); NANDFS_WRITEUNLOCK(node->nn_nandfsdev); return (0); } /* * Global vfs data structures */ struct vop_vector nandfs_vnodeops = { .vop_default = &default_vnodeops, .vop_access = nandfs_access, .vop_advlock = nandfs_advlock, .vop_bmap = nandfs_bmap, .vop_close = nandfs_close, .vop_create = nandfs_create, .vop_fsync = nandfs_fsync, .vop_getattr = nandfs_getattr, .vop_inactive = nandfs_inactive, .vop_cachedlookup = nandfs_lookup, .vop_ioctl = nandfs_ioctl, .vop_link = nandfs_link, .vop_lookup = vfs_cache_lookup, .vop_mkdir = nandfs_mkdir, .vop_mknod = nandfs_mknod, .vop_open = nandfs_open, .vop_pathconf = nandfs_pathconf, .vop_print = nandfs_print, .vop_read = nandfs_read, .vop_readdir = nandfs_readdir, .vop_readlink = nandfs_readlink, .vop_reclaim = nandfs_reclaim, .vop_remove = nandfs_remove, .vop_rename = nandfs_rename, .vop_rmdir = nandfs_rmdir, .vop_whiteout = nandfs_whiteout, .vop_write = nandfs_write, .vop_setattr = nandfs_setattr, .vop_strategy = nandfs_strategy, .vop_symlink = nandfs_symlink, .vop_lock1 = nandfs_vnlock1, .vop_unlock = nandfs_vnunlock, }; struct vop_vector nandfs_system_vnodeops = { .vop_default = &default_vnodeops, .vop_close = nandfs_close, .vop_inactive = nandfs_inactive, .vop_reclaim = nandfs_reclaim, .vop_strategy = nandfs_strategy, .vop_fsync = nandfs_fsync, .vop_bmap = nandfs_bmap, .vop_access = VOP_PANIC, .vop_advlock = VOP_PANIC, .vop_create = VOP_PANIC, .vop_getattr = VOP_PANIC, .vop_cachedlookup = VOP_PANIC, .vop_ioctl = VOP_PANIC, .vop_link = VOP_PANIC, .vop_lookup = VOP_PANIC, .vop_mkdir = VOP_PANIC, .vop_mknod = VOP_PANIC, .vop_open = VOP_PANIC, .vop_pathconf = VOP_PANIC, .vop_print = VOP_PANIC, .vop_read = VOP_PANIC, .vop_readdir = VOP_PANIC, .vop_readlink = VOP_PANIC, .vop_remove = VOP_PANIC, .vop_rename = VOP_PANIC, .vop_rmdir = VOP_PANIC, .vop_whiteout = VOP_PANIC, .vop_write = VOP_PANIC, .vop_setattr = VOP_PANIC, .vop_symlink = VOP_PANIC, }; static int nandfsfifo_close(struct vop_close_args *ap) { struct vnode *vp = ap->a_vp; struct nandfs_node *node = VTON(vp); DPRINTF(VNCALL, ("%s: vp %p node %p\n", __func__, vp, node)); mtx_lock(&vp->v_interlock); if (vp->v_usecount > 1) nandfs_itimes_locked(vp); mtx_unlock(&vp->v_interlock); return (fifo_specops.vop_close(ap)); } struct vop_vector nandfs_fifoops = { .vop_default = &fifo_specops, .vop_fsync = VOP_PANIC, .vop_access = nandfs_access, .vop_close = nandfsfifo_close, .vop_getattr = nandfs_getattr, .vop_inactive = nandfs_inactive, .vop_pathconf = nandfs_pathconf, .vop_print = nandfs_print, .vop_read = VOP_PANIC, .vop_reclaim = nandfs_reclaim, .vop_setattr = nandfs_setattr, .vop_write = VOP_PANIC, .vop_lock1 = nandfs_vnlock1, .vop_unlock = nandfs_vnunlock, }; int nandfs_vinit(struct vnode *vp, uint64_t ino) { struct nandfs_node *node; ASSERT_VOP_LOCKED(vp, __func__); node = VTON(vp); /* Check if we're fetching the root */ if (ino == NANDFS_ROOT_INO) vp->v_vflag |= VV_ROOT; if (ino != NANDFS_GC_INO) vp->v_type = IFTOVT(node->nn_inode.i_mode); else vp->v_type = VREG; if (vp->v_type == VFIFO) vp->v_op = &nandfs_fifoops; return (0); } Index: projects/import-googletest-1.8.1/sys/fs/nfsclient/nfs_clvnops.c =================================================================== --- projects/import-googletest-1.8.1/sys/fs/nfsclient/nfs_clvnops.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/fs/nfsclient/nfs_clvnops.c (revision 345026) @@ -1,3570 +1,3564 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1989, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Rick Macklem at The University of Guelph. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from nfs_vnops.c 8.16 (Berkeley) 5/27/95 */ #include __FBSDID("$FreeBSD$"); /* * vnode op calls for Sun NFS version 2, 3 and 4 */ #include "opt_inet.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include dtrace_nfsclient_accesscache_flush_probe_func_t dtrace_nfscl_accesscache_flush_done_probe; uint32_t nfscl_accesscache_flush_done_id; dtrace_nfsclient_accesscache_get_probe_func_t dtrace_nfscl_accesscache_get_hit_probe, dtrace_nfscl_accesscache_get_miss_probe; uint32_t nfscl_accesscache_get_hit_id; uint32_t nfscl_accesscache_get_miss_id; dtrace_nfsclient_accesscache_load_probe_func_t dtrace_nfscl_accesscache_load_done_probe; uint32_t nfscl_accesscache_load_done_id; #endif /* !KDTRACE_HOOKS */ /* Defs */ #define TRUE 1 #define FALSE 0 extern struct nfsstatsv1 nfsstatsv1; extern int nfsrv_useacl; extern int nfscl_debuglevel; MALLOC_DECLARE(M_NEWNFSREQ); static vop_read_t nfsfifo_read; static vop_write_t nfsfifo_write; static vop_close_t nfsfifo_close; static int nfs_setattrrpc(struct vnode *, struct vattr *, struct ucred *, struct thread *); static vop_lookup_t nfs_lookup; static vop_create_t nfs_create; static vop_mknod_t nfs_mknod; static vop_open_t nfs_open; static vop_pathconf_t nfs_pathconf; static vop_close_t nfs_close; static vop_access_t nfs_access; static vop_getattr_t nfs_getattr; static vop_setattr_t nfs_setattr; static vop_read_t nfs_read; static vop_fsync_t nfs_fsync; static vop_remove_t nfs_remove; static vop_link_t nfs_link; static vop_rename_t nfs_rename; static vop_mkdir_t nfs_mkdir; static vop_rmdir_t nfs_rmdir; static vop_symlink_t nfs_symlink; static vop_readdir_t nfs_readdir; static vop_strategy_t nfs_strategy; static int nfs_lookitup(struct vnode *, char *, int, struct ucred *, struct thread *, struct nfsnode **); static int nfs_sillyrename(struct vnode *, struct vnode *, struct componentname *); static vop_access_t nfsspec_access; static vop_readlink_t nfs_readlink; static vop_print_t nfs_print; static vop_advlock_t nfs_advlock; static vop_advlockasync_t nfs_advlockasync; static vop_getacl_t nfs_getacl; static vop_setacl_t nfs_setacl; static vop_set_text_t nfs_set_text; /* * Global vfs data structures for nfs */ static struct vop_vector newnfs_vnodeops_nosig = { .vop_default = &default_vnodeops, .vop_access = nfs_access, .vop_advlock = nfs_advlock, .vop_advlockasync = nfs_advlockasync, .vop_close = nfs_close, .vop_create = nfs_create, .vop_fsync = nfs_fsync, .vop_getattr = nfs_getattr, .vop_getpages = ncl_getpages, .vop_putpages = ncl_putpages, .vop_inactive = ncl_inactive, .vop_link = nfs_link, .vop_lookup = nfs_lookup, .vop_mkdir = nfs_mkdir, .vop_mknod = nfs_mknod, .vop_open = nfs_open, .vop_pathconf = nfs_pathconf, .vop_print = nfs_print, .vop_read = nfs_read, .vop_readdir = nfs_readdir, .vop_readlink = nfs_readlink, .vop_reclaim = ncl_reclaim, .vop_remove = nfs_remove, .vop_rename = nfs_rename, .vop_rmdir = nfs_rmdir, .vop_setattr = nfs_setattr, .vop_strategy = nfs_strategy, .vop_symlink = nfs_symlink, .vop_write = ncl_write, .vop_getacl = nfs_getacl, .vop_setacl = nfs_setacl, .vop_set_text = nfs_set_text, }; static int nfs_vnodeops_bypass(struct vop_generic_args *a) { return (vop_sigdefer(&newnfs_vnodeops_nosig, a)); } struct vop_vector newnfs_vnodeops = { .vop_default = &default_vnodeops, .vop_bypass = nfs_vnodeops_bypass, }; static struct vop_vector newnfs_fifoops_nosig = { .vop_default = &fifo_specops, .vop_access = nfsspec_access, .vop_close = nfsfifo_close, .vop_fsync = nfs_fsync, .vop_getattr = nfs_getattr, .vop_inactive = ncl_inactive, .vop_pathconf = nfs_pathconf, .vop_print = nfs_print, .vop_read = nfsfifo_read, .vop_reclaim = ncl_reclaim, .vop_setattr = nfs_setattr, .vop_write = nfsfifo_write, }; static int nfs_fifoops_bypass(struct vop_generic_args *a) { return (vop_sigdefer(&newnfs_fifoops_nosig, a)); } struct vop_vector newnfs_fifoops = { .vop_default = &default_vnodeops, .vop_bypass = nfs_fifoops_bypass, }; static int nfs_mknodrpc(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, struct vattr *vap); static int nfs_removerpc(struct vnode *dvp, struct vnode *vp, char *name, int namelen, struct ucred *cred, struct thread *td); static int nfs_renamerpc(struct vnode *fdvp, struct vnode *fvp, char *fnameptr, int fnamelen, struct vnode *tdvp, struct vnode *tvp, char *tnameptr, int tnamelen, struct ucred *cred, struct thread *td); static int nfs_renameit(struct vnode *sdvp, struct vnode *svp, struct componentname *scnp, struct sillyrename *sp); /* * Global variables */ SYSCTL_DECL(_vfs_nfs); static int nfsaccess_cache_timeout = NFS_MAXATTRTIMO; SYSCTL_INT(_vfs_nfs, OID_AUTO, access_cache_timeout, CTLFLAG_RW, &nfsaccess_cache_timeout, 0, "NFS ACCESS cache timeout"); static int nfs_prime_access_cache = 0; SYSCTL_INT(_vfs_nfs, OID_AUTO, prime_access_cache, CTLFLAG_RW, &nfs_prime_access_cache, 0, "Prime NFS ACCESS cache when fetching attributes"); static int newnfs_commit_on_close = 0; SYSCTL_INT(_vfs_nfs, OID_AUTO, commit_on_close, CTLFLAG_RW, &newnfs_commit_on_close, 0, "write+commit on close, else only write"); static int nfs_clean_pages_on_close = 1; SYSCTL_INT(_vfs_nfs, OID_AUTO, clean_pages_on_close, CTLFLAG_RW, &nfs_clean_pages_on_close, 0, "NFS clean dirty pages on close"); int newnfs_directio_enable = 0; SYSCTL_INT(_vfs_nfs, OID_AUTO, nfs_directio_enable, CTLFLAG_RW, &newnfs_directio_enable, 0, "Enable NFS directio"); int nfs_keep_dirty_on_error; SYSCTL_INT(_vfs_nfs, OID_AUTO, nfs_keep_dirty_on_error, CTLFLAG_RW, &nfs_keep_dirty_on_error, 0, "Retry pageout if error returned"); /* * This sysctl allows other processes to mmap a file that has been opened * O_DIRECT by a process. In general, having processes mmap the file while * Direct IO is in progress can lead to Data Inconsistencies. But, we allow * this by default to prevent DoS attacks - to prevent a malicious user from * opening up files O_DIRECT preventing other users from mmap'ing these * files. "Protected" environments where stricter consistency guarantees are * required can disable this knob. The process that opened the file O_DIRECT * cannot mmap() the file, because mmap'ed IO on an O_DIRECT open() is not * meaningful. */ int newnfs_directio_allow_mmap = 1; SYSCTL_INT(_vfs_nfs, OID_AUTO, nfs_directio_allow_mmap, CTLFLAG_RW, &newnfs_directio_allow_mmap, 0, "Enable mmaped IO on file with O_DIRECT opens"); #define NFSACCESS_ALL (NFSACCESS_READ | NFSACCESS_MODIFY \ | NFSACCESS_EXTEND | NFSACCESS_EXECUTE \ | NFSACCESS_DELETE | NFSACCESS_LOOKUP) /* * SMP Locking Note : * The list of locks after the description of the lock is the ordering * of other locks acquired with the lock held. * np->n_mtx : Protects the fields in the nfsnode. VM Object Lock VI_MTX (acquired indirectly) * nmp->nm_mtx : Protects the fields in the nfsmount. rep->r_mtx * ncl_iod_mutex : Global lock, protects shared nfsiod state. * nfs_reqq_mtx : Global lock, protects the nfs_reqq list. nmp->nm_mtx rep->r_mtx * rep->r_mtx : Protects the fields in an nfsreq. */ static int nfs34_access_otw(struct vnode *vp, int wmode, struct thread *td, struct ucred *cred, u_int32_t *retmode) { int error = 0, attrflag, i, lrupos; u_int32_t rmode; struct nfsnode *np = VTONFS(vp); struct nfsvattr nfsva; error = nfsrpc_accessrpc(vp, wmode, cred, td, &nfsva, &attrflag, &rmode, NULL); if (attrflag) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (!error) { lrupos = 0; mtx_lock(&np->n_mtx); for (i = 0; i < NFS_ACCESSCACHESIZE; i++) { if (np->n_accesscache[i].uid == cred->cr_uid) { np->n_accesscache[i].mode = rmode; np->n_accesscache[i].stamp = time_second; break; } if (i > 0 && np->n_accesscache[i].stamp < np->n_accesscache[lrupos].stamp) lrupos = i; } if (i == NFS_ACCESSCACHESIZE) { np->n_accesscache[lrupos].uid = cred->cr_uid; np->n_accesscache[lrupos].mode = rmode; np->n_accesscache[lrupos].stamp = time_second; } mtx_unlock(&np->n_mtx); if (retmode != NULL) *retmode = rmode; KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, rmode, 0); } else if (NFS_ISV4(vp)) { error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); } #ifdef KDTRACE_HOOKS if (error != 0) KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, 0, error); #endif return (error); } /* * nfs access vnode op. * For nfs version 2, just return ok. File accesses may fail later. * For nfs version 3, use the access rpc to check accessibility. If file modes * are changed on the server, accesses might still fail later. */ static int nfs_access(struct vop_access_args *ap) { struct vnode *vp = ap->a_vp; int error = 0, i, gotahit; u_int32_t mode, wmode, rmode; int v34 = NFS_ISV34(vp); struct nfsnode *np = VTONFS(vp); /* * Disallow write attempts on filesystems mounted read-only; * unless the file is a socket, fifo, or a block or character * device resident on the filesystem. */ if ((ap->a_accmode & (VWRITE | VAPPEND | VWRITE_NAMED_ATTRS | VDELETE_CHILD | VWRITE_ATTRIBUTES | VDELETE | VWRITE_ACL | VWRITE_OWNER)) != 0 && (vp->v_mount->mnt_flag & MNT_RDONLY) != 0) { switch (vp->v_type) { case VREG: case VDIR: case VLNK: return (EROFS); default: break; } } /* * For nfs v3 or v4, check to see if we have done this recently, and if * so return our cached result instead of making an ACCESS call. * If not, do an access rpc, otherwise you are stuck emulating * ufs_access() locally using the vattr. This may not be correct, * since the server may apply other access criteria such as * client uid-->server uid mapping that we do not know about. */ if (v34) { if (ap->a_accmode & VREAD) mode = NFSACCESS_READ; else mode = 0; if (vp->v_type != VDIR) { if (ap->a_accmode & VWRITE) mode |= (NFSACCESS_MODIFY | NFSACCESS_EXTEND); if (ap->a_accmode & VAPPEND) mode |= NFSACCESS_EXTEND; if (ap->a_accmode & VEXEC) mode |= NFSACCESS_EXECUTE; if (ap->a_accmode & VDELETE) mode |= NFSACCESS_DELETE; } else { if (ap->a_accmode & VWRITE) mode |= (NFSACCESS_MODIFY | NFSACCESS_EXTEND); if (ap->a_accmode & VAPPEND) mode |= NFSACCESS_EXTEND; if (ap->a_accmode & VEXEC) mode |= NFSACCESS_LOOKUP; if (ap->a_accmode & VDELETE) mode |= NFSACCESS_DELETE; if (ap->a_accmode & VDELETE_CHILD) mode |= NFSACCESS_MODIFY; } /* XXX safety belt, only make blanket request if caching */ if (nfsaccess_cache_timeout > 0) { wmode = NFSACCESS_READ | NFSACCESS_MODIFY | NFSACCESS_EXTEND | NFSACCESS_EXECUTE | NFSACCESS_DELETE | NFSACCESS_LOOKUP; } else { wmode = mode; } /* * Does our cached result allow us to give a definite yes to * this request? */ gotahit = 0; mtx_lock(&np->n_mtx); for (i = 0; i < NFS_ACCESSCACHESIZE; i++) { if (ap->a_cred->cr_uid == np->n_accesscache[i].uid) { if (time_second < (np->n_accesscache[i].stamp + nfsaccess_cache_timeout) && (np->n_accesscache[i].mode & mode) == mode) { NFSINCRGLOBAL(nfsstatsv1.accesscache_hits); gotahit = 1; } break; } } mtx_unlock(&np->n_mtx); #ifdef KDTRACE_HOOKS if (gotahit != 0) KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, ap->a_cred->cr_uid, mode); else KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, ap->a_cred->cr_uid, mode); #endif if (gotahit == 0) { /* * Either a no, or a don't know. Go to the wire. */ NFSINCRGLOBAL(nfsstatsv1.accesscache_misses); error = nfs34_access_otw(vp, wmode, ap->a_td, ap->a_cred, &rmode); if (!error && (rmode & mode) != mode) error = EACCES; } return (error); } else { if ((error = nfsspec_access(ap)) != 0) { return (error); } /* * Attempt to prevent a mapped root from accessing a file * which it shouldn't. We try to read a byte from the file * if the user is root and the file is not zero length. * After calling nfsspec_access, we should have the correct * file size cached. */ mtx_lock(&np->n_mtx); if (ap->a_cred->cr_uid == 0 && (ap->a_accmode & VREAD) && VTONFS(vp)->n_size > 0) { struct iovec aiov; struct uio auio; char buf[1]; mtx_unlock(&np->n_mtx); aiov.iov_base = buf; aiov.iov_len = 1; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = 0; auio.uio_resid = 1; auio.uio_segflg = UIO_SYSSPACE; auio.uio_rw = UIO_READ; auio.uio_td = ap->a_td; if (vp->v_type == VREG) error = ncl_readrpc(vp, &auio, ap->a_cred); else if (vp->v_type == VDIR) { char* bp; bp = malloc(NFS_DIRBLKSIZ, M_TEMP, M_WAITOK); aiov.iov_base = bp; aiov.iov_len = auio.uio_resid = NFS_DIRBLKSIZ; error = ncl_readdirrpc(vp, &auio, ap->a_cred, ap->a_td); free(bp, M_TEMP); } else if (vp->v_type == VLNK) error = ncl_readlinkrpc(vp, &auio, ap->a_cred); else error = EACCES; } else mtx_unlock(&np->n_mtx); return (error); } } /* * nfs open vnode op * Check to see if the type is ok * and that deletion is not in progress. * For paged in text files, you will need to flush the page cache * if consistency is lost. */ /* ARGSUSED */ static int nfs_open(struct vop_open_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct vattr vattr; int error; int fmode = ap->a_mode; struct ucred *cred; if (vp->v_type != VREG && vp->v_type != VDIR && vp->v_type != VLNK) return (EOPNOTSUPP); /* * For NFSv4, we need to do the Open Op before cache validation, * so that we conform to RFC3530 Sec. 9.3.1. */ if (NFS_ISV4(vp)) { error = nfsrpc_open(vp, fmode, ap->a_cred, ap->a_td); if (error) { error = nfscl_maperr(ap->a_td, error, (uid_t)0, (gid_t)0); return (error); } } /* * Now, if this Open will be doing reading, re-validate/flush the * cache, so that Close/Open coherency is maintained. */ mtx_lock(&np->n_mtx); if (np->n_flag & NMODIFIED) { mtx_unlock(&np->n_mtx); error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1); if (error == EINTR || error == EIO) { if (NFS_ISV4(vp)) (void) nfsrpc_close(vp, 0, ap->a_td); return (error); } mtx_lock(&np->n_mtx); np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); if (vp->v_type == VDIR) np->n_direofoffset = 0; mtx_unlock(&np->n_mtx); error = VOP_GETATTR(vp, &vattr, ap->a_cred); if (error) { if (NFS_ISV4(vp)) (void) nfsrpc_close(vp, 0, ap->a_td); return (error); } mtx_lock(&np->n_mtx); np->n_mtime = vattr.va_mtime; if (NFS_ISV4(vp)) np->n_change = vattr.va_filerev; } else { mtx_unlock(&np->n_mtx); error = VOP_GETATTR(vp, &vattr, ap->a_cred); if (error) { if (NFS_ISV4(vp)) (void) nfsrpc_close(vp, 0, ap->a_td); return (error); } mtx_lock(&np->n_mtx); if ((NFS_ISV4(vp) && np->n_change != vattr.va_filerev) || NFS_TIMESPEC_COMPARE(&np->n_mtime, &vattr.va_mtime)) { if (vp->v_type == VDIR) np->n_direofoffset = 0; mtx_unlock(&np->n_mtx); error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1); if (error == EINTR || error == EIO) { if (NFS_ISV4(vp)) (void) nfsrpc_close(vp, 0, ap->a_td); return (error); } mtx_lock(&np->n_mtx); np->n_mtime = vattr.va_mtime; if (NFS_ISV4(vp)) np->n_change = vattr.va_filerev; } } /* * If the object has >= 1 O_DIRECT active opens, we disable caching. */ if (newnfs_directio_enable && (fmode & O_DIRECT) && (vp->v_type == VREG)) { if (np->n_directio_opens == 0) { mtx_unlock(&np->n_mtx); error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1); if (error) { if (NFS_ISV4(vp)) (void) nfsrpc_close(vp, 0, ap->a_td); return (error); } mtx_lock(&np->n_mtx); np->n_flag |= NNONCACHE; } np->n_directio_opens++; } /* If opened for writing via NFSv4.1 or later, mark that for pNFS. */ if (NFSHASPNFS(VFSTONFS(vp->v_mount)) && (fmode & FWRITE) != 0) np->n_flag |= NWRITEOPENED; /* * If this is an open for writing, capture a reference to the * credentials, so they can be used by ncl_putpages(). Using * these write credentials is preferable to the credentials of * whatever thread happens to be doing the VOP_PUTPAGES() since * the write RPCs are less likely to fail with EACCES. */ if ((fmode & FWRITE) != 0) { cred = np->n_writecred; np->n_writecred = crhold(ap->a_cred); } else cred = NULL; mtx_unlock(&np->n_mtx); if (cred != NULL) crfree(cred); vnode_create_vobject(vp, vattr.va_size, ap->a_td); return (0); } /* * nfs close vnode op * What an NFS client should do upon close after writing is a debatable issue. * Most NFS clients push delayed writes to the server upon close, basically for * two reasons: * 1 - So that any write errors may be reported back to the client process * doing the close system call. By far the two most likely errors are * NFSERR_NOSPC and NFSERR_DQUOT to indicate space allocation failure. * 2 - To put a worst case upper bound on cache inconsistency between * multiple clients for the file. * There is also a consistency problem for Version 2 of the protocol w.r.t. * not being able to tell if other clients are writing a file concurrently, * since there is no way of knowing if the changed modify time in the reply * is only due to the write for this client. * (NFS Version 3 provides weak cache consistency data in the reply that * should be sufficient to detect and handle this case.) * * The current code does the following: * for NFS Version 2 - play it safe and flush/invalidate all dirty buffers * for NFS Version 3 - flush dirty buffers to the server but don't invalidate * or commit them (this satisfies 1 and 2 except for the * case where the server crashes after this close but * before the commit RPC, which is felt to be "good * enough". Changing the last argument to ncl_flush() to * a 1 would force a commit operation, if it is felt a * commit is necessary now. * for NFS Version 4 - flush the dirty buffers and commit them, if * nfscl_mustflush() says this is necessary. * It is necessary if there is no write delegation held, * in order to satisfy open/close coherency. * If the file isn't cached on local stable storage, * it may be necessary in order to detect "out of space" * errors from the server, if the write delegation * issued by the server doesn't allow the file to grow. */ /* ARGSUSED */ static int nfs_close(struct vop_close_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct nfsvattr nfsva; struct ucred *cred; int error = 0, ret, localcred = 0; int fmode = ap->a_fflag; if (NFSCL_FORCEDISM(vp->v_mount)) return (0); /* * During shutdown, a_cred isn't valid, so just use root. */ if (ap->a_cred == NOCRED) { cred = newnfs_getcred(); localcred = 1; } else { cred = ap->a_cred; } if (vp->v_type == VREG) { /* * Examine and clean dirty pages, regardless of NMODIFIED. * This closes a major hole in close-to-open consistency. * We want to push out all dirty pages (and buffers) on * close, regardless of whether they were dirtied by * mmap'ed writes or via write(). */ if (nfs_clean_pages_on_close && vp->v_object) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, 0); VM_OBJECT_WUNLOCK(vp->v_object); } mtx_lock(&np->n_mtx); if (np->n_flag & NMODIFIED) { mtx_unlock(&np->n_mtx); if (NFS_ISV3(vp)) { /* * Under NFSv3 we have dirty buffers to dispose of. We * must flush them to the NFS server. We have the option * of waiting all the way through the commit rpc or just * waiting for the initial write. The default is to only * wait through the initial write so the data is in the * server's cache, which is roughly similar to the state * a standard disk subsystem leaves the file in on close(). * * We cannot clear the NMODIFIED bit in np->n_flag due to * potential races with other processes, and certainly * cannot clear it if we don't commit. * These races occur when there is no longer the old * traditional vnode locking implemented for Vnode Ops. */ int cm = newnfs_commit_on_close ? 1 : 0; error = ncl_flush(vp, MNT_WAIT, ap->a_td, cm, 0); /* np->n_flag &= ~NMODIFIED; */ } else if (NFS_ISV4(vp)) { if (nfscl_mustflush(vp) != 0) { int cm = newnfs_commit_on_close ? 1 : 0; error = ncl_flush(vp, MNT_WAIT, ap->a_td, cm, 0); /* * as above w.r.t races when clearing * NMODIFIED. * np->n_flag &= ~NMODIFIED; */ } } else { error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1); } mtx_lock(&np->n_mtx); } /* * Invalidate the attribute cache in all cases. * An open is going to fetch fresh attrs any way, other procs * on this node that have file open will be forced to do an * otw attr fetch, but this is safe. * --> A user found that their RPC count dropped by 20% when * this was commented out and I can't see any requirement * for it, so I've disabled it when negative lookups are * enabled. (What does this have to do with negative lookup * caching? Well nothing, except it was reported by the * same user that needed negative lookup caching and I wanted * there to be a way to disable it to see if it * is the cause of some caching/coherency issue that might * crop up.) */ if (VFSTONFS(vp->v_mount)->nm_negnametimeo == 0) { np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); } if (np->n_flag & NWRITEERR) { np->n_flag &= ~NWRITEERR; error = np->n_error; } mtx_unlock(&np->n_mtx); } if (NFS_ISV4(vp)) { /* * Get attributes so "change" is up to date. */ if (error == 0 && nfscl_mustflush(vp) != 0 && vp->v_type == VREG && (VFSTONFS(vp->v_mount)->nm_flag & NFSMNT_NOCTO) == 0) { ret = nfsrpc_getattr(vp, cred, ap->a_td, &nfsva, NULL); if (!ret) { np->n_change = nfsva.na_filerev; (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 0); } } /* * and do the close. */ ret = nfsrpc_close(vp, 0, ap->a_td); if (!error && ret) error = ret; if (error) error = nfscl_maperr(ap->a_td, error, (uid_t)0, (gid_t)0); } if (newnfs_directio_enable) KASSERT((np->n_directio_asyncwr == 0), ("nfs_close: dirty unflushed (%d) directio buffers\n", np->n_directio_asyncwr)); if (newnfs_directio_enable && (fmode & O_DIRECT) && (vp->v_type == VREG)) { mtx_lock(&np->n_mtx); KASSERT((np->n_directio_opens > 0), ("nfs_close: unexpectedly value (0) of n_directio_opens\n")); np->n_directio_opens--; if (np->n_directio_opens == 0) np->n_flag &= ~NNONCACHE; mtx_unlock(&np->n_mtx); } if (localcred) NFSFREECRED(cred); return (error); } /* * nfs getattr call from vfs. */ static int nfs_getattr(struct vop_getattr_args *ap) { struct vnode *vp = ap->a_vp; struct thread *td = curthread; /* XXX */ struct nfsnode *np = VTONFS(vp); int error = 0; struct nfsvattr nfsva; struct vattr *vap = ap->a_vap; struct vattr vattr; /* * Update local times for special files. */ mtx_lock(&np->n_mtx); if (np->n_flag & (NACC | NUPD)) np->n_flag |= NCHG; mtx_unlock(&np->n_mtx); /* * First look in the cache. */ if (ncl_getattrcache(vp, &vattr) == 0) { vap->va_type = vattr.va_type; vap->va_mode = vattr.va_mode; vap->va_nlink = vattr.va_nlink; vap->va_uid = vattr.va_uid; vap->va_gid = vattr.va_gid; vap->va_fsid = vattr.va_fsid; vap->va_fileid = vattr.va_fileid; vap->va_size = vattr.va_size; vap->va_blocksize = vattr.va_blocksize; vap->va_atime = vattr.va_atime; vap->va_mtime = vattr.va_mtime; vap->va_ctime = vattr.va_ctime; vap->va_gen = vattr.va_gen; vap->va_flags = vattr.va_flags; vap->va_rdev = vattr.va_rdev; vap->va_bytes = vattr.va_bytes; vap->va_filerev = vattr.va_filerev; /* * Get the local modify time for the case of a write * delegation. */ nfscl_deleggetmodtime(vp, &vap->va_mtime); return (0); } if (NFS_ISV34(vp) && nfs_prime_access_cache && nfsaccess_cache_timeout > 0) { NFSINCRGLOBAL(nfsstatsv1.accesscache_misses); nfs34_access_otw(vp, NFSACCESS_ALL, td, ap->a_cred, NULL); if (ncl_getattrcache(vp, ap->a_vap) == 0) { nfscl_deleggetmodtime(vp, &ap->a_vap->va_mtime); return (0); } } error = nfsrpc_getattr(vp, ap->a_cred, td, &nfsva, NULL); if (!error) error = nfscl_loadattrcache(&vp, &nfsva, vap, NULL, 0, 0); if (!error) { /* * Get the local modify time for the case of a write * delegation. */ nfscl_deleggetmodtime(vp, &vap->va_mtime); } else if (NFS_ISV4(vp)) { error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); } return (error); } /* * nfs setattr call. */ static int nfs_setattr(struct vop_setattr_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct thread *td = curthread; /* XXX */ struct vattr *vap = ap->a_vap; int error = 0; u_quad_t tsize; #ifndef nolint tsize = (u_quad_t)0; #endif /* * Setting of flags and marking of atimes are not supported. */ if (vap->va_flags != VNOVAL) return (EOPNOTSUPP); /* * Disallow write attempts if the filesystem is mounted read-only. */ if ((vap->va_flags != VNOVAL || vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || vap->va_mode != (mode_t)VNOVAL) && (vp->v_mount->mnt_flag & MNT_RDONLY)) return (EROFS); if (vap->va_size != VNOVAL) { switch (vp->v_type) { case VDIR: return (EISDIR); case VCHR: case VBLK: case VSOCK: case VFIFO: if (vap->va_mtime.tv_sec == VNOVAL && vap->va_atime.tv_sec == VNOVAL && vap->va_mode == (mode_t)VNOVAL && vap->va_uid == (uid_t)VNOVAL && vap->va_gid == (gid_t)VNOVAL) return (0); vap->va_size = VNOVAL; break; default: /* * Disallow write attempts if the filesystem is * mounted read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); /* * We run vnode_pager_setsize() early (why?), * we must set np->n_size now to avoid vinvalbuf * V_SAVE races that might setsize a lower * value. */ mtx_lock(&np->n_mtx); tsize = np->n_size; mtx_unlock(&np->n_mtx); error = ncl_meta_setsize(vp, ap->a_cred, td, vap->va_size); mtx_lock(&np->n_mtx); if (np->n_flag & NMODIFIED) { tsize = np->n_size; mtx_unlock(&np->n_mtx); error = ncl_vinvalbuf(vp, vap->va_size == 0 ? 0 : V_SAVE, td, 1); if (error != 0) { vnode_pager_setsize(vp, tsize); return (error); } /* * Call nfscl_delegmodtime() to set the modify time * locally, as required. */ nfscl_delegmodtime(vp); } else mtx_unlock(&np->n_mtx); /* * np->n_size has already been set to vap->va_size * in ncl_meta_setsize(). We must set it again since * nfs_loadattrcache() could be called through * ncl_meta_setsize() and could modify np->n_size. */ mtx_lock(&np->n_mtx); np->n_vattr.na_size = np->n_size = vap->va_size; mtx_unlock(&np->n_mtx); } } else { mtx_lock(&np->n_mtx); if ((vap->va_mtime.tv_sec != VNOVAL || vap->va_atime.tv_sec != VNOVAL) && (np->n_flag & NMODIFIED) && vp->v_type == VREG) { mtx_unlock(&np->n_mtx); error = ncl_vinvalbuf(vp, V_SAVE, td, 1); if (error == EINTR || error == EIO) return (error); } else mtx_unlock(&np->n_mtx); } error = nfs_setattrrpc(vp, vap, ap->a_cred, td); if (error && vap->va_size != VNOVAL) { mtx_lock(&np->n_mtx); np->n_size = np->n_vattr.na_size = tsize; vnode_pager_setsize(vp, tsize); mtx_unlock(&np->n_mtx); } return (error); } /* * Do an nfs setattr rpc. */ static int nfs_setattrrpc(struct vnode *vp, struct vattr *vap, struct ucred *cred, struct thread *td) { struct nfsnode *np = VTONFS(vp); int error, ret, attrflag, i; struct nfsvattr nfsva; if (NFS_ISV34(vp)) { mtx_lock(&np->n_mtx); for (i = 0; i < NFS_ACCESSCACHESIZE; i++) np->n_accesscache[i].stamp = 0; np->n_flag |= NDELEGMOD; mtx_unlock(&np->n_mtx); KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp); } error = nfsrpc_setattr(vp, vap, NULL, cred, td, &nfsva, &attrflag, NULL); if (attrflag) { ret = nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (ret && !error) error = ret; } if (error && NFS_ISV4(vp)) error = nfscl_maperr(td, error, vap->va_uid, vap->va_gid); return (error); } /* * nfs lookup call, one step at a time... * First look in cache * If not found, unlock the directory nfsnode and do the rpc */ static int nfs_lookup(struct vop_lookup_args *ap) { struct componentname *cnp = ap->a_cnp; struct vnode *dvp = ap->a_dvp; struct vnode **vpp = ap->a_vpp; struct mount *mp = dvp->v_mount; int flags = cnp->cn_flags; struct vnode *newvp; struct nfsmount *nmp; struct nfsnode *np, *newnp; int error = 0, attrflag, dattrflag, ltype, ncticks; struct thread *td = cnp->cn_thread; struct nfsfh *nfhp; struct nfsvattr dnfsva, nfsva; struct vattr vattr; struct timespec nctime; *vpp = NULLVP; if ((flags & ISLASTCN) && (mp->mnt_flag & MNT_RDONLY) && (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) return (EROFS); if (dvp->v_type != VDIR) return (ENOTDIR); nmp = VFSTONFS(mp); np = VTONFS(dvp); /* For NFSv4, wait until any remove is done. */ mtx_lock(&np->n_mtx); while (NFSHASNFSV4(nmp) && (np->n_flag & NREMOVEINPROG)) { np->n_flag |= NREMOVEWANT; (void) msleep((caddr_t)np, &np->n_mtx, PZERO, "nfslkup", 0); } mtx_unlock(&np->n_mtx); if ((error = VOP_ACCESS(dvp, VEXEC, cnp->cn_cred, td)) != 0) return (error); error = cache_lookup(dvp, vpp, cnp, &nctime, &ncticks); if (error > 0 && error != ENOENT) return (error); if (error == -1) { /* * Lookups of "." are special and always return the * current directory. cache_lookup() already handles * associated locking bookkeeping, etc. */ if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { /* XXX: Is this really correct? */ if (cnp->cn_nameiop != LOOKUP && (flags & ISLASTCN)) cnp->cn_flags |= SAVENAME; return (0); } /* * We only accept a positive hit in the cache if the * change time of the file matches our cached copy. * Otherwise, we discard the cache entry and fallback * to doing a lookup RPC. We also only trust cache * entries for less than nm_nametimeo seconds. * * To better handle stale file handles and attributes, * clear the attribute cache of this node if it is a * leaf component, part of an open() call, and not * locally modified before fetching the attributes. * This should allow stale file handles to be detected * here where we can fall back to a LOOKUP RPC to * recover rather than having nfs_open() detect the * stale file handle and failing open(2) with ESTALE. */ newvp = *vpp; newnp = VTONFS(newvp); if (!(nmp->nm_flag & NFSMNT_NOCTO) && (flags & (ISLASTCN | ISOPEN)) == (ISLASTCN | ISOPEN) && !(newnp->n_flag & NMODIFIED)) { mtx_lock(&newnp->n_mtx); newnp->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(newvp); mtx_unlock(&newnp->n_mtx); } if (nfscl_nodeleg(newvp, 0) == 0 || ((u_int)(ticks - ncticks) < (nmp->nm_nametimeo * hz) && VOP_GETATTR(newvp, &vattr, cnp->cn_cred) == 0 && timespeccmp(&vattr.va_ctime, &nctime, ==))) { NFSINCRGLOBAL(nfsstatsv1.lookupcache_hits); if (cnp->cn_nameiop != LOOKUP && (flags & ISLASTCN)) cnp->cn_flags |= SAVENAME; return (0); } cache_purge(newvp); if (dvp != newvp) vput(newvp); else vrele(newvp); *vpp = NULLVP; } else if (error == ENOENT) { if (dvp->v_iflag & VI_DOOMED) return (ENOENT); /* * We only accept a negative hit in the cache if the * modification time of the parent directory matches * the cached copy in the name cache entry. * Otherwise, we discard all of the negative cache * entries for this directory. We also only trust * negative cache entries for up to nm_negnametimeo * seconds. */ if ((u_int)(ticks - ncticks) < (nmp->nm_negnametimeo * hz) && VOP_GETATTR(dvp, &vattr, cnp->cn_cred) == 0 && timespeccmp(&vattr.va_mtime, &nctime, ==)) { NFSINCRGLOBAL(nfsstatsv1.lookupcache_hits); return (ENOENT); } cache_purge_negative(dvp); } error = 0; newvp = NULLVP; NFSINCRGLOBAL(nfsstatsv1.lookupcache_misses); error = nfsrpc_lookup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, td, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (dattrflag) (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); if (error) { if (newvp != NULLVP) { vput(newvp); *vpp = NULLVP; } if (error != ENOENT) { if (NFS_ISV4(dvp)) error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); return (error); } /* The requested file was not found. */ if ((cnp->cn_nameiop == CREATE || cnp->cn_nameiop == RENAME) && (flags & ISLASTCN)) { /* * XXX: UFS does a full VOP_ACCESS(dvp, * VWRITE) here instead of just checking * MNT_RDONLY. */ if (mp->mnt_flag & MNT_RDONLY) return (EROFS); cnp->cn_flags |= SAVENAME; return (EJUSTRETURN); } if ((cnp->cn_flags & MAKEENTRY) != 0 && dattrflag) { /* * Cache the modification time of the parent * directory from the post-op attributes in * the name cache entry. The negative cache * entry will be ignored once the directory * has changed. Don't bother adding the entry * if the directory has already changed. */ mtx_lock(&np->n_mtx); if (timespeccmp(&np->n_vattr.na_mtime, &dnfsva.na_mtime, ==)) { mtx_unlock(&np->n_mtx); cache_enter_time(dvp, NULL, cnp, &dnfsva.na_mtime, NULL); } else mtx_unlock(&np->n_mtx); } return (ENOENT); } /* * Handle RENAME case... */ if (cnp->cn_nameiop == RENAME && (flags & ISLASTCN)) { if (NFS_CMPFH(np, nfhp->nfh_fh, nfhp->nfh_len)) { free(nfhp, M_NFSFH); return (EISDIR); } error = nfscl_nget(mp, dvp, nfhp, cnp, td, &np, NULL, LK_EXCLUSIVE); if (error) return (error); newvp = NFSTOV(np); if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); *vpp = newvp; cnp->cn_flags |= SAVENAME; return (0); } if (flags & ISDOTDOT) { ltype = NFSVOPISLOCKED(dvp); error = vfs_busy(mp, MBF_NOWAIT); if (error != 0) { vfs_ref(mp); NFSVOPUNLOCK(dvp, 0); error = vfs_busy(mp, 0); NFSVOPLOCK(dvp, ltype | LK_RETRY); vfs_rel(mp); if (error == 0 && (dvp->v_iflag & VI_DOOMED)) { vfs_unbusy(mp); error = ENOENT; } if (error != 0) return (error); } NFSVOPUNLOCK(dvp, 0); error = nfscl_nget(mp, dvp, nfhp, cnp, td, &np, NULL, cnp->cn_lkflags); if (error == 0) newvp = NFSTOV(np); vfs_unbusy(mp); if (newvp != dvp) NFSVOPLOCK(dvp, ltype | LK_RETRY); if (dvp->v_iflag & VI_DOOMED) { if (error == 0) { if (newvp == dvp) vrele(newvp); else vput(newvp); } error = ENOENT; } if (error != 0) return (error); if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } else if (NFS_CMPFH(np, nfhp->nfh_fh, nfhp->nfh_len)) { free(nfhp, M_NFSFH); VREF(dvp); newvp = dvp; if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } else { error = nfscl_nget(mp, dvp, nfhp, cnp, td, &np, NULL, cnp->cn_lkflags); if (error) return (error); newvp = NFSTOV(np); if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); else if ((flags & (ISLASTCN | ISOPEN)) == (ISLASTCN | ISOPEN) && !(np->n_flag & NMODIFIED)) { /* * Flush the attribute cache when opening a * leaf node to ensure that fresh attributes * are fetched in nfs_open() since we did not * fetch attributes from the LOOKUP reply. */ mtx_lock(&np->n_mtx); np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(newvp); mtx_unlock(&np->n_mtx); } } if (cnp->cn_nameiop != LOOKUP && (flags & ISLASTCN)) cnp->cn_flags |= SAVENAME; if ((cnp->cn_flags & MAKEENTRY) && (cnp->cn_nameiop != DELETE || !(flags & ISLASTCN)) && attrflag != 0 && (newvp->v_type != VDIR || dattrflag != 0)) cache_enter_time(dvp, newvp, cnp, &nfsva.na_ctime, newvp->v_type != VDIR ? NULL : &dnfsva.na_ctime); *vpp = newvp; return (0); } /* * nfs read call. * Just call ncl_bioread() to do the work. */ static int nfs_read(struct vop_read_args *ap) { struct vnode *vp = ap->a_vp; switch (vp->v_type) { case VREG: return (ncl_bioread(vp, ap->a_uio, ap->a_ioflag, ap->a_cred)); case VDIR: return (EISDIR); default: return (EOPNOTSUPP); } } /* * nfs readlink call */ static int nfs_readlink(struct vop_readlink_args *ap) { struct vnode *vp = ap->a_vp; if (vp->v_type != VLNK) return (EINVAL); return (ncl_bioread(vp, ap->a_uio, 0, ap->a_cred)); } /* * Do a readlink rpc. * Called by ncl_doio() from below the buffer cache. */ int ncl_readlinkrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred) { int error, ret, attrflag; struct nfsvattr nfsva; error = nfsrpc_readlink(vp, uiop, cred, uiop->uio_td, &nfsva, &attrflag, NULL); if (attrflag) { ret = nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (ret && !error) error = ret; } if (error && NFS_ISV4(vp)) error = nfscl_maperr(uiop->uio_td, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs read rpc call * Ditto above */ int ncl_readrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred) { int error, ret, attrflag; struct nfsvattr nfsva; struct nfsmount *nmp; nmp = VFSTONFS(vnode_mount(vp)); error = EIO; attrflag = 0; if (NFSHASPNFS(nmp)) error = nfscl_doiods(vp, uiop, NULL, NULL, NFSV4OPEN_ACCESSREAD, 0, cred, uiop->uio_td); NFSCL_DEBUG(4, "readrpc: aft doiods=%d\n", error); if (error != 0) error = nfsrpc_read(vp, uiop, cred, uiop->uio_td, &nfsva, &attrflag, NULL); if (attrflag) { ret = nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (ret && !error) error = ret; } if (error && NFS_ISV4(vp)) error = nfscl_maperr(uiop->uio_td, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs write call */ int ncl_writerpc(struct vnode *vp, struct uio *uiop, struct ucred *cred, int *iomode, int *must_commit, int called_from_strategy) { struct nfsvattr nfsva; int error, attrflag, ret; struct nfsmount *nmp; nmp = VFSTONFS(vnode_mount(vp)); error = EIO; attrflag = 0; if (NFSHASPNFS(nmp)) error = nfscl_doiods(vp, uiop, iomode, must_commit, NFSV4OPEN_ACCESSWRITE, 0, cred, uiop->uio_td); NFSCL_DEBUG(4, "writerpc: aft doiods=%d\n", error); if (error != 0) error = nfsrpc_write(vp, uiop, iomode, must_commit, cred, uiop->uio_td, &nfsva, &attrflag, NULL, called_from_strategy); if (attrflag) { if (VTONFS(vp)->n_flag & ND_NFSV4) ret = nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 1, 1); else ret = nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (ret && !error) error = ret; } if (DOINGASYNC(vp)) *iomode = NFSWRITE_FILESYNC; if (error && NFS_ISV4(vp)) error = nfscl_maperr(uiop->uio_td, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs mknod rpc * For NFS v2 this is a kludge. Use a create rpc but with the IFMT bits of the * mode set to specify the file type and the size field for rdev. */ static int nfs_mknodrpc(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, struct vattr *vap) { struct nfsvattr nfsva, dnfsva; struct vnode *newvp = NULL; struct nfsnode *np = NULL, *dnp; struct nfsfh *nfhp; struct vattr vattr; int error = 0, attrflag, dattrflag; u_int32_t rdev; if (vap->va_type == VCHR || vap->va_type == VBLK) rdev = vap->va_rdev; else if (vap->va_type == VFIFO || vap->va_type == VSOCK) rdev = 0xffffffff; else return (EOPNOTSUPP); if ((error = VOP_GETATTR(dvp, &vattr, cnp->cn_cred))) return (error); error = nfsrpc_mknod(dvp, cnp->cn_nameptr, cnp->cn_namelen, vap, rdev, vap->va_type, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (!error) { if (!nfhp) (void) nfsrpc_lookup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (nfhp) error = nfscl_nget(dvp->v_mount, dvp, nfhp, cnp, cnp->cn_thread, &np, NULL, LK_EXCLUSIVE); } if (dattrflag) (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); if (!error) { newvp = NFSTOV(np); if (attrflag != 0) { error = nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); if (error != 0) vput(newvp); } } if (!error) { *vpp = newvp; } else if (NFS_ISV4(dvp)) { error = nfscl_maperr(cnp->cn_thread, error, vap->va_uid, vap->va_gid); } dnp = VTONFS(dvp); mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (!dattrflag) { dnp->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } mtx_unlock(&dnp->n_mtx); return (error); } /* * nfs mknod vop * just call nfs_mknodrpc() to do the work. */ /* ARGSUSED */ static int nfs_mknod(struct vop_mknod_args *ap) { return (nfs_mknodrpc(ap->a_dvp, ap->a_vpp, ap->a_cnp, ap->a_vap)); } static struct mtx nfs_cverf_mtx; MTX_SYSINIT(nfs_cverf_mtx, &nfs_cverf_mtx, "NFS create verifier mutex", MTX_DEF); static nfsquad_t nfs_get_cverf(void) { static nfsquad_t cverf; nfsquad_t ret; static int cverf_initialized = 0; mtx_lock(&nfs_cverf_mtx); if (cverf_initialized == 0) { cverf.lval[0] = arc4random(); cverf.lval[1] = arc4random(); cverf_initialized = 1; } else cverf.qval++; ret = cverf; mtx_unlock(&nfs_cverf_mtx); return (ret); } /* * nfs file create call */ static int nfs_create(struct vop_create_args *ap) { struct vnode *dvp = ap->a_dvp; struct vattr *vap = ap->a_vap; struct componentname *cnp = ap->a_cnp; struct nfsnode *np = NULL, *dnp; struct vnode *newvp = NULL; struct nfsmount *nmp; struct nfsvattr dnfsva, nfsva; struct nfsfh *nfhp; nfsquad_t cverf; int error = 0, attrflag, dattrflag, fmode = 0; struct vattr vattr; /* * Oops, not for me.. */ if (vap->va_type == VSOCK) return (nfs_mknodrpc(dvp, ap->a_vpp, cnp, vap)); if ((error = VOP_GETATTR(dvp, &vattr, cnp->cn_cred))) return (error); if (vap->va_vaflags & VA_EXCLUSIVE) fmode |= O_EXCL; dnp = VTONFS(dvp); nmp = VFSTONFS(vnode_mount(dvp)); again: /* For NFSv4, wait until any remove is done. */ mtx_lock(&dnp->n_mtx); while (NFSHASNFSV4(nmp) && (dnp->n_flag & NREMOVEINPROG)) { dnp->n_flag |= NREMOVEWANT; (void) msleep((caddr_t)dnp, &dnp->n_mtx, PZERO, "nfscrt", 0); } mtx_unlock(&dnp->n_mtx); cverf = nfs_get_cverf(); error = nfsrpc_create(dvp, cnp->cn_nameptr, cnp->cn_namelen, vap, cverf, fmode, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (!error) { if (nfhp == NULL) (void) nfsrpc_lookup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (nfhp != NULL) error = nfscl_nget(dvp->v_mount, dvp, nfhp, cnp, cnp->cn_thread, &np, NULL, LK_EXCLUSIVE); } if (dattrflag) (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); if (!error) { newvp = NFSTOV(np); if (attrflag == 0) error = nfsrpc_getattr(newvp, cnp->cn_cred, cnp->cn_thread, &nfsva, NULL); if (error == 0) error = nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } if (error) { if (newvp != NULL) { vput(newvp); newvp = NULL; } if (NFS_ISV34(dvp) && (fmode & O_EXCL) && error == NFSERR_NOTSUPP) { fmode &= ~O_EXCL; goto again; } } else if (NFS_ISV34(dvp) && (fmode & O_EXCL)) { if (nfscl_checksattr(vap, &nfsva)) { error = nfsrpc_setattr(newvp, vap, NULL, cnp->cn_cred, cnp->cn_thread, &nfsva, &attrflag, NULL); if (error && (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL)) { /* try again without setting uid/gid */ vap->va_uid = (uid_t)VNOVAL; vap->va_gid = (uid_t)VNOVAL; error = nfsrpc_setattr(newvp, vap, NULL, cnp->cn_cred, cnp->cn_thread, &nfsva, &attrflag, NULL); } if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); if (error != 0) vput(newvp); } } if (!error) { if ((cnp->cn_flags & MAKEENTRY) && attrflag) cache_enter_time(dvp, newvp, cnp, &nfsva.na_ctime, NULL); *ap->a_vpp = newvp; } else if (NFS_ISV4(dvp)) { error = nfscl_maperr(cnp->cn_thread, error, vap->va_uid, vap->va_gid); } mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (!dattrflag) { dnp->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } mtx_unlock(&dnp->n_mtx); return (error); } /* * nfs file remove call * To try and make nfs semantics closer to ufs semantics, a file that has * other processes using the vnode is renamed instead of removed and then * removed later on the last close. * - If v_usecount > 1 * If a rename is not already in the works * call nfs_sillyrename() to set it up * else * do the remove rpc */ static int nfs_remove(struct vop_remove_args *ap) { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct nfsnode *np = VTONFS(vp); int error = 0; struct vattr vattr; KASSERT((cnp->cn_flags & HASBUF) != 0, ("nfs_remove: no name")); KASSERT(vrefcnt(vp) > 0, ("nfs_remove: bad v_usecount")); if (vp->v_type == VDIR) error = EPERM; else if (vrefcnt(vp) == 1 || (np->n_sillyrename && VOP_GETATTR(vp, &vattr, cnp->cn_cred) == 0 && vattr.va_nlink > 1)) { /* * Purge the name cache so that the chance of a lookup for * the name succeeding while the remove is in progress is * minimized. Without node locking it can still happen, such * that an I/O op returns ESTALE, but since you get this if * another host removes the file.. */ cache_purge(vp); /* * throw away biocache buffers, mainly to avoid * unnecessary delayed writes later. */ error = ncl_vinvalbuf(vp, 0, cnp->cn_thread, 1); if (error != EINTR && error != EIO) /* Do the rpc */ error = nfs_removerpc(dvp, vp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread); /* * Kludge City: If the first reply to the remove rpc is lost.. * the reply to the retransmitted request will be ENOENT * since the file was in fact removed * Therefore, we cheat and return success. */ if (error == ENOENT) error = 0; } else if (!np->n_sillyrename) error = nfs_sillyrename(dvp, vp, cnp); mtx_lock(&np->n_mtx); np->n_attrstamp = 0; mtx_unlock(&np->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); return (error); } /* * nfs file remove rpc called from nfs_inactive */ int ncl_removeit(struct sillyrename *sp, struct vnode *vp) { /* * Make sure that the directory vnode is still valid. * XXX we should lock sp->s_dvp here. */ if (sp->s_dvp->v_type == VBAD) return (0); return (nfs_removerpc(sp->s_dvp, vp, sp->s_name, sp->s_namlen, sp->s_cred, NULL)); } /* * Nfs remove rpc, called from nfs_remove() and ncl_removeit(). */ static int nfs_removerpc(struct vnode *dvp, struct vnode *vp, char *name, int namelen, struct ucred *cred, struct thread *td) { struct nfsvattr dnfsva; struct nfsnode *dnp = VTONFS(dvp); int error = 0, dattrflag; mtx_lock(&dnp->n_mtx); dnp->n_flag |= NREMOVEINPROG; mtx_unlock(&dnp->n_mtx); error = nfsrpc_remove(dvp, name, namelen, vp, cred, td, &dnfsva, &dattrflag, NULL); mtx_lock(&dnp->n_mtx); if ((dnp->n_flag & NREMOVEWANT)) { dnp->n_flag &= ~(NREMOVEWANT | NREMOVEINPROG); mtx_unlock(&dnp->n_mtx); wakeup((caddr_t)dnp); } else { dnp->n_flag &= ~NREMOVEINPROG; mtx_unlock(&dnp->n_mtx); } if (dattrflag) (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (!dattrflag) { dnp->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } mtx_unlock(&dnp->n_mtx); if (error && NFS_ISV4(dvp)) error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs file rename call */ static int nfs_rename(struct vop_rename_args *ap) { struct vnode *fvp = ap->a_fvp; struct vnode *tvp = ap->a_tvp; struct vnode *fdvp = ap->a_fdvp; struct vnode *tdvp = ap->a_tdvp; struct componentname *tcnp = ap->a_tcnp; struct componentname *fcnp = ap->a_fcnp; struct nfsnode *fnp = VTONFS(ap->a_fvp); struct nfsnode *tdnp = VTONFS(ap->a_tdvp); struct nfsv4node *newv4 = NULL; int error; KASSERT((tcnp->cn_flags & HASBUF) != 0 && (fcnp->cn_flags & HASBUF) != 0, ("nfs_rename: no name")); /* Check for cross-device rename */ if ((fvp->v_mount != tdvp->v_mount) || (tvp && (fvp->v_mount != tvp->v_mount))) { error = EXDEV; goto out; } if (fvp == tvp) { printf("nfs_rename: fvp == tvp (can't happen)\n"); error = 0; goto out; } if ((error = NFSVOPLOCK(fvp, LK_EXCLUSIVE)) != 0) goto out; /* * We have to flush B_DELWRI data prior to renaming * the file. If we don't, the delayed-write buffers * can be flushed out later after the file has gone stale * under NFSV3. NFSV2 does not have this problem because * ( as far as I can tell ) it flushes dirty buffers more * often. * * Skip the rename operation if the fsync fails, this can happen * due to the server's volume being full, when we pushed out data * that was written back to our cache earlier. Not checking for * this condition can result in potential (silent) data loss. */ error = VOP_FSYNC(fvp, MNT_WAIT, fcnp->cn_thread); NFSVOPUNLOCK(fvp, 0); if (!error && tvp) error = VOP_FSYNC(tvp, MNT_WAIT, tcnp->cn_thread); if (error) goto out; /* * If the tvp exists and is in use, sillyrename it before doing the * rename of the new file over it. * XXX Can't sillyrename a directory. */ if (tvp && vrefcnt(tvp) > 1 && !VTONFS(tvp)->n_sillyrename && tvp->v_type != VDIR && !nfs_sillyrename(tdvp, tvp, tcnp)) { vput(tvp); tvp = NULL; } error = nfs_renamerpc(fdvp, fvp, fcnp->cn_nameptr, fcnp->cn_namelen, tdvp, tvp, tcnp->cn_nameptr, tcnp->cn_namelen, tcnp->cn_cred, tcnp->cn_thread); if (error == 0 && NFS_ISV4(tdvp)) { /* * For NFSv4, check to see if it is the same name and * replace the name, if it is different. */ newv4 = malloc( sizeof (struct nfsv4node) + tdnp->n_fhp->nfh_len + tcnp->cn_namelen - 1, M_NFSV4NODE, M_WAITOK); mtx_lock(&tdnp->n_mtx); mtx_lock(&fnp->n_mtx); if (fnp->n_v4 != NULL && fvp->v_type == VREG && (fnp->n_v4->n4_namelen != tcnp->cn_namelen || NFSBCMP(tcnp->cn_nameptr, NFS4NODENAME(fnp->n_v4), tcnp->cn_namelen) || tdnp->n_fhp->nfh_len != fnp->n_v4->n4_fhlen || NFSBCMP(tdnp->n_fhp->nfh_fh, fnp->n_v4->n4_data, tdnp->n_fhp->nfh_len))) { #ifdef notdef { char nnn[100]; int nnnl; nnnl = (tcnp->cn_namelen < 100) ? tcnp->cn_namelen : 99; bcopy(tcnp->cn_nameptr, nnn, nnnl); nnn[nnnl] = '\0'; printf("ren replace=%s\n",nnn); } #endif free(fnp->n_v4, M_NFSV4NODE); fnp->n_v4 = newv4; newv4 = NULL; fnp->n_v4->n4_fhlen = tdnp->n_fhp->nfh_len; fnp->n_v4->n4_namelen = tcnp->cn_namelen; NFSBCOPY(tdnp->n_fhp->nfh_fh, fnp->n_v4->n4_data, tdnp->n_fhp->nfh_len); NFSBCOPY(tcnp->cn_nameptr, NFS4NODENAME(fnp->n_v4), tcnp->cn_namelen); } mtx_unlock(&tdnp->n_mtx); mtx_unlock(&fnp->n_mtx); if (newv4 != NULL) free(newv4, M_NFSV4NODE); } if (fvp->v_type == VDIR) { if (tvp != NULL && tvp->v_type == VDIR) cache_purge(tdvp); cache_purge(fdvp); } out: if (tdvp == tvp) vrele(tdvp); else vput(tdvp); if (tvp) vput(tvp); vrele(fdvp); vrele(fvp); /* * Kludge: Map ENOENT => 0 assuming that it is a reply to a retry. */ if (error == ENOENT) error = 0; return (error); } /* * nfs file rename rpc called from nfs_remove() above */ static int nfs_renameit(struct vnode *sdvp, struct vnode *svp, struct componentname *scnp, struct sillyrename *sp) { return (nfs_renamerpc(sdvp, svp, scnp->cn_nameptr, scnp->cn_namelen, sdvp, NULL, sp->s_name, sp->s_namlen, scnp->cn_cred, scnp->cn_thread)); } /* * Do an nfs rename rpc. Called from nfs_rename() and nfs_renameit(). */ static int nfs_renamerpc(struct vnode *fdvp, struct vnode *fvp, char *fnameptr, int fnamelen, struct vnode *tdvp, struct vnode *tvp, char *tnameptr, int tnamelen, struct ucred *cred, struct thread *td) { struct nfsvattr fnfsva, tnfsva; struct nfsnode *fdnp = VTONFS(fdvp); struct nfsnode *tdnp = VTONFS(tdvp); int error = 0, fattrflag, tattrflag; error = nfsrpc_rename(fdvp, fvp, fnameptr, fnamelen, tdvp, tvp, tnameptr, tnamelen, cred, td, &fnfsva, &tnfsva, &fattrflag, &tattrflag, NULL, NULL); mtx_lock(&fdnp->n_mtx); fdnp->n_flag |= NMODIFIED; if (fattrflag != 0) { mtx_unlock(&fdnp->n_mtx); (void) nfscl_loadattrcache(&fdvp, &fnfsva, NULL, NULL, 0, 1); } else { fdnp->n_attrstamp = 0; mtx_unlock(&fdnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(fdvp); } mtx_lock(&tdnp->n_mtx); tdnp->n_flag |= NMODIFIED; if (tattrflag != 0) { mtx_unlock(&tdnp->n_mtx); (void) nfscl_loadattrcache(&tdvp, &tnfsva, NULL, NULL, 0, 1); } else { tdnp->n_attrstamp = 0; mtx_unlock(&tdnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp); } if (error && NFS_ISV4(fdvp)) error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs hard link create call */ static int nfs_link(struct vop_link_args *ap) { struct vnode *vp = ap->a_vp; struct vnode *tdvp = ap->a_tdvp; struct componentname *cnp = ap->a_cnp; struct nfsnode *np, *tdnp; struct nfsvattr nfsva, dnfsva; int error = 0, attrflag, dattrflag; /* * Push all writes to the server, so that the attribute cache * doesn't get "out of sync" with the server. * XXX There should be a better way! */ VOP_FSYNC(vp, MNT_WAIT, cnp->cn_thread); error = nfsrpc_link(tdvp, vp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &attrflag, &dattrflag, NULL); tdnp = VTONFS(tdvp); mtx_lock(&tdnp->n_mtx); tdnp->n_flag |= NMODIFIED; if (dattrflag != 0) { mtx_unlock(&tdnp->n_mtx); (void) nfscl_loadattrcache(&tdvp, &dnfsva, NULL, NULL, 0, 1); } else { tdnp->n_attrstamp = 0; mtx_unlock(&tdnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp); } if (attrflag) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); else { np = VTONFS(vp); mtx_lock(&np->n_mtx); np->n_attrstamp = 0; mtx_unlock(&np->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); } /* * If negative lookup caching is enabled, I might as well * add an entry for this node. Not necessary for correctness, * but if negative caching is enabled, then the system * must care about lookup caching hit rate, so... */ if (VFSTONFS(vp->v_mount)->nm_negnametimeo != 0 && (cnp->cn_flags & MAKEENTRY) && attrflag != 0 && error == 0) { cache_enter_time(tdvp, vp, cnp, &nfsva.na_ctime, NULL); } if (error && NFS_ISV4(vp)) error = nfscl_maperr(cnp->cn_thread, error, (uid_t)0, (gid_t)0); return (error); } /* * nfs symbolic link create call */ static int nfs_symlink(struct vop_symlink_args *ap) { struct vnode *dvp = ap->a_dvp; struct vattr *vap = ap->a_vap; struct componentname *cnp = ap->a_cnp; struct nfsvattr nfsva, dnfsva; struct nfsfh *nfhp; struct nfsnode *np = NULL, *dnp; struct vnode *newvp = NULL; int error = 0, attrflag, dattrflag, ret; vap->va_type = VLNK; error = nfsrpc_symlink(dvp, cnp->cn_nameptr, cnp->cn_namelen, ap->a_target, vap, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (nfhp) { ret = nfscl_nget(dvp->v_mount, dvp, nfhp, cnp, cnp->cn_thread, &np, NULL, LK_EXCLUSIVE); if (!ret) newvp = NFSTOV(np); else if (!error) error = ret; } if (newvp != NULL) { if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } else if (!error) { /* * If we do not have an error and we could not extract the * newvp from the response due to the request being NFSv2, we * have to do a lookup in order to obtain a newvp to return. */ error = nfs_lookitup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &np); if (!error) newvp = NFSTOV(np); } if (error) { if (newvp) vput(newvp); if (NFS_ISV4(dvp)) error = nfscl_maperr(cnp->cn_thread, error, vap->va_uid, vap->va_gid); } else { *ap->a_vpp = newvp; } dnp = VTONFS(dvp); mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (dattrflag != 0) { mtx_unlock(&dnp->n_mtx); (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); } else { dnp->n_attrstamp = 0; mtx_unlock(&dnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } /* * If negative lookup caching is enabled, I might as well * add an entry for this node. Not necessary for correctness, * but if negative caching is enabled, then the system * must care about lookup caching hit rate, so... */ if (VFSTONFS(dvp->v_mount)->nm_negnametimeo != 0 && (cnp->cn_flags & MAKEENTRY) && attrflag != 0 && error == 0) { cache_enter_time(dvp, newvp, cnp, &nfsva.na_ctime, NULL); } return (error); } /* * nfs make dir call */ static int nfs_mkdir(struct vop_mkdir_args *ap) { struct vnode *dvp = ap->a_dvp; struct vattr *vap = ap->a_vap; struct componentname *cnp = ap->a_cnp; struct nfsnode *np = NULL, *dnp; struct vnode *newvp = NULL; struct vattr vattr; struct nfsfh *nfhp; struct nfsvattr nfsva, dnfsva; int error = 0, attrflag, dattrflag, ret; if ((error = VOP_GETATTR(dvp, &vattr, cnp->cn_cred)) != 0) return (error); vap->va_type = VDIR; error = nfsrpc_mkdir(dvp, cnp->cn_nameptr, cnp->cn_namelen, vap, cnp->cn_cred, cnp->cn_thread, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); dnp = VTONFS(dvp); mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (dattrflag != 0) { mtx_unlock(&dnp->n_mtx); (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); } else { dnp->n_attrstamp = 0; mtx_unlock(&dnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } if (nfhp) { ret = nfscl_nget(dvp->v_mount, dvp, nfhp, cnp, cnp->cn_thread, &np, NULL, LK_EXCLUSIVE); if (!ret) { newvp = NFSTOV(np); if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } else if (!error) error = ret; } if (!error && newvp == NULL) { error = nfs_lookitup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &np); if (!error) { newvp = NFSTOV(np); if (newvp->v_type != VDIR) error = EEXIST; } } if (error) { if (newvp) vput(newvp); if (NFS_ISV4(dvp)) error = nfscl_maperr(cnp->cn_thread, error, vap->va_uid, vap->va_gid); } else { /* * If negative lookup caching is enabled, I might as well * add an entry for this node. Not necessary for correctness, * but if negative caching is enabled, then the system * must care about lookup caching hit rate, so... */ if (VFSTONFS(dvp->v_mount)->nm_negnametimeo != 0 && (cnp->cn_flags & MAKEENTRY) && attrflag != 0 && dattrflag != 0) cache_enter_time(dvp, newvp, cnp, &nfsva.na_ctime, &dnfsva.na_ctime); *ap->a_vpp = newvp; } return (error); } /* * nfs remove directory call */ static int nfs_rmdir(struct vop_rmdir_args *ap) { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct nfsnode *dnp; struct nfsvattr dnfsva; int error, dattrflag; if (dvp == vp) return (EINVAL); error = nfsrpc_rmdir(dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_cred, cnp->cn_thread, &dnfsva, &dattrflag, NULL); dnp = VTONFS(dvp); mtx_lock(&dnp->n_mtx); dnp->n_flag |= NMODIFIED; if (dattrflag != 0) { mtx_unlock(&dnp->n_mtx); (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); } else { dnp->n_attrstamp = 0; mtx_unlock(&dnp->n_mtx); KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp); } cache_purge(dvp); cache_purge(vp); if (error && NFS_ISV4(dvp)) error = nfscl_maperr(cnp->cn_thread, error, (uid_t)0, (gid_t)0); /* * Kludge: Map ENOENT => 0 assuming that you have a reply to a retry. */ if (error == ENOENT) error = 0; return (error); } /* * nfs readdir call */ static int nfs_readdir(struct vop_readdir_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct uio *uio = ap->a_uio; ssize_t tresid, left; int error = 0; struct vattr vattr; if (ap->a_eofflag != NULL) *ap->a_eofflag = 0; if (vp->v_type != VDIR) return(EPERM); /* * First, check for hit on the EOF offset cache */ if (np->n_direofoffset > 0 && uio->uio_offset >= np->n_direofoffset && (np->n_flag & NMODIFIED) == 0) { if (VOP_GETATTR(vp, &vattr, ap->a_cred) == 0) { mtx_lock(&np->n_mtx); if ((NFS_ISV4(vp) && np->n_change == vattr.va_filerev) || !NFS_TIMESPEC_COMPARE(&np->n_mtime, &vattr.va_mtime)) { mtx_unlock(&np->n_mtx); NFSINCRGLOBAL(nfsstatsv1.direofcache_hits); if (ap->a_eofflag != NULL) *ap->a_eofflag = 1; return (0); } else mtx_unlock(&np->n_mtx); } } /* * NFS always guarantees that directory entries don't straddle * DIRBLKSIZ boundaries. As such, we need to limit the size * to an exact multiple of DIRBLKSIZ, to avoid copying a partial * directory entry. */ left = uio->uio_resid % DIRBLKSIZ; if (left == uio->uio_resid) return (EINVAL); uio->uio_resid -= left; /* * Call ncl_bioread() to do the real work. */ tresid = uio->uio_resid; error = ncl_bioread(vp, uio, 0, ap->a_cred); if (!error && uio->uio_resid == tresid) { NFSINCRGLOBAL(nfsstatsv1.direofcache_misses); if (ap->a_eofflag != NULL) *ap->a_eofflag = 1; } /* Add the partial DIRBLKSIZ (left) back in. */ uio->uio_resid += left; return (error); } /* * Readdir rpc call. * Called from below the buffer cache by ncl_doio(). */ int ncl_readdirrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred, struct thread *td) { struct nfsvattr nfsva; nfsuint64 *cookiep, cookie; struct nfsnode *dnp = VTONFS(vp); struct nfsmount *nmp = VFSTONFS(vp->v_mount); int error = 0, eof, attrflag; KASSERT(uiop->uio_iovcnt == 1 && (uiop->uio_offset & (DIRBLKSIZ - 1)) == 0 && (uiop->uio_resid & (DIRBLKSIZ - 1)) == 0, ("nfs readdirrpc bad uio")); /* * If there is no cookie, assume directory was stale. */ ncl_dircookie_lock(dnp); cookiep = ncl_getcookie(dnp, uiop->uio_offset, 0); if (cookiep) { cookie = *cookiep; ncl_dircookie_unlock(dnp); } else { ncl_dircookie_unlock(dnp); return (NFSERR_BAD_COOKIE); } if (NFSHASNFSV3(nmp) && !NFSHASGOTFSINFO(nmp)) (void)ncl_fsinfo(nmp, vp, cred, td); error = nfsrpc_readdir(vp, uiop, &cookie, cred, td, &nfsva, &attrflag, &eof, NULL); if (attrflag) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (!error) { /* * We are now either at the end of the directory or have filled * the block. */ if (eof) dnp->n_direofoffset = uiop->uio_offset; else { if (uiop->uio_resid > 0) printf("EEK! readdirrpc resid > 0\n"); ncl_dircookie_lock(dnp); cookiep = ncl_getcookie(dnp, uiop->uio_offset, 1); *cookiep = cookie; ncl_dircookie_unlock(dnp); } } else if (NFS_ISV4(vp)) { error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); } return (error); } /* * NFS V3 readdir plus RPC. Used in place of ncl_readdirrpc(). */ int ncl_readdirplusrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred, struct thread *td) { struct nfsvattr nfsva; nfsuint64 *cookiep, cookie; struct nfsnode *dnp = VTONFS(vp); struct nfsmount *nmp = VFSTONFS(vp->v_mount); int error = 0, attrflag, eof; KASSERT(uiop->uio_iovcnt == 1 && (uiop->uio_offset & (DIRBLKSIZ - 1)) == 0 && (uiop->uio_resid & (DIRBLKSIZ - 1)) == 0, ("nfs readdirplusrpc bad uio")); /* * If there is no cookie, assume directory was stale. */ ncl_dircookie_lock(dnp); cookiep = ncl_getcookie(dnp, uiop->uio_offset, 0); if (cookiep) { cookie = *cookiep; ncl_dircookie_unlock(dnp); } else { ncl_dircookie_unlock(dnp); return (NFSERR_BAD_COOKIE); } if (NFSHASNFSV3(nmp) && !NFSHASGOTFSINFO(nmp)) (void)ncl_fsinfo(nmp, vp, cred, td); error = nfsrpc_readdirplus(vp, uiop, &cookie, cred, td, &nfsva, &attrflag, &eof, NULL); if (attrflag) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (!error) { /* * We are now either at end of the directory or have filled the * the block. */ if (eof) dnp->n_direofoffset = uiop->uio_offset; else { if (uiop->uio_resid > 0) printf("EEK! readdirplusrpc resid > 0\n"); ncl_dircookie_lock(dnp); cookiep = ncl_getcookie(dnp, uiop->uio_offset, 1); *cookiep = cookie; ncl_dircookie_unlock(dnp); } } else if (NFS_ISV4(vp)) { error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); } return (error); } /* * Silly rename. To make the NFS filesystem that is stateless look a little * more like the "ufs" a remove of an active vnode is translated to a rename * to a funny looking filename that is removed by nfs_inactive on the * nfsnode. There is the potential for another process on a different client * to create the same funny name between the nfs_lookitup() fails and the * nfs_rename() completes, but... */ static int nfs_sillyrename(struct vnode *dvp, struct vnode *vp, struct componentname *cnp) { struct sillyrename *sp; struct nfsnode *np; int error; short pid; unsigned int lticks; cache_purge(dvp); np = VTONFS(vp); KASSERT(vp->v_type != VDIR, ("nfs: sillyrename dir")); sp = malloc(sizeof (struct sillyrename), M_NEWNFSREQ, M_WAITOK); sp->s_cred = crhold(cnp->cn_cred); sp->s_dvp = dvp; VREF(dvp); /* * Fudge together a funny name. * Changing the format of the funny name to accommodate more * sillynames per directory. * The name is now changed to .nfs...4, where ticks is * CPU ticks since boot. */ pid = cnp->cn_thread->td_proc->p_pid; lticks = (unsigned int)ticks; for ( ; ; ) { sp->s_namlen = sprintf(sp->s_name, ".nfs.%08x.%04x4.4", lticks, pid); if (nfs_lookitup(dvp, sp->s_name, sp->s_namlen, sp->s_cred, cnp->cn_thread, NULL)) break; lticks++; } error = nfs_renameit(dvp, vp, cnp, sp); if (error) goto bad; error = nfs_lookitup(dvp, sp->s_name, sp->s_namlen, sp->s_cred, cnp->cn_thread, &np); np->n_sillyrename = sp; return (0); bad: vrele(sp->s_dvp); crfree(sp->s_cred); free(sp, M_NEWNFSREQ); return (error); } /* * Look up a file name and optionally either update the file handle or * allocate an nfsnode, depending on the value of npp. * npp == NULL --> just do the lookup * *npp == NULL --> allocate a new nfsnode and make sure attributes are * handled too * *npp != NULL --> update the file handle in the vnode */ static int nfs_lookitup(struct vnode *dvp, char *name, int len, struct ucred *cred, struct thread *td, struct nfsnode **npp) { struct vnode *newvp = NULL, *vp; struct nfsnode *np, *dnp = VTONFS(dvp); struct nfsfh *nfhp, *onfhp; struct nfsvattr nfsva, dnfsva; struct componentname cn; int error = 0, attrflag, dattrflag; u_int hash; error = nfsrpc_lookup(dvp, name, len, cred, td, &dnfsva, &nfsva, &nfhp, &attrflag, &dattrflag, NULL); if (dattrflag) (void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1); if (npp && !error) { if (*npp != NULL) { np = *npp; vp = NFSTOV(np); /* * For NFSv4, check to see if it is the same name and * replace the name, if it is different. */ if (np->n_v4 != NULL && nfsva.na_type == VREG && (np->n_v4->n4_namelen != len || NFSBCMP(name, NFS4NODENAME(np->n_v4), len) || dnp->n_fhp->nfh_len != np->n_v4->n4_fhlen || NFSBCMP(dnp->n_fhp->nfh_fh, np->n_v4->n4_data, dnp->n_fhp->nfh_len))) { #ifdef notdef { char nnn[100]; int nnnl; nnnl = (len < 100) ? len : 99; bcopy(name, nnn, nnnl); nnn[nnnl] = '\0'; printf("replace=%s\n",nnn); } #endif free(np->n_v4, M_NFSV4NODE); np->n_v4 = malloc( sizeof (struct nfsv4node) + dnp->n_fhp->nfh_len + len - 1, M_NFSV4NODE, M_WAITOK); np->n_v4->n4_fhlen = dnp->n_fhp->nfh_len; np->n_v4->n4_namelen = len; NFSBCOPY(dnp->n_fhp->nfh_fh, np->n_v4->n4_data, dnp->n_fhp->nfh_len); NFSBCOPY(name, NFS4NODENAME(np->n_v4), len); } hash = fnv_32_buf(nfhp->nfh_fh, nfhp->nfh_len, FNV1_32_INIT); onfhp = np->n_fhp; /* * Rehash node for new file handle. */ vfs_hash_rehash(vp, hash); np->n_fhp = nfhp; if (onfhp != NULL) free(onfhp, M_NFSFH); newvp = NFSTOV(np); } else if (NFS_CMPFH(dnp, nfhp->nfh_fh, nfhp->nfh_len)) { free(nfhp, M_NFSFH); VREF(dvp); newvp = dvp; } else { cn.cn_nameptr = name; cn.cn_namelen = len; error = nfscl_nget(dvp->v_mount, dvp, nfhp, &cn, td, &np, NULL, LK_EXCLUSIVE); if (error) return (error); newvp = NFSTOV(np); } if (!attrflag && *npp == NULL) { if (newvp == dvp) vrele(newvp); else vput(newvp); return (ENOENT); } if (attrflag) (void) nfscl_loadattrcache(&newvp, &nfsva, NULL, NULL, 0, 1); } if (npp && *npp == NULL) { if (error) { if (newvp) { if (newvp == dvp) vrele(newvp); else vput(newvp); } } else *npp = np; } if (error && NFS_ISV4(dvp)) error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); return (error); } /* * Nfs Version 3 and 4 commit rpc */ int ncl_commit(struct vnode *vp, u_quad_t offset, int cnt, struct ucred *cred, struct thread *td) { struct nfsvattr nfsva; struct nfsmount *nmp = VFSTONFS(vp->v_mount); struct nfsnode *np; struct uio uio; int error, attrflag; np = VTONFS(vp); error = EIO; attrflag = 0; if (NFSHASPNFS(nmp) && (np->n_flag & NDSCOMMIT) != 0) { uio.uio_offset = offset; uio.uio_resid = cnt; error = nfscl_doiods(vp, &uio, NULL, NULL, NFSV4OPEN_ACCESSWRITE, 1, cred, td); if (error != 0) { mtx_lock(&np->n_mtx); np->n_flag &= ~NDSCOMMIT; mtx_unlock(&np->n_mtx); } } if (error != 0) { mtx_lock(&nmp->nm_mtx); if ((nmp->nm_state & NFSSTA_HASWRITEVERF) == 0) { mtx_unlock(&nmp->nm_mtx); return (0); } mtx_unlock(&nmp->nm_mtx); error = nfsrpc_commit(vp, offset, cnt, cred, td, &nfsva, &attrflag, NULL); } if (attrflag != 0) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (error != 0 && NFS_ISV4(vp)) error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0); return (error); } /* * Strategy routine. * For async requests when nfsiod(s) are running, queue the request by * calling ncl_asyncio(), otherwise just all ncl_doio() to do the * request. */ static int nfs_strategy(struct vop_strategy_args *ap) { struct buf *bp; struct vnode *vp; struct ucred *cr; bp = ap->a_bp; vp = ap->a_vp; KASSERT(bp->b_vp == vp, ("missing b_getvp")); KASSERT(!(bp->b_flags & B_DONE), ("nfs_strategy: buffer %p unexpectedly marked B_DONE", bp)); BUF_ASSERT_HELD(bp); if (vp->v_type == VREG && bp->b_blkno == bp->b_lblkno) bp->b_blkno = bp->b_lblkno * (vp->v_bufobj.bo_bsize / DEV_BSIZE); if (bp->b_iocmd == BIO_READ) cr = bp->b_rcred; else cr = bp->b_wcred; /* * If the op is asynchronous and an i/o daemon is waiting * queue the request, wake it up and wait for completion * otherwise just do it ourselves. */ if ((bp->b_flags & B_ASYNC) == 0 || ncl_asyncio(VFSTONFS(vp->v_mount), bp, NOCRED, curthread)) (void) ncl_doio(vp, bp, cr, curthread, 1); return (0); } /* * fsync vnode op. Just call ncl_flush() with commit == 1. */ /* ARGSUSED */ static int nfs_fsync(struct vop_fsync_args *ap) { if (ap->a_vp->v_type != VREG) { /* * For NFS, metadata is changed synchronously on the server, * so there is nothing to flush. Also, ncl_flush() clears * the NMODIFIED flag and that shouldn't be done here for * directories. */ return (0); } return (ncl_flush(ap->a_vp, ap->a_waitfor, ap->a_td, 1, 0)); } /* * Flush all the blocks associated with a vnode. * Walk through the buffer pool and push any dirty pages * associated with the vnode. * If the called_from_renewthread argument is TRUE, it has been called * from the NFSv4 renew thread and, as such, cannot block indefinitely * waiting for a buffer write to complete. */ int ncl_flush(struct vnode *vp, int waitfor, struct thread *td, int commit, int called_from_renewthread) { struct nfsnode *np = VTONFS(vp); struct buf *bp; int i; struct buf *nbp; struct nfsmount *nmp = VFSTONFS(vp->v_mount); int error = 0, slptimeo = 0, slpflag = 0, retv, bvecpos; int passone = 1, trycnt = 0; u_quad_t off, endoff, toff; struct ucred* wcred = NULL; struct buf **bvec = NULL; struct bufobj *bo; #ifndef NFS_COMMITBVECSIZ #define NFS_COMMITBVECSIZ 20 #endif struct buf *bvec_on_stack[NFS_COMMITBVECSIZ]; u_int bvecsize = 0, bveccount; if (called_from_renewthread != 0) slptimeo = hz; if (nmp->nm_flag & NFSMNT_INT) slpflag = PCATCH; if (!commit) passone = 0; bo = &vp->v_bufobj; /* * A b_flags == (B_DELWRI | B_NEEDCOMMIT) block has been written to the * server, but has not been committed to stable storage on the server * yet. On the first pass, the byte range is worked out and the commit * rpc is done. On the second pass, ncl_writebp() is called to do the * job. */ again: off = (u_quad_t)-1; endoff = 0; bvecpos = 0; if (NFS_ISV34(vp) && commit) { if (bvec != NULL && bvec != bvec_on_stack) free(bvec, M_TEMP); /* * Count up how many buffers waiting for a commit. */ bveccount = 0; BO_LOCK(bo); TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (!BUF_ISLOCKED(bp) && (bp->b_flags & (B_DELWRI | B_NEEDCOMMIT)) == (B_DELWRI | B_NEEDCOMMIT)) bveccount++; } /* * Allocate space to remember the list of bufs to commit. It is * important to use M_NOWAIT here to avoid a race with nfs_write. * If we can't get memory (for whatever reason), we will end up * committing the buffers one-by-one in the loop below. */ if (bveccount > NFS_COMMITBVECSIZ) { /* * Release the vnode interlock to avoid a lock * order reversal. */ BO_UNLOCK(bo); bvec = (struct buf **) malloc(bveccount * sizeof(struct buf *), M_TEMP, M_NOWAIT); BO_LOCK(bo); if (bvec == NULL) { bvec = bvec_on_stack; bvecsize = NFS_COMMITBVECSIZ; } else bvecsize = bveccount; } else { bvec = bvec_on_stack; bvecsize = NFS_COMMITBVECSIZ; } TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (bvecpos >= bvecsize) break; if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) { nbp = TAILQ_NEXT(bp, b_bobufs); continue; } if ((bp->b_flags & (B_DELWRI | B_NEEDCOMMIT)) != (B_DELWRI | B_NEEDCOMMIT)) { BUF_UNLOCK(bp); nbp = TAILQ_NEXT(bp, b_bobufs); continue; } BO_UNLOCK(bo); bremfree(bp); /* * Work out if all buffers are using the same cred * so we can deal with them all with one commit. * * NOTE: we are not clearing B_DONE here, so we have * to do it later on in this routine if we intend to * initiate I/O on the bp. * * Note: to avoid loopback deadlocks, we do not * assign b_runningbufspace. */ if (wcred == NULL) wcred = bp->b_wcred; else if (wcred != bp->b_wcred) wcred = NOCRED; vfs_busy_pages(bp, 1); BO_LOCK(bo); /* * bp is protected by being locked, but nbp is not * and vfs_busy_pages() may sleep. We have to * recalculate nbp. */ nbp = TAILQ_NEXT(bp, b_bobufs); /* * A list of these buffers is kept so that the * second loop knows which buffers have actually * been committed. This is necessary, since there * may be a race between the commit rpc and new * uncommitted writes on the file. */ bvec[bvecpos++] = bp; toff = ((u_quad_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff; if (toff < off) off = toff; toff += (u_quad_t)(bp->b_dirtyend - bp->b_dirtyoff); if (toff > endoff) endoff = toff; } BO_UNLOCK(bo); } if (bvecpos > 0) { /* * Commit data on the server, as required. * If all bufs are using the same wcred, then use that with * one call for all of them, otherwise commit each one * separately. */ if (wcred != NOCRED) retv = ncl_commit(vp, off, (int)(endoff - off), wcred, td); else { retv = 0; for (i = 0; i < bvecpos; i++) { off_t off, size; bp = bvec[i]; off = ((u_quad_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff; size = (u_quad_t)(bp->b_dirtyend - bp->b_dirtyoff); retv = ncl_commit(vp, off, (int)size, bp->b_wcred, td); if (retv) break; } } if (retv == NFSERR_STALEWRITEVERF) ncl_clearcommit(vp->v_mount); /* * Now, either mark the blocks I/O done or mark the * blocks dirty, depending on whether the commit * succeeded. */ for (i = 0; i < bvecpos; i++) { bp = bvec[i]; bp->b_flags &= ~(B_NEEDCOMMIT | B_CLUSTEROK); if (retv) { /* * Error, leave B_DELWRI intact */ vfs_unbusy_pages(bp); brelse(bp); } else { /* * Success, remove B_DELWRI ( bundirty() ). * * b_dirtyoff/b_dirtyend seem to be NFS * specific. We should probably move that * into bundirty(). XXX */ bufobj_wref(bo); bp->b_flags |= B_ASYNC; bundirty(bp); bp->b_flags &= ~B_DONE; bp->b_ioflags &= ~BIO_ERROR; bp->b_dirtyoff = bp->b_dirtyend = 0; bufdone(bp); } } } /* * Start/do any write(s) that are required. */ loop: BO_LOCK(bo); TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) { if (waitfor != MNT_WAIT || passone) continue; error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo), "nfsfsync", slpflag, slptimeo); if (error == 0) { BUF_UNLOCK(bp); goto loop; } if (error == ENOLCK) { error = 0; goto loop; } if (called_from_renewthread != 0) { /* * Return EIO so the flush will be retried * later. */ error = EIO; goto done; } if (newnfs_sigintr(nmp, td)) { error = EINTR; goto done; } if (slpflag == PCATCH) { slpflag = 0; slptimeo = 2 * hz; } goto loop; } if ((bp->b_flags & B_DELWRI) == 0) panic("nfs_fsync: not dirty"); if ((passone || !commit) && (bp->b_flags & B_NEEDCOMMIT)) { BUF_UNLOCK(bp); continue; } BO_UNLOCK(bo); bremfree(bp); if (passone || !commit) bp->b_flags |= B_ASYNC; else bp->b_flags |= B_ASYNC; bwrite(bp); if (newnfs_sigintr(nmp, td)) { error = EINTR; goto done; } goto loop; } if (passone) { passone = 0; BO_UNLOCK(bo); goto again; } if (waitfor == MNT_WAIT) { while (bo->bo_numoutput) { error = bufobj_wwait(bo, slpflag, slptimeo); if (error) { BO_UNLOCK(bo); if (called_from_renewthread != 0) { /* * Return EIO so that the flush will be * retried later. */ error = EIO; goto done; } error = newnfs_sigintr(nmp, td); if (error) goto done; if (slpflag == PCATCH) { slpflag = 0; slptimeo = 2 * hz; } BO_LOCK(bo); } } if (bo->bo_dirty.bv_cnt != 0 && commit) { BO_UNLOCK(bo); goto loop; } /* * Wait for all the async IO requests to drain */ BO_UNLOCK(bo); mtx_lock(&np->n_mtx); while (np->n_directio_asyncwr > 0) { np->n_flag |= NFSYNCWAIT; error = newnfs_msleep(td, &np->n_directio_asyncwr, &np->n_mtx, slpflag | (PRIBIO + 1), "nfsfsync", 0); if (error) { if (newnfs_sigintr(nmp, td)) { mtx_unlock(&np->n_mtx); error = EINTR; goto done; } } } mtx_unlock(&np->n_mtx); } else BO_UNLOCK(bo); if (NFSHASPNFS(nmp)) { nfscl_layoutcommit(vp, td); /* * Invalidate the attribute cache, since writes to a DS * won't update the size attribute. */ mtx_lock(&np->n_mtx); np->n_attrstamp = 0; } else mtx_lock(&np->n_mtx); if (np->n_flag & NWRITEERR) { error = np->n_error; np->n_flag &= ~NWRITEERR; } if (commit && bo->bo_dirty.bv_cnt == 0 && bo->bo_numoutput == 0 && np->n_directio_asyncwr == 0) np->n_flag &= ~NMODIFIED; mtx_unlock(&np->n_mtx); done: if (bvec != NULL && bvec != bvec_on_stack) free(bvec, M_TEMP); if (error == 0 && commit != 0 && waitfor == MNT_WAIT && (bo->bo_dirty.bv_cnt != 0 || bo->bo_numoutput != 0 || np->n_directio_asyncwr != 0)) { if (trycnt++ < 5) { /* try, try again... */ passone = 1; wcred = NULL; bvec = NULL; bvecsize = 0; goto again; } vn_printf(vp, "ncl_flush failed"); error = called_from_renewthread != 0 ? EIO : EBUSY; } return (error); } /* * NFS advisory byte-level locks. */ static int nfs_advlock(struct vop_advlock_args *ap) { struct vnode *vp = ap->a_vp; struct ucred *cred; struct nfsnode *np = VTONFS(ap->a_vp); struct proc *p = (struct proc *)ap->a_id; struct thread *td = curthread; /* XXX */ struct vattr va; int ret, error; u_quad_t size; error = NFSVOPLOCK(vp, LK_SHARED); if (error != 0) return (EBADF); if (NFS_ISV4(vp) && (ap->a_flags & (F_POSIX | F_FLOCK)) != 0) { if (vp->v_type != VREG) { error = EINVAL; goto out; } if ((ap->a_flags & F_POSIX) != 0) cred = p->p_ucred; else cred = td->td_ucred; NFSVOPLOCK(vp, LK_UPGRADE | LK_RETRY); if (vp->v_iflag & VI_DOOMED) { error = EBADF; goto out; } /* * If this is unlocking a write locked region, flush and * commit them before unlocking. This is required by * RFC3530 Sec. 9.3.2. */ if (ap->a_op == F_UNLCK && nfscl_checkwritelocked(vp, ap->a_fl, cred, td, ap->a_id, ap->a_flags)) (void) ncl_flush(vp, MNT_WAIT, td, 1, 0); /* * Loop around doing the lock op, while a blocking lock * must wait for the lock op to succeed. */ do { ret = nfsrpc_advlock(vp, np->n_size, ap->a_op, ap->a_fl, 0, cred, td, ap->a_id, ap->a_flags); if (ret == NFSERR_DENIED && (ap->a_flags & F_WAIT) && ap->a_op == F_SETLK) { NFSVOPUNLOCK(vp, 0); error = nfs_catnap(PZERO | PCATCH, ret, "ncladvl"); if (error) return (EINTR); NFSVOPLOCK(vp, LK_EXCLUSIVE | LK_RETRY); if (vp->v_iflag & VI_DOOMED) { error = EBADF; goto out; } } } while (ret == NFSERR_DENIED && (ap->a_flags & F_WAIT) && ap->a_op == F_SETLK); if (ret == NFSERR_DENIED) { error = EAGAIN; goto out; } else if (ret == EINVAL || ret == EBADF || ret == EINTR) { error = ret; goto out; } else if (ret != 0) { error = EACCES; goto out; } /* * Now, if we just got a lock, invalidate data in the buffer * cache, as required, so that the coherency conforms with * RFC3530 Sec. 9.3.2. */ if (ap->a_op == F_SETLK) { if ((np->n_flag & NMODIFIED) == 0) { np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); ret = VOP_GETATTR(vp, &va, cred); } if ((np->n_flag & NMODIFIED) || ret || np->n_change != va.va_filerev) { (void) ncl_vinvalbuf(vp, V_SAVE, td, 1); np->n_attrstamp = 0; KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); ret = VOP_GETATTR(vp, &va, cred); if (!ret) { np->n_mtime = va.va_mtime; np->n_change = va.va_filerev; } } /* Mark that a file lock has been acquired. */ mtx_lock(&np->n_mtx); np->n_flag |= NHASBEENLOCKED; mtx_unlock(&np->n_mtx); } } else if (!NFS_ISV4(vp)) { if ((VFSTONFS(vp->v_mount)->nm_flag & NFSMNT_NOLOCKD) != 0) { size = VTONFS(vp)->n_size; NFSVOPUNLOCK(vp, 0); error = lf_advlock(ap, &(vp->v_lockf), size); } else { if (nfs_advlock_p != NULL) error = nfs_advlock_p(ap); else { NFSVOPUNLOCK(vp, 0); error = ENOLCK; } } if (error == 0 && ap->a_op == F_SETLK) { error = NFSVOPLOCK(vp, LK_SHARED); if (error == 0) { /* Mark that a file lock has been acquired. */ mtx_lock(&np->n_mtx); np->n_flag |= NHASBEENLOCKED; mtx_unlock(&np->n_mtx); NFSVOPUNLOCK(vp, 0); } } return (error); } else error = EOPNOTSUPP; out: NFSVOPUNLOCK(vp, 0); return (error); } /* * NFS advisory byte-level locks. */ static int nfs_advlockasync(struct vop_advlockasync_args *ap) { struct vnode *vp = ap->a_vp; u_quad_t size; int error; if (NFS_ISV4(vp)) return (EOPNOTSUPP); error = NFSVOPLOCK(vp, LK_SHARED); if (error) return (error); if ((VFSTONFS(vp->v_mount)->nm_flag & NFSMNT_NOLOCKD) != 0) { size = VTONFS(vp)->n_size; NFSVOPUNLOCK(vp, 0); error = lf_advlockasync(ap, &(vp->v_lockf), size); } else { NFSVOPUNLOCK(vp, 0); error = EOPNOTSUPP; } return (error); } /* * Print out the contents of an nfsnode. */ static int nfs_print(struct vop_print_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); printf("\tfileid %jd fsid 0x%jx", (uintmax_t)np->n_vattr.na_fileid, (uintmax_t)np->n_vattr.na_fsid); if (vp->v_type == VFIFO) fifo_printinfo(vp); printf("\n"); return (0); } /* * This is the "real" nfs::bwrite(struct buf*). * We set B_CACHE if this is a VMIO buffer. */ int ncl_writebp(struct buf *bp, int force __unused, struct thread *td) { int oldflags, rtval; BUF_ASSERT_HELD(bp); if (bp->b_flags & B_INVAL) { brelse(bp); return (0); } oldflags = bp->b_flags; bp->b_flags |= B_CACHE; /* * Undirty the bp. We will redirty it later if the I/O fails. */ bundirty(bp); bp->b_flags &= ~B_DONE; bp->b_ioflags &= ~BIO_ERROR; bp->b_iocmd = BIO_WRITE; bufobj_wref(bp->b_bufobj); curthread->td_ru.ru_oublock++; /* * Note: to avoid loopback deadlocks, we do not * assign b_runningbufspace. */ vfs_busy_pages(bp, 1); BUF_KERNPROC(bp); bp->b_iooffset = dbtob(bp->b_blkno); bstrategy(bp); if ((oldflags & B_ASYNC) != 0) return (0); rtval = bufwait(bp); if (oldflags & B_DELWRI) reassignbuf(bp); brelse(bp); return (rtval); } /* * nfs special file access vnode op. * Essentially just get vattr and then imitate iaccess() since the device is * local to the client. */ static int nfsspec_access(struct vop_access_args *ap) { struct vattr *vap; struct ucred *cred = ap->a_cred; struct vnode *vp = ap->a_vp; accmode_t accmode = ap->a_accmode; struct vattr vattr; int error; /* * Disallow write attempts on filesystems mounted read-only; * unless the file is a socket, fifo, or a block or character * device resident on the filesystem. */ if ((accmode & VWRITE) && (vp->v_mount->mnt_flag & MNT_RDONLY)) { switch (vp->v_type) { case VREG: case VDIR: case VLNK: return (EROFS); default: break; } } vap = &vattr; error = VOP_GETATTR(vp, vap, cred); if (error) goto out; error = vaccess(vp->v_type, vap->va_mode, vap->va_uid, vap->va_gid, accmode, cred, NULL); out: return error; } /* * Read wrapper for fifos. */ static int nfsfifo_read(struct vop_read_args *ap) { struct nfsnode *np = VTONFS(ap->a_vp); int error; /* * Set access flag. */ mtx_lock(&np->n_mtx); np->n_flag |= NACC; vfs_timestamp(&np->n_atim); mtx_unlock(&np->n_mtx); error = fifo_specops.vop_read(ap); return error; } /* * Write wrapper for fifos. */ static int nfsfifo_write(struct vop_write_args *ap) { struct nfsnode *np = VTONFS(ap->a_vp); /* * Set update flag. */ mtx_lock(&np->n_mtx); np->n_flag |= NUPD; vfs_timestamp(&np->n_mtim); mtx_unlock(&np->n_mtx); return(fifo_specops.vop_write(ap)); } /* * Close wrapper for fifos. * * Update the times on the nfsnode then do fifo close. */ static int nfsfifo_close(struct vop_close_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np = VTONFS(vp); struct vattr vattr; struct timespec ts; mtx_lock(&np->n_mtx); if (np->n_flag & (NACC | NUPD)) { vfs_timestamp(&ts); if (np->n_flag & NACC) np->n_atim = ts; if (np->n_flag & NUPD) np->n_mtim = ts; np->n_flag |= NCHG; if (vrefcnt(vp) == 1 && (vp->v_mount->mnt_flag & MNT_RDONLY) == 0) { VATTR_NULL(&vattr); if (np->n_flag & NACC) vattr.va_atime = np->n_atim; if (np->n_flag & NUPD) vattr.va_mtime = np->n_mtim; mtx_unlock(&np->n_mtx); (void)VOP_SETATTR(vp, &vattr, ap->a_cred); goto out; } } mtx_unlock(&np->n_mtx); out: return (fifo_specops.vop_close(ap)); } /* * Just call ncl_writebp() with the force argument set to 1. * * NOTE: B_DONE may or may not be set in a_bp on call. */ static int nfs_bwrite(struct buf *bp) { return (ncl_writebp(bp, 1, curthread)); } struct buf_ops buf_ops_newnfs = { .bop_name = "buf_ops_nfs", .bop_write = nfs_bwrite, .bop_strategy = bufstrategy, .bop_sync = bufsync, .bop_bdflush = bufbdflush, }; static int nfs_getacl(struct vop_getacl_args *ap) { int error; if (ap->a_type != ACL_TYPE_NFS4) return (EOPNOTSUPP); error = nfsrpc_getacl(ap->a_vp, ap->a_cred, ap->a_td, ap->a_aclp, NULL); if (error > NFSERR_STALE) { (void) nfscl_maperr(ap->a_td, error, (uid_t)0, (gid_t)0); error = EPERM; } return (error); } static int nfs_setacl(struct vop_setacl_args *ap) { int error; if (ap->a_type != ACL_TYPE_NFS4) return (EOPNOTSUPP); error = nfsrpc_setacl(ap->a_vp, ap->a_cred, ap->a_td, ap->a_aclp, NULL); if (error > NFSERR_STALE) { (void) nfscl_maperr(ap->a_td, error, (uid_t)0, (gid_t)0); error = EPERM; } return (error); } static int nfs_set_text(struct vop_set_text_args *ap) { struct vnode *vp = ap->a_vp; struct nfsnode *np; /* * If the text file has been mmap'd, flush any dirty pages to the * buffer cache and then... * Make sure all writes are pushed to the NFS server. If this is not * done, the modify time of the file can change while the text * file is being executed. This will cause the process that is * executing the text file to be terminated. */ if (vp->v_object != NULL) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(vp->v_object); } /* Now, flush the buffer cache. */ ncl_flush(vp, MNT_WAIT, curthread, 0, 0); /* And, finally, make sure that n_mtime is up to date. */ np = VTONFS(vp); mtx_lock(&np->n_mtx); np->n_mtime = np->n_vattr.na_mtime; mtx_unlock(&np->n_mtx); vp->v_vflag |= VV_TEXT; return (0); } /* * Return POSIX pathconf information applicable to nfs filesystems. */ static int nfs_pathconf(struct vop_pathconf_args *ap) { struct nfsv3_pathconf pc; struct nfsvattr nfsva; struct vnode *vp = ap->a_vp; struct thread *td = curthread; int attrflag, error; if ((NFS_ISV34(vp) && (ap->a_name == _PC_LINK_MAX || ap->a_name == _PC_NAME_MAX || ap->a_name == _PC_CHOWN_RESTRICTED || ap->a_name == _PC_NO_TRUNC)) || (NFS_ISV4(vp) && ap->a_name == _PC_ACL_NFS4)) { /* * Since only the above 4 a_names are returned by the NFSv3 * Pathconf RPC, there is no point in doing it for others. * For NFSv4, the Pathconf RPC (actually a Getattr Op.) can * be used for _PC_NFS4_ACL as well. */ error = nfsrpc_pathconf(vp, &pc, td->td_ucred, td, &nfsva, &attrflag, NULL); if (attrflag != 0) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1); if (error != 0) return (error); } else { /* * For NFSv2 (or NFSv3 when not one of the above 4 a_names), * just fake them. */ pc.pc_linkmax = NFS_LINK_MAX; pc.pc_namemax = NFS_MAXNAMLEN; pc.pc_notrunc = 1; pc.pc_chownrestricted = 1; pc.pc_caseinsensitive = 0; pc.pc_casepreserving = 1; error = 0; } switch (ap->a_name) { case _PC_LINK_MAX: #ifdef _LP64 *ap->a_retval = pc.pc_linkmax; #else *ap->a_retval = MIN(LONG_MAX, pc.pc_linkmax); #endif break; case _PC_NAME_MAX: *ap->a_retval = pc.pc_namemax; break; case _PC_PIPE_BUF: if (ap->a_vp->v_type == VDIR || ap->a_vp->v_type == VFIFO) *ap->a_retval = PIPE_BUF; else error = EINVAL; break; case _PC_CHOWN_RESTRICTED: *ap->a_retval = pc.pc_chownrestricted; break; case _PC_NO_TRUNC: *ap->a_retval = pc.pc_notrunc; break; - case _PC_ACL_EXTENDED: - *ap->a_retval = 0; - break; case _PC_ACL_NFS4: if (NFS_ISV4(vp) && nfsrv_useacl != 0 && attrflag != 0 && NFSISSET_ATTRBIT(&nfsva.na_suppattr, NFSATTRBIT_ACL)) *ap->a_retval = 1; else *ap->a_retval = 0; break; case _PC_ACL_PATH_MAX: if (NFS_ISV4(vp)) *ap->a_retval = ACL_MAX_ENTRIES; else *ap->a_retval = 3; - break; - case _PC_MAC_PRESENT: - *ap->a_retval = 0; break; case _PC_PRIO_IO: *ap->a_retval = 0; break; case _PC_SYNC_IO: *ap->a_retval = 0; break; case _PC_ALLOC_SIZE_MIN: *ap->a_retval = vp->v_mount->mnt_stat.f_bsize; break; case _PC_FILESIZEBITS: if (NFS_ISV34(vp)) *ap->a_retval = 64; else *ap->a_retval = 32; break; case _PC_REC_INCR_XFER_SIZE: *ap->a_retval = vp->v_mount->mnt_stat.f_iosize; break; case _PC_REC_MAX_XFER_SIZE: *ap->a_retval = -1; /* means ``unlimited'' */ break; case _PC_REC_MIN_XFER_SIZE: *ap->a_retval = vp->v_mount->mnt_stat.f_iosize; break; case _PC_REC_XFER_ALIGN: *ap->a_retval = PAGE_SIZE; break; case _PC_SYMLINK_MAX: *ap->a_retval = NFS_MAXPATHLEN; break; default: error = vop_stdpathconf(ap); break; } return (error); } Index: projects/import-googletest-1.8.1/sys/kern/vfs_default.c =================================================================== --- projects/import-googletest-1.8.1/sys/kern/vfs_default.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/kern/vfs_default.c (revision 345026) @@ -1,1370 +1,1377 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1989, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed * to Berkeley by John Heidemann of the UCLA Ficus project. * * Source: * @(#)i405_init.c 2.10 92/04/27 UCLA Ficus project * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int vop_nolookup(struct vop_lookup_args *); static int vop_norename(struct vop_rename_args *); static int vop_nostrategy(struct vop_strategy_args *); static int get_next_dirent(struct vnode *vp, struct dirent **dpp, char *dirbuf, int dirbuflen, off_t *off, char **cpos, int *len, int *eofflag, struct thread *td); static int dirent_exists(struct vnode *vp, const char *dirname, struct thread *td); #define DIRENT_MINSIZE (sizeof(struct dirent) - (MAXNAMLEN+1) + 4) static int vop_stdis_text(struct vop_is_text_args *ap); static int vop_stdset_text(struct vop_set_text_args *ap); static int vop_stdunset_text(struct vop_unset_text_args *ap); static int vop_stdget_writecount(struct vop_get_writecount_args *ap); static int vop_stdadd_writecount(struct vop_add_writecount_args *ap); static int vop_stdfdatasync(struct vop_fdatasync_args *ap); static int vop_stdgetpages_async(struct vop_getpages_async_args *ap); /* * This vnode table stores what we want to do if the filesystem doesn't * implement a particular VOP. * * If there is no specific entry here, we will return EOPNOTSUPP. * * Note that every filesystem has to implement either vop_access * or vop_accessx; failing to do so will result in immediate crash * due to stack overflow, as vop_stdaccess() calls vop_stdaccessx(), * which calls vop_stdaccess() etc. */ struct vop_vector default_vnodeops = { .vop_default = NULL, .vop_bypass = VOP_EOPNOTSUPP, .vop_access = vop_stdaccess, .vop_accessx = vop_stdaccessx, .vop_advise = vop_stdadvise, .vop_advlock = vop_stdadvlock, .vop_advlockasync = vop_stdadvlockasync, .vop_advlockpurge = vop_stdadvlockpurge, .vop_allocate = vop_stdallocate, .vop_bmap = vop_stdbmap, .vop_close = VOP_NULL, .vop_fsync = VOP_NULL, .vop_fdatasync = vop_stdfdatasync, .vop_getpages = vop_stdgetpages, .vop_getpages_async = vop_stdgetpages_async, .vop_getwritemount = vop_stdgetwritemount, .vop_inactive = VOP_NULL, .vop_ioctl = VOP_ENOTTY, .vop_kqfilter = vop_stdkqfilter, .vop_islocked = vop_stdislocked, .vop_lock1 = vop_stdlock, .vop_lookup = vop_nolookup, .vop_open = VOP_NULL, .vop_pathconf = VOP_EINVAL, .vop_poll = vop_nopoll, .vop_putpages = vop_stdputpages, .vop_readlink = VOP_EINVAL, .vop_rename = vop_norename, .vop_revoke = VOP_PANIC, .vop_strategy = vop_nostrategy, .vop_unlock = vop_stdunlock, .vop_vptocnp = vop_stdvptocnp, .vop_vptofh = vop_stdvptofh, .vop_unp_bind = vop_stdunp_bind, .vop_unp_connect = vop_stdunp_connect, .vop_unp_detach = vop_stdunp_detach, .vop_is_text = vop_stdis_text, .vop_set_text = vop_stdset_text, .vop_unset_text = vop_stdunset_text, .vop_get_writecount = vop_stdget_writecount, .vop_add_writecount = vop_stdadd_writecount, }; /* * Series of placeholder functions for various error returns for * VOPs. */ int vop_eopnotsupp(struct vop_generic_args *ap) { /* printf("vop_notsupp[%s]\n", ap->a_desc->vdesc_name); */ return (EOPNOTSUPP); } int vop_ebadf(struct vop_generic_args *ap) { return (EBADF); } int vop_enotty(struct vop_generic_args *ap) { return (ENOTTY); } int vop_einval(struct vop_generic_args *ap) { return (EINVAL); } int vop_enoent(struct vop_generic_args *ap) { return (ENOENT); } int vop_null(struct vop_generic_args *ap) { return (0); } /* * Helper function to panic on some bad VOPs in some filesystems. */ int vop_panic(struct vop_generic_args *ap) { panic("filesystem goof: vop_panic[%s]", ap->a_desc->vdesc_name); } /* * vop_std and vop_no are default functions for use by * filesystems that need the "default reasonable" implementation for a * particular operation. * * The documentation for the operations they implement exists (if it exists) * in the VOP_(9) manpage (all uppercase). */ /* * Default vop for filesystems that do not support name lookup */ static int vop_nolookup(ap) struct vop_lookup_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; } */ *ap; { *ap->a_vpp = NULL; return (ENOTDIR); } /* * vop_norename: * * Handle unlock and reference counting for arguments of vop_rename * for filesystems that do not implement rename operation. */ static int vop_norename(struct vop_rename_args *ap) { vop_rename_fail(ap); return (EOPNOTSUPP); } /* * vop_nostrategy: * * Strategy routine for VFS devices that have none. * * BIO_ERROR and B_INVAL must be cleared prior to calling any strategy * routine. Typically this is done for a BIO_READ strategy call. * Typically B_INVAL is assumed to already be clear prior to a write * and should not be cleared manually unless you just made the buffer * invalid. BIO_ERROR should be cleared either way. */ static int vop_nostrategy (struct vop_strategy_args *ap) { printf("No strategy for buffer at %p\n", ap->a_bp); vn_printf(ap->a_vp, "vnode "); ap->a_bp->b_ioflags |= BIO_ERROR; ap->a_bp->b_error = EOPNOTSUPP; bufdone(ap->a_bp); return (EOPNOTSUPP); } static int get_next_dirent(struct vnode *vp, struct dirent **dpp, char *dirbuf, int dirbuflen, off_t *off, char **cpos, int *len, int *eofflag, struct thread *td) { int error, reclen; struct uio uio; struct iovec iov; struct dirent *dp; KASSERT(VOP_ISLOCKED(vp), ("vp %p is not locked", vp)); KASSERT(vp->v_type == VDIR, ("vp %p is not a directory", vp)); if (*len == 0) { iov.iov_base = dirbuf; iov.iov_len = dirbuflen; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = *off; uio.uio_resid = dirbuflen; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; uio.uio_td = td; *eofflag = 0; #ifdef MAC error = mac_vnode_check_readdir(td->td_ucred, vp); if (error == 0) #endif error = VOP_READDIR(vp, &uio, td->td_ucred, eofflag, NULL, NULL); if (error) return (error); *off = uio.uio_offset; *cpos = dirbuf; *len = (dirbuflen - uio.uio_resid); if (*len == 0) return (ENOENT); } dp = (struct dirent *)(*cpos); reclen = dp->d_reclen; *dpp = dp; /* check for malformed directory.. */ if (reclen < DIRENT_MINSIZE) return (EINVAL); *cpos += reclen; *len -= reclen; return (0); } /* * Check if a named file exists in a given directory vnode. */ static int dirent_exists(struct vnode *vp, const char *dirname, struct thread *td) { char *dirbuf, *cpos; int error, eofflag, dirbuflen, len, found; off_t off; struct dirent *dp; struct vattr va; KASSERT(VOP_ISLOCKED(vp), ("vp %p is not locked", vp)); KASSERT(vp->v_type == VDIR, ("vp %p is not a directory", vp)); found = 0; error = VOP_GETATTR(vp, &va, td->td_ucred); if (error) return (found); dirbuflen = DEV_BSIZE; if (dirbuflen < va.va_blocksize) dirbuflen = va.va_blocksize; dirbuf = (char *)malloc(dirbuflen, M_TEMP, M_WAITOK); off = 0; len = 0; do { error = get_next_dirent(vp, &dp, dirbuf, dirbuflen, &off, &cpos, &len, &eofflag, td); if (error) goto out; if (dp->d_type != DT_WHT && dp->d_fileno != 0 && strcmp(dp->d_name, dirname) == 0) { found = 1; goto out; } } while (len > 0 || !eofflag); out: free(dirbuf, M_TEMP); return (found); } int vop_stdaccess(struct vop_access_args *ap) { KASSERT((ap->a_accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0, ("invalid bit in accmode")); return (VOP_ACCESSX(ap->a_vp, ap->a_accmode, ap->a_cred, ap->a_td)); } int vop_stdaccessx(struct vop_accessx_args *ap) { int error; accmode_t accmode = ap->a_accmode; error = vfs_unixify_accmode(&accmode); if (error != 0) return (error); if (accmode == 0) return (0); return (VOP_ACCESS(ap->a_vp, accmode, ap->a_cred, ap->a_td)); } /* * Advisory record locking support */ int vop_stdadvlock(struct vop_advlock_args *ap) { struct vnode *vp; struct vattr vattr; int error; vp = ap->a_vp; if (ap->a_fl->l_whence == SEEK_END) { /* * The NFSv4 server must avoid doing a vn_lock() here, since it * can deadlock the nfsd threads, due to a LOR. Fortunately * the NFSv4 server always uses SEEK_SET and this code is * only required for the SEEK_END case. */ vn_lock(vp, LK_SHARED | LK_RETRY); error = VOP_GETATTR(vp, &vattr, curthread->td_ucred); VOP_UNLOCK(vp, 0); if (error) return (error); } else vattr.va_size = 0; return (lf_advlock(ap, &(vp->v_lockf), vattr.va_size)); } int vop_stdadvlockasync(struct vop_advlockasync_args *ap) { struct vnode *vp; struct vattr vattr; int error; vp = ap->a_vp; if (ap->a_fl->l_whence == SEEK_END) { /* The size argument is only needed for SEEK_END. */ vn_lock(vp, LK_SHARED | LK_RETRY); error = VOP_GETATTR(vp, &vattr, curthread->td_ucred); VOP_UNLOCK(vp, 0); if (error) return (error); } else vattr.va_size = 0; return (lf_advlockasync(ap, &(vp->v_lockf), vattr.va_size)); } int vop_stdadvlockpurge(struct vop_advlockpurge_args *ap) { struct vnode *vp; vp = ap->a_vp; lf_purgelocks(vp, &vp->v_lockf); return (0); } /* * vop_stdpathconf: * * Standard implementation of POSIX pathconf, to get information about limits * for a filesystem. * Override per filesystem for the case where the filesystem has smaller * limits. */ int vop_stdpathconf(ap) struct vop_pathconf_args /* { struct vnode *a_vp; int a_name; int *a_retval; } */ *ap; { switch (ap->a_name) { case _PC_ASYNC_IO: *ap->a_retval = _POSIX_ASYNCHRONOUS_IO; return (0); case _PC_PATH_MAX: *ap->a_retval = PATH_MAX; return (0); + case _PC_ACL_EXTENDED: + case _PC_ACL_NFS4: + case _PC_CAP_PRESENT: + case _PC_INF_PRESENT: + case _PC_MAC_PRESENT: + *ap->a_retval = 0; + return (0); default: return (EINVAL); } /* NOTREACHED */ } /* * Standard lock, unlock and islocked functions. */ int vop_stdlock(ap) struct vop_lock1_args /* { struct vnode *a_vp; int a_flags; char *file; int line; } */ *ap; { struct vnode *vp = ap->a_vp; struct mtx *ilk; ilk = VI_MTX(vp); return (lockmgr_lock_fast_path(vp->v_vnlock, ap->a_flags, &ilk->lock_object, ap->a_file, ap->a_line)); } /* See above. */ int vop_stdunlock(ap) struct vop_unlock_args /* { struct vnode *a_vp; int a_flags; } */ *ap; { struct vnode *vp = ap->a_vp; struct mtx *ilk; ilk = VI_MTX(vp); return (lockmgr_unlock_fast_path(vp->v_vnlock, ap->a_flags, &ilk->lock_object)); } /* See above. */ int vop_stdislocked(ap) struct vop_islocked_args /* { struct vnode *a_vp; } */ *ap; { return (lockstatus(ap->a_vp->v_vnlock)); } /* * Return true for select/poll. */ int vop_nopoll(ap) struct vop_poll_args /* { struct vnode *a_vp; int a_events; struct ucred *a_cred; struct thread *a_td; } */ *ap; { return (poll_no_poll(ap->a_events)); } /* * Implement poll for local filesystems that support it. */ int vop_stdpoll(ap) struct vop_poll_args /* { struct vnode *a_vp; int a_events; struct ucred *a_cred; struct thread *a_td; } */ *ap; { if (ap->a_events & ~POLLSTANDARD) return (vn_pollrecord(ap->a_vp, ap->a_td, ap->a_events)); return (ap->a_events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM)); } /* * Return our mount point, as we will take charge of the writes. */ int vop_stdgetwritemount(ap) struct vop_getwritemount_args /* { struct vnode *a_vp; struct mount **a_mpp; } */ *ap; { struct mount *mp; /* * XXX Since this is called unlocked we may be recycled while * attempting to ref the mount. If this is the case or mountpoint * will be set to NULL. We only have to prevent this call from * returning with a ref to an incorrect mountpoint. It is not * harmful to return with a ref to our previous mountpoint. */ mp = ap->a_vp->v_mount; if (mp != NULL) { vfs_ref(mp); if (mp != ap->a_vp->v_mount) { vfs_rel(mp); mp = NULL; } } *(ap->a_mpp) = mp; return (0); } /* XXX Needs good comment and VOP_BMAP(9) manpage */ int vop_stdbmap(ap) struct vop_bmap_args /* { struct vnode *a_vp; daddr_t a_bn; struct bufobj **a_bop; daddr_t *a_bnp; int *a_runp; int *a_runb; } */ *ap; { if (ap->a_bop != NULL) *ap->a_bop = &ap->a_vp->v_bufobj; if (ap->a_bnp != NULL) *ap->a_bnp = ap->a_bn * btodb(ap->a_vp->v_mount->mnt_stat.f_iosize); if (ap->a_runp != NULL) *ap->a_runp = 0; if (ap->a_runb != NULL) *ap->a_runb = 0; return (0); } int vop_stdfsync(ap) struct vop_fsync_args /* { struct vnode *a_vp; int a_waitfor; struct thread *a_td; } */ *ap; { struct vnode *vp; struct buf *bp, *nbp; struct bufobj *bo; struct mount *mp; int error, maxretry; error = 0; maxretry = 10000; /* large, arbitrarily chosen */ vp = ap->a_vp; mp = NULL; if (vp->v_type == VCHR) { VI_LOCK(vp); mp = vp->v_rdev->si_mountpt; VI_UNLOCK(vp); } bo = &vp->v_bufobj; BO_LOCK(bo); loop1: /* * MARK/SCAN initialization to avoid infinite loops. */ TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) { bp->b_vflags &= ~BV_SCANNED; bp->b_error = 0; } /* * Flush all dirty buffers associated with a vnode. */ loop2: TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { if ((bp->b_vflags & BV_SCANNED) != 0) continue; bp->b_vflags |= BV_SCANNED; if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) { if (ap->a_waitfor != MNT_WAIT) continue; if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_INTERLOCK | LK_SLEEPFAIL, BO_LOCKPTR(bo)) != 0) { BO_LOCK(bo); goto loop1; } BO_LOCK(bo); } BO_UNLOCK(bo); KASSERT(bp->b_bufobj == bo, ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo)); if ((bp->b_flags & B_DELWRI) == 0) panic("fsync: not dirty"); if ((vp->v_object != NULL) && (bp->b_flags & B_CLUSTEROK)) { vfs_bio_awrite(bp); } else { bremfree(bp); bawrite(bp); } if (maxretry < 1000) pause("dirty", hz < 1000 ? 1 : hz / 1000); BO_LOCK(bo); goto loop2; } /* * If synchronous the caller expects us to completely resolve all * dirty buffers in the system. Wait for in-progress I/O to * complete (which could include background bitmap writes), then * retry if dirty blocks still exist. */ if (ap->a_waitfor == MNT_WAIT) { bufobj_wwait(bo, 0, 0); if (bo->bo_dirty.bv_cnt > 0) { /* * If we are unable to write any of these buffers * then we fail now rather than trying endlessly * to write them out. */ TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) if ((error = bp->b_error) != 0) break; if ((mp != NULL && mp->mnt_secondary_writes > 0) || (error == 0 && --maxretry >= 0)) goto loop1; if (error == 0) error = EAGAIN; } } BO_UNLOCK(bo); if (error != 0) vn_printf(vp, "fsync: giving up on dirty (error = %d) ", error); return (error); } static int vop_stdfdatasync(struct vop_fdatasync_args *ap) { return (VOP_FSYNC(ap->a_vp, MNT_WAIT, ap->a_td)); } int vop_stdfdatasync_buf(struct vop_fdatasync_args *ap) { struct vop_fsync_args apf; apf.a_vp = ap->a_vp; apf.a_waitfor = MNT_WAIT; apf.a_td = ap->a_td; return (vop_stdfsync(&apf)); } /* XXX Needs good comment and more info in the manpage (VOP_GETPAGES(9)). */ int vop_stdgetpages(ap) struct vop_getpages_args /* { struct vnode *a_vp; vm_page_t *a_m; int a_count; int *a_rbehind; int *a_rahead; } */ *ap; { return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, ap->a_rahead, NULL, NULL); } static int vop_stdgetpages_async(struct vop_getpages_async_args *ap) { int error; error = VOP_GETPAGES(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, ap->a_rahead); ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); return (error); } int vop_stdkqfilter(struct vop_kqfilter_args *ap) { return vfs_kqfilter(ap); } /* XXX Needs good comment and more info in the manpage (VOP_PUTPAGES(9)). */ int vop_stdputpages(ap) struct vop_putpages_args /* { struct vnode *a_vp; vm_page_t *a_m; int a_count; int a_sync; int *a_rtvals; } */ *ap; { return vnode_pager_generic_putpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_sync, ap->a_rtvals); } int vop_stdvptofh(struct vop_vptofh_args *ap) { return (EOPNOTSUPP); } int vop_stdvptocnp(struct vop_vptocnp_args *ap) { struct vnode *vp = ap->a_vp; struct vnode **dvp = ap->a_vpp; struct ucred *cred = ap->a_cred; char *buf = ap->a_buf; int *buflen = ap->a_buflen; char *dirbuf, *cpos; int i, error, eofflag, dirbuflen, flags, locked, len, covered; off_t off; ino_t fileno; struct vattr va; struct nameidata nd; struct thread *td; struct dirent *dp; struct vnode *mvp; i = *buflen; error = 0; covered = 0; td = curthread; if (vp->v_type != VDIR) return (ENOENT); error = VOP_GETATTR(vp, &va, cred); if (error) return (error); VREF(vp); locked = VOP_ISLOCKED(vp); VOP_UNLOCK(vp, 0); NDINIT_ATVP(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, UIO_SYSSPACE, "..", vp, td); flags = FREAD; error = vn_open_cred(&nd, &flags, 0, VN_OPEN_NOAUDIT, cred, NULL); if (error) { vn_lock(vp, locked | LK_RETRY); return (error); } NDFREE(&nd, NDF_ONLY_PNBUF); mvp = *dvp = nd.ni_vp; if (vp->v_mount != (*dvp)->v_mount && ((*dvp)->v_vflag & VV_ROOT) && ((*dvp)->v_mount->mnt_flag & MNT_UNION)) { *dvp = (*dvp)->v_mount->mnt_vnodecovered; VREF(mvp); VOP_UNLOCK(mvp, 0); vn_close(mvp, FREAD, cred, td); VREF(*dvp); vn_lock(*dvp, LK_SHARED | LK_RETRY); covered = 1; } fileno = va.va_fileid; dirbuflen = DEV_BSIZE; if (dirbuflen < va.va_blocksize) dirbuflen = va.va_blocksize; dirbuf = (char *)malloc(dirbuflen, M_TEMP, M_WAITOK); if ((*dvp)->v_type != VDIR) { error = ENOENT; goto out; } off = 0; len = 0; do { /* call VOP_READDIR of parent */ error = get_next_dirent(*dvp, &dp, dirbuf, dirbuflen, &off, &cpos, &len, &eofflag, td); if (error) goto out; if ((dp->d_type != DT_WHT) && (dp->d_fileno == fileno)) { if (covered) { VOP_UNLOCK(*dvp, 0); vn_lock(mvp, LK_SHARED | LK_RETRY); if (dirent_exists(mvp, dp->d_name, td)) { error = ENOENT; VOP_UNLOCK(mvp, 0); vn_lock(*dvp, LK_SHARED | LK_RETRY); goto out; } VOP_UNLOCK(mvp, 0); vn_lock(*dvp, LK_SHARED | LK_RETRY); } i -= dp->d_namlen; if (i < 0) { error = ENOMEM; goto out; } if (dp->d_namlen == 1 && dp->d_name[0] == '.') { error = ENOENT; } else { bcopy(dp->d_name, buf + i, dp->d_namlen); error = 0; } goto out; } } while (len > 0 || !eofflag); error = ENOENT; out: free(dirbuf, M_TEMP); if (!error) { *buflen = i; vref(*dvp); } if (covered) { vput(*dvp); vrele(mvp); } else { VOP_UNLOCK(mvp, 0); vn_close(mvp, FREAD, cred, td); } vn_lock(vp, locked | LK_RETRY); return (error); } int vop_stdallocate(struct vop_allocate_args *ap) { #ifdef __notyet__ struct statfs *sfs; off_t maxfilesize = 0; #endif struct iovec aiov; struct vattr vattr, *vap; struct uio auio; off_t fsize, len, cur, offset; uint8_t *buf; struct thread *td; struct vnode *vp; size_t iosize; int error; buf = NULL; error = 0; td = curthread; vap = &vattr; vp = ap->a_vp; len = *ap->a_len; offset = *ap->a_offset; error = VOP_GETATTR(vp, vap, td->td_ucred); if (error != 0) goto out; fsize = vap->va_size; iosize = vap->va_blocksize; if (iosize == 0) iosize = BLKDEV_IOSIZE; if (iosize > MAXPHYS) iosize = MAXPHYS; buf = malloc(iosize, M_TEMP, M_WAITOK); #ifdef __notyet__ /* * Check if the filesystem sets f_maxfilesize; if not use * VOP_SETATTR to perform the check. */ sfs = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK); error = VFS_STATFS(vp->v_mount, sfs, td); if (error == 0) maxfilesize = sfs->f_maxfilesize; free(sfs, M_STATFS); if (error != 0) goto out; if (maxfilesize) { if (offset > maxfilesize || len > maxfilesize || offset + len > maxfilesize) { error = EFBIG; goto out; } } else #endif if (offset + len > vap->va_size) { /* * Test offset + len against the filesystem's maxfilesize. */ VATTR_NULL(vap); vap->va_size = offset + len; error = VOP_SETATTR(vp, vap, td->td_ucred); if (error != 0) goto out; VATTR_NULL(vap); vap->va_size = fsize; error = VOP_SETATTR(vp, vap, td->td_ucred); if (error != 0) goto out; } for (;;) { /* * Read and write back anything below the nominal file * size. There's currently no way outside the filesystem * to know whether this area is sparse or not. */ cur = iosize; if ((offset % iosize) != 0) cur -= (offset % iosize); if (cur > len) cur = len; if (offset < fsize) { aiov.iov_base = buf; aiov.iov_len = cur; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = offset; auio.uio_resid = cur; auio.uio_segflg = UIO_SYSSPACE; auio.uio_rw = UIO_READ; auio.uio_td = td; error = VOP_READ(vp, &auio, 0, td->td_ucred); if (error != 0) break; if (auio.uio_resid > 0) { bzero(buf + cur - auio.uio_resid, auio.uio_resid); } } else { bzero(buf, cur); } aiov.iov_base = buf; aiov.iov_len = cur; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = offset; auio.uio_resid = cur; auio.uio_segflg = UIO_SYSSPACE; auio.uio_rw = UIO_WRITE; auio.uio_td = td; error = VOP_WRITE(vp, &auio, 0, td->td_ucred); if (error != 0) break; len -= cur; offset += cur; if (len == 0) break; if (should_yield()) break; } out: *ap->a_len = len; *ap->a_offset = offset; free(buf, M_TEMP); return (error); } int vop_stdadvise(struct vop_advise_args *ap) { struct vnode *vp; struct bufobj *bo; daddr_t startn, endn; off_t bstart, bend, start, end; int bsize, error; vp = ap->a_vp; switch (ap->a_advice) { case POSIX_FADV_WILLNEED: /* * Do nothing for now. Filesystems should provide a * custom method which starts an asynchronous read of * the requested region. */ error = 0; break; case POSIX_FADV_DONTNEED: error = 0; vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); if (vp->v_iflag & VI_DOOMED) { VOP_UNLOCK(vp, 0); break; } /* * Round to block boundaries (and later possibly further to * page boundaries). Applications cannot reasonably be aware * of the boundaries, and the rounding must be to expand at * both extremities to cover enough. It still doesn't cover * read-ahead. For partial blocks, this gives unnecessary * discarding of buffers but is efficient enough since the * pages usually remain in VMIO for some time. */ bsize = vp->v_bufobj.bo_bsize; bstart = rounddown(ap->a_start, bsize); bend = roundup(ap->a_end, bsize); /* * Deactivate pages in the specified range from the backing VM * object. Pages that are resident in the buffer cache will * remain wired until their corresponding buffers are released * below. */ if (vp->v_object != NULL) { start = trunc_page(bstart); end = round_page(bend); VM_OBJECT_RLOCK(vp->v_object); vm_object_page_noreuse(vp->v_object, OFF_TO_IDX(start), OFF_TO_IDX(end)); VM_OBJECT_RUNLOCK(vp->v_object); } bo = &vp->v_bufobj; BO_RLOCK(bo); startn = bstart / bsize; endn = bend / bsize; error = bnoreuselist(&bo->bo_clean, bo, startn, endn); if (error == 0) error = bnoreuselist(&bo->bo_dirty, bo, startn, endn); BO_RUNLOCK(bo); VOP_UNLOCK(vp, 0); break; default: error = EINVAL; break; } return (error); } int vop_stdunp_bind(struct vop_unp_bind_args *ap) { ap->a_vp->v_unpcb = ap->a_unpcb; return (0); } int vop_stdunp_connect(struct vop_unp_connect_args *ap) { *ap->a_unpcb = ap->a_vp->v_unpcb; return (0); } int vop_stdunp_detach(struct vop_unp_detach_args *ap) { ap->a_vp->v_unpcb = NULL; return (0); } static int vop_stdis_text(struct vop_is_text_args *ap) { return ((ap->a_vp->v_vflag & VV_TEXT) != 0); } static int vop_stdset_text(struct vop_set_text_args *ap) { ap->a_vp->v_vflag |= VV_TEXT; return (0); } static int vop_stdunset_text(struct vop_unset_text_args *ap) { ap->a_vp->v_vflag &= ~VV_TEXT; return (0); } static int vop_stdget_writecount(struct vop_get_writecount_args *ap) { *ap->a_writecount = ap->a_vp->v_writecount; return (0); } static int vop_stdadd_writecount(struct vop_add_writecount_args *ap) { ap->a_vp->v_writecount += ap->a_inc; return (0); } /* * vfs default ops * used to fill the vfs function table to get reasonable default return values. */ int vfs_stdroot (mp, flags, vpp) struct mount *mp; int flags; struct vnode **vpp; { return (EOPNOTSUPP); } int vfs_stdstatfs (mp, sbp) struct mount *mp; struct statfs *sbp; { return (EOPNOTSUPP); } int vfs_stdquotactl (mp, cmds, uid, arg) struct mount *mp; int cmds; uid_t uid; void *arg; { return (EOPNOTSUPP); } int vfs_stdsync(mp, waitfor) struct mount *mp; int waitfor; { struct vnode *vp, *mvp; struct thread *td; int error, lockreq, allerror = 0; td = curthread; lockreq = LK_EXCLUSIVE | LK_INTERLOCK; if (waitfor != MNT_WAIT) lockreq |= LK_NOWAIT; /* * Force stale buffer cache information to be flushed. */ loop: MNT_VNODE_FOREACH_ALL(vp, mp, mvp) { if (vp->v_bufobj.bo_dirty.bv_cnt == 0) { VI_UNLOCK(vp); continue; } if ((error = vget(vp, lockreq, td)) != 0) { if (error == ENOENT) { MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp); goto loop; } continue; } error = VOP_FSYNC(vp, waitfor, td); if (error) allerror = error; vput(vp); } return (allerror); } int vfs_stdnosync (mp, waitfor) struct mount *mp; int waitfor; { return (0); } int vfs_stdvget (mp, ino, flags, vpp) struct mount *mp; ino_t ino; int flags; struct vnode **vpp; { return (EOPNOTSUPP); } int vfs_stdfhtovp (mp, fhp, flags, vpp) struct mount *mp; struct fid *fhp; int flags; struct vnode **vpp; { return (EOPNOTSUPP); } int vfs_stdinit (vfsp) struct vfsconf *vfsp; { return (0); } int vfs_stduninit (vfsp) struct vfsconf *vfsp; { return(0); } int vfs_stdextattrctl(mp, cmd, filename_vp, attrnamespace, attrname) struct mount *mp; int cmd; struct vnode *filename_vp; int attrnamespace; const char *attrname; { if (filename_vp != NULL) VOP_UNLOCK(filename_vp, 0); return (EOPNOTSUPP); } int vfs_stdsysctl(mp, op, req) struct mount *mp; fsctlop_t op; struct sysctl_req *req; { return (EOPNOTSUPP); } static vop_bypass_t * bp_by_off(struct vop_vector *vop, struct vop_generic_args *a) { return (*(vop_bypass_t **)((char *)vop + a->a_desc->vdesc_vop_offset)); } int vop_sigdefer(struct vop_vector *vop, struct vop_generic_args *a) { vop_bypass_t *bp; int prev_stops, rc; for (; vop != NULL; vop = vop->vop_default) { bp = bp_by_off(vop, a); if (bp != NULL) break; /* * Bypass is not really supported. It is done for * fallback to unimplemented vops in the default * vector. */ bp = vop->vop_bypass; if (bp != NULL) break; } MPASS(bp != NULL); prev_stops = sigdeferstop(SIGDEFERSTOP_SILENT); rc = bp(a); sigallowstop(prev_stops); return (rc); } Index: projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_pfil.c =================================================================== --- projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_pfil.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_pfil.c (revision 345026) @@ -1,663 +1,667 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2004 Andre Oppermann, Internet Business Solutions AG * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ipfw.h" #include "opt_inet.h" #include "opt_inet6.h" #ifndef INET #error IPFIREWALL requires INET. #endif /* INET */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET6 #include #include #include #endif #include #include #include VNET_DEFINE_STATIC(int, fw_enable) = 1; #define V_fw_enable VNET(fw_enable) #ifdef INET6 VNET_DEFINE_STATIC(int, fw6_enable) = 1; #define V_fw6_enable VNET(fw6_enable) #endif VNET_DEFINE_STATIC(int, fwlink_enable) = 0; #define V_fwlink_enable VNET(fwlink_enable) int ipfw_chg_hook(SYSCTL_HANDLER_ARGS); /* Forward declarations. */ static int ipfw_divert(struct mbuf **, int, struct ipfw_rule_ref *, int); #ifdef SYSCTL_NODE SYSBEGIN(f1) SYSCTL_DECL(_net_inet_ip_fw); SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, enable, CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE3, &VNET_NAME(fw_enable), 0, ipfw_chg_hook, "I", "Enable ipfw"); #ifdef INET6 SYSCTL_DECL(_net_inet6_ip6_fw); SYSCTL_PROC(_net_inet6_ip6_fw, OID_AUTO, enable, CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE3, &VNET_NAME(fw6_enable), 0, ipfw_chg_hook, "I", "Enable ipfw+6"); #endif /* INET6 */ SYSCTL_DECL(_net_link_ether); SYSCTL_PROC(_net_link_ether, OID_AUTO, ipfw, CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_SECURE3, &VNET_NAME(fwlink_enable), 0, ipfw_chg_hook, "I", "Pass ether pkts through firewall"); SYSEND #endif /* SYSCTL_NODE */ /* * The pfilter hook to pass packets to ipfw_chk and then to * dummynet, divert, netgraph or other modules. * The packet may be consumed. */ static pfil_return_t ipfw_check_packet(struct mbuf **m0, struct ifnet *ifp, int dir, void *ruleset __unused, struct inpcb *inp) { struct ip_fw_args args; struct m_tag *tag; pfil_return_t ret; int ipfw; /* convert dir to IPFW values */ dir = (dir & PFIL_IN) ? DIR_IN : DIR_OUT; args.flags = 0; again: /* * extract and remove the tag if present. If we are left * with onepass, optimize the outgoing path. */ tag = m_tag_locate(*m0, MTAG_IPFW_RULE, 0, NULL); if (tag != NULL) { args.rule = *((struct ipfw_rule_ref *)(tag+1)); m_tag_delete(*m0, tag); if (args.rule.info & IPFW_ONEPASS) return (0); args.flags |= IPFW_ARGS_REF; } args.m = *m0; args.oif = dir == DIR_OUT ? ifp : NULL; args.inp = inp; ipfw = ipfw_chk(&args); *m0 = args.m; KASSERT(*m0 != NULL || ipfw == IP_FW_DENY, ("%s: m0 is NULL", __func__)); ret = PFIL_PASS; switch (ipfw) { case IP_FW_PASS: /* next_hop may be set by ipfw_chk */ if ((args.flags & (IPFW_ARGS_NH4 | IPFW_ARGS_NH4PTR | IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)) == 0) break; #if (!defined(INET6) && !defined(INET)) ret = PFIL_DROPPED; #else { void *psa; size_t len; #ifdef INET if (args.flags & (IPFW_ARGS_NH4 | IPFW_ARGS_NH4PTR)) { MPASS((args.flags & (IPFW_ARGS_NH4 | IPFW_ARGS_NH4PTR)) != (IPFW_ARGS_NH4 | IPFW_ARGS_NH4PTR)); MPASS((args.flags & (IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)) == 0); len = sizeof(struct sockaddr_in); psa = (args.flags & IPFW_ARGS_NH4) ? &args.hopstore : args.next_hop; if (in_localip(satosin(psa)->sin_addr)) (*m0)->m_flags |= M_FASTFWD_OURS; (*m0)->m_flags |= M_IP_NEXTHOP; } #endif /* INET */ #ifdef INET6 if (args.flags & (IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)) { MPASS((args.flags & (IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)) != (IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)); MPASS((args.flags & (IPFW_ARGS_NH4 | IPFW_ARGS_NH4PTR)) == 0); len = sizeof(struct sockaddr_in6); psa = args.next_hop6; (*m0)->m_flags |= M_IP6_NEXTHOP; } #endif /* INET6 */ /* * Incoming packets should not be tagged so we do not * m_tag_find. Outgoing packets may be tagged, so we * reuse the tag if present. */ tag = (dir == DIR_IN) ? NULL : m_tag_find(*m0, PACKET_TAG_IPFORWARD, NULL); if (tag != NULL) { m_tag_unlink(*m0, tag); } else { tag = m_tag_get(PACKET_TAG_IPFORWARD, len, M_NOWAIT); if (tag == NULL) { ret = PFIL_DROPPED; break; } } if ((args.flags & IPFW_ARGS_NH6) == 0) bcopy(psa, tag + 1, len); m_tag_prepend(*m0, tag); ret = 0; #ifdef INET6 /* IPv6 next hop needs additional handling */ if (args.flags & (IPFW_ARGS_NH6 | IPFW_ARGS_NH6PTR)) { struct sockaddr_in6 *sa6; sa6 = satosin6(tag + 1); if (args.flags & IPFW_ARGS_NH6) { sa6->sin6_family = AF_INET6; sa6->sin6_len = sizeof(*sa6); sa6->sin6_addr = args.hopstore6.sin6_addr; sa6->sin6_port = args.hopstore6.sin6_port; sa6->sin6_scope_id = args.hopstore6.sin6_scope_id; } /* * If nh6 address is link-local we should convert * it to kernel internal form before doing any * comparisons. */ if (sa6_embedscope(sa6, V_ip6_use_defzone) != 0) { ret = PFIL_DROPPED; break; } if (in6_localip(&sa6->sin6_addr)) (*m0)->m_flags |= M_FASTFWD_OURS; } #endif /* INET6 */ } #endif /* INET || INET6 */ break; case IP_FW_DENY: ret = PFIL_DROPPED; break; case IP_FW_DUMMYNET: if (ip_dn_io_ptr == NULL) { ret = PFIL_DROPPED; break; } MPASS(args.flags & IPFW_ARGS_REF); if (mtod(*m0, struct ip *)->ip_v == 4) (void )ip_dn_io_ptr(m0, dir, &args); else if (mtod(*m0, struct ip *)->ip_v == 6) (void )ip_dn_io_ptr(m0, dir | PROTO_IPV6, &args); else { ret = PFIL_DROPPED; break; } /* * XXX should read the return value. * dummynet normally eats the packet and sets *m0=NULL * unless the packet can be sent immediately. In this * case args is updated and we should re-run the * check without clearing args. */ if (*m0 != NULL) goto again; ret = PFIL_CONSUMED; break; case IP_FW_TEE: case IP_FW_DIVERT: if (ip_divert_ptr == NULL) { ret = PFIL_DROPPED; break; } MPASS(args.flags & IPFW_ARGS_REF); (void )ipfw_divert(m0, dir, &args.rule, (ipfw == IP_FW_TEE) ? 1 : 0); /* continue processing for the original packet (tee). */ if (*m0) goto again; ret = PFIL_CONSUMED; break; case IP_FW_NGTEE: case IP_FW_NETGRAPH: if (ng_ipfw_input_p == NULL) { ret = PFIL_DROPPED; break; } MPASS(args.flags & IPFW_ARGS_REF); (void )ng_ipfw_input_p(m0, dir, &args, (ipfw == IP_FW_NGTEE) ? 1 : 0); if (ipfw == IP_FW_NGTEE) /* ignore errors for NGTEE */ goto again; /* continue with packet */ ret = PFIL_CONSUMED; break; case IP_FW_NAT: /* honor one-pass in case of successful nat */ if (V_fw_one_pass) break; goto again; case IP_FW_REASS: goto again; /* continue with packet */ + case IP_FW_NAT64: + ret = PFIL_CONSUMED; + break; + default: KASSERT(0, ("%s: unknown retval", __func__)); } if (ret != PFIL_PASS) { if (*m0) FREE_PKT(*m0); *m0 = NULL; } return (ret); } /* * ipfw processing for ethernet packets (in and out). */ static pfil_return_t ipfw_check_frame(struct mbuf **m0, struct ifnet *ifp, int dir, void *ruleset __unused, struct inpcb *inp) { struct ip_fw_args args; struct ether_header save_eh; struct ether_header *eh; struct m_tag *mtag; struct mbuf *m; pfil_return_t ret; int i; args.flags = IPFW_ARGS_ETHER; again: /* fetch start point from rule, if any. remove the tag if present. */ mtag = m_tag_locate(*m0, MTAG_IPFW_RULE, 0, NULL); if (mtag != NULL) { args.rule = *((struct ipfw_rule_ref *)(mtag+1)); m_tag_delete(*m0, mtag); if (args.rule.info & IPFW_ONEPASS) return (0); args.flags |= IPFW_ARGS_REF; } /* I need some amt of data to be contiguous */ m = *m0; i = min(m->m_pkthdr.len, max_protohdr); if (m->m_len < i) { m = m_pullup(m, i); if (m == NULL) { *m0 = m; return (0); } } eh = mtod(m, struct ether_header *); save_eh = *eh; /* save copy for restore below */ m_adj(m, ETHER_HDR_LEN); /* strip ethernet header */ args.m = m; /* the packet we are looking at */ args.oif = dir & PFIL_OUT ? ifp: NULL; /* destination, if any */ args.eh = &save_eh; /* MAC header for bridged/MAC packets */ args.inp = inp; /* used by ipfw uid/gid/jail rules */ i = ipfw_chk(&args); m = args.m; if (m != NULL) { /* * Restore Ethernet header, as needed, in case the * mbuf chain was replaced by ipfw. */ M_PREPEND(m, ETHER_HDR_LEN, M_NOWAIT); if (m == NULL) { *m0 = NULL; return (0); } if (eh != mtod(m, struct ether_header *)) bcopy(&save_eh, mtod(m, struct ether_header *), ETHER_HDR_LEN); } *m0 = m; ret = PFIL_PASS; /* Check result of ipfw_chk() */ switch (i) { case IP_FW_PASS: break; case IP_FW_DENY: ret = PFIL_DROPPED; break; case IP_FW_DUMMYNET: if (ip_dn_io_ptr == NULL) { ret = PFIL_DROPPED; break; } *m0 = NULL; dir = (dir & PFIL_IN) ? DIR_IN : DIR_OUT; MPASS(args.flags & IPFW_ARGS_REF); ip_dn_io_ptr(&m, dir | PROTO_LAYER2, &args); return (PFIL_CONSUMED); case IP_FW_NGTEE: case IP_FW_NETGRAPH: if (ng_ipfw_input_p == NULL) { ret = PFIL_DROPPED; break; } MPASS(args.flags & IPFW_ARGS_REF); (void )ng_ipfw_input_p(m0, (dir & PFIL_IN) ? DIR_IN : DIR_OUT, &args, (i == IP_FW_NGTEE) ? 1 : 0); if (i == IP_FW_NGTEE) /* ignore errors for NGTEE */ goto again; /* continue with packet */ ret = PFIL_CONSUMED; break; default: KASSERT(0, ("%s: unknown retval", __func__)); } if (ret != PFIL_PASS) { if (*m0) FREE_PKT(*m0); *m0 = NULL; } return (ret); } /* do the divert, return 1 on error 0 on success */ static int ipfw_divert(struct mbuf **m0, int incoming, struct ipfw_rule_ref *rule, int tee) { /* * ipfw_chk() has already tagged the packet with the divert tag. * If tee is set, copy packet and return original. * If not tee, consume packet and send it to divert socket. */ struct mbuf *clone; struct ip *ip = mtod(*m0, struct ip *); struct m_tag *tag; /* Cloning needed for tee? */ if (tee == 0) { clone = *m0; /* use the original mbuf */ *m0 = NULL; } else { clone = m_dup(*m0, M_NOWAIT); /* If we cannot duplicate the mbuf, we sacrifice the divert * chain and continue with the tee-ed packet. */ if (clone == NULL) return 1; } /* * Divert listeners can normally handle non-fragmented packets, * but we can only reass in the non-tee case. * This means that listeners on a tee rule may get fragments, * and have to live with that. * Note that we now have the 'reass' ipfw option so if we care * we can do it before a 'tee'. */ if (!tee) switch (ip->ip_v) { case IPVERSION: if (ntohs(ip->ip_off) & (IP_MF | IP_OFFMASK)) { int hlen; struct mbuf *reass; reass = ip_reass(clone); /* Reassemble packet. */ if (reass == NULL) return 0; /* not an error */ /* if reass = NULL then it was consumed by ip_reass */ /* * IP header checksum fixup after reassembly and leave header * in network byte order. */ ip = mtod(reass, struct ip *); hlen = ip->ip_hl << 2; ip->ip_sum = 0; if (hlen == sizeof(struct ip)) ip->ip_sum = in_cksum_hdr(ip); else ip->ip_sum = in_cksum(reass, hlen); clone = reass; } break; #ifdef INET6 case IPV6_VERSION >> 4: { struct ip6_hdr *const ip6 = mtod(clone, struct ip6_hdr *); if (ip6->ip6_nxt == IPPROTO_FRAGMENT) { int nxt, off; off = sizeof(struct ip6_hdr); nxt = frag6_input(&clone, &off, 0); if (nxt == IPPROTO_DONE) return (0); } break; } #endif } /* attach a tag to the packet with the reinject info */ tag = m_tag_alloc(MTAG_IPFW_RULE, 0, sizeof(struct ipfw_rule_ref), M_NOWAIT); if (tag == NULL) { FREE_PKT(clone); return 1; } *((struct ipfw_rule_ref *)(tag+1)) = *rule; m_tag_prepend(clone, tag); /* Do the dirty job... */ ip_divert_ptr(clone, incoming); return 0; } /* * attach or detach hooks for a given protocol family */ VNET_DEFINE_STATIC(pfil_hook_t, ipfw_inet_hook); #define V_ipfw_inet_hook VNET(ipfw_inet_hook) #ifdef INET6 VNET_DEFINE_STATIC(pfil_hook_t, ipfw_inet6_hook); #define V_ipfw_inet6_hook VNET(ipfw_inet6_hook) #endif VNET_DEFINE_STATIC(pfil_hook_t, ipfw_link_hook); #define V_ipfw_link_hook VNET(ipfw_link_hook) static int ipfw_hook(int onoff, int pf) { struct pfil_hook_args pha; struct pfil_link_args pla; pfil_hook_t *h; pha.pa_version = PFIL_VERSION; pha.pa_flags = PFIL_IN | PFIL_OUT; pha.pa_modname = "ipfw"; pha.pa_ruleset = NULL; pla.pa_version = PFIL_VERSION; pla.pa_flags = PFIL_IN | PFIL_OUT | PFIL_HEADPTR | PFIL_HOOKPTR; switch (pf) { case AF_INET: pha.pa_func = ipfw_check_packet; pha.pa_type = PFIL_TYPE_IP4; pha.pa_rulname = "default"; h = &V_ipfw_inet_hook; pla.pa_head = V_inet_pfil_head; break; #ifdef INET6 case AF_INET6: pha.pa_func = ipfw_check_packet; pha.pa_type = PFIL_TYPE_IP6; pha.pa_rulname = "default6"; h = &V_ipfw_inet6_hook; pla.pa_head = V_inet6_pfil_head; break; #endif case AF_LINK: pha.pa_func = ipfw_check_frame; pha.pa_type = PFIL_TYPE_ETHERNET; pha.pa_rulname = "default-link"; h = &V_ipfw_link_hook; pla.pa_head = V_link_pfil_head; break; } if (onoff) { *h = pfil_add_hook(&pha); pla.pa_hook = *h; (void)pfil_link(&pla); } else if (*h != NULL) pfil_remove_hook(*h); return 0; } int ipfw_attach_hooks(int arg) { int error = 0; if (arg == 0) /* detach */ ipfw_hook(0, AF_INET); else if (V_fw_enable && ipfw_hook(1, AF_INET) != 0) { error = ENOENT; /* see ip_fw_pfil.c::ipfw_hook() */ printf("ipfw_hook() error\n"); } #ifdef INET6 if (arg == 0) /* detach */ ipfw_hook(0, AF_INET6); else if (V_fw6_enable && ipfw_hook(1, AF_INET6) != 0) { error = ENOENT; printf("ipfw6_hook() error\n"); } #endif if (arg == 0) /* detach */ ipfw_hook(0, AF_LINK); else if (V_fwlink_enable && ipfw_hook(1, AF_LINK) != 0) { error = ENOENT; printf("ipfw_link_hook() error\n"); } return error; } int ipfw_chg_hook(SYSCTL_HANDLER_ARGS) { int newval; int error; int af; if (arg1 == &V_fw_enable) af = AF_INET; #ifdef INET6 else if (arg1 == &V_fw6_enable) af = AF_INET6; #endif else if (arg1 == &V_fwlink_enable) af = AF_LINK; else return (EINVAL); newval = *(int *)arg1; /* Handle sysctl change */ error = sysctl_handle_int(oidp, &newval, 0, req); if (error) return (error); /* Formalize new value */ newval = (newval) ? 1 : 0; if (*(int *)arg1 == newval) return (0); error = ipfw_hook(newval, af); if (error) return (error); *(int *)arg1 = newval; return (0); } /* end of file */ Index: projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_private.h =================================================================== --- projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_private.h (revision 345025) +++ projects/import-googletest-1.8.1/sys/netpfil/ipfw/ip_fw_private.h (revision 345026) @@ -1,837 +1,838 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2002-2009 Luigi Rizzo, Universita` di Pisa * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _IPFW2_PRIVATE_H #define _IPFW2_PRIVATE_H /* * Internal constants and data structures used by ipfw components * and not meant to be exported outside the kernel. */ #ifdef _KERNEL /* * For platforms that do not have SYSCTL support, we wrap the * SYSCTL_* into a function (one per file) to collect the values * into an array at module initialization. The wrapping macros, * SYSBEGIN() and SYSEND, are empty in the default case. */ #ifndef SYSBEGIN #define SYSBEGIN(x) #endif #ifndef SYSEND #define SYSEND #endif /* Return values from ipfw_chk() */ enum { IP_FW_PASS = 0, IP_FW_DENY, IP_FW_DIVERT, IP_FW_TEE, IP_FW_DUMMYNET, IP_FW_NETGRAPH, IP_FW_NGTEE, IP_FW_NAT, IP_FW_REASS, + IP_FW_NAT64, }; /* * Structure for collecting parameters to dummynet for ip6_output forwarding */ struct _ip6dn_args { struct ip6_pktopts *opt_or; int flags_or; struct ip6_moptions *im6o_or; struct ifnet *origifp_or; struct ifnet *ifp_or; struct sockaddr_in6 dst_or; u_long mtu_or; }; /* * Arguments for calling ipfw_chk() and dummynet_io(). We put them * all into a structure because this way it is easier and more * efficient to pass variables around and extend the interface. */ struct ip_fw_args { uint32_t flags; #define IPFW_ARGS_ETHER 0x0001 /* has valid ethernet header */ #define IPFW_ARGS_NH4 0x0002 /* has IPv4 next hop in hopstore */ #define IPFW_ARGS_NH6 0x0004 /* has IPv6 next hop in hopstore */ #define IPFW_ARGS_NH4PTR 0x0008 /* has IPv4 next hop in next_hop */ #define IPFW_ARGS_NH6PTR 0x0010 /* has IPv6 next hop in next_hop6 */ #define IPFW_ARGS_REF 0x0020 /* has valid ipfw_rule_ref */ /* * On return, it points to the matching rule. * On entry, rule.slot > 0 means the info is valid and * contains the starting rule for an ipfw search. * If chain_id == chain->id && slot >0 then jump to that slot. * Otherwise, we locate the first rule >= rulenum:rule_id */ struct ipfw_rule_ref rule; /* match/restart info */ struct ifnet *oif; /* output interface */ struct inpcb *inp; union { /* * We don't support forwarding on layer2, thus we can * keep eh pointer in this union. * next_hop[6] pointers can be used to point to next hop * stored in rule's opcode to avoid copying into hopstore. * Also, it is expected that all 0x1-0x10 flags are mutually * exclusive. */ struct ether_header *eh; /* for bridged packets */ struct sockaddr_in *next_hop; struct sockaddr_in6 *next_hop6; /* ipfw next hop storage */ struct sockaddr_in hopstore; struct ip_fw_nh6 { struct in6_addr sin6_addr; uint32_t sin6_scope_id; uint16_t sin6_port; } hopstore6; }; struct mbuf *m; /* the mbuf chain */ struct ipfw_flow_id f_id; /* grabbed from IP header */ }; MALLOC_DECLARE(M_IPFW); /* * Hooks sometime need to know the direction of the packet * (divert, dummynet, netgraph, ...) * We use a generic definition here, with bit0-1 indicating the * direction, bit 2 indicating layer2 or 3, bit 3-4 indicating the * specific protocol * indicating the protocol (if necessary) */ enum { DIR_MASK = 0x3, DIR_OUT = 0, DIR_IN = 1, DIR_FWD = 2, DIR_DROP = 3, PROTO_LAYER2 = 0x4, /* set for layer 2 */ /* PROTO_DEFAULT = 0, */ PROTO_IPV4 = 0x08, PROTO_IPV6 = 0x10, PROTO_IFB = 0x0c, /* layer2 + ifbridge */ /* PROTO_OLDBDG = 0x14, unused, old bridge */ }; /* wrapper for freeing a packet, in case we need to do more work */ #ifndef FREE_PKT #if defined(__linux__) || defined(_WIN32) #define FREE_PKT(m) netisr_dispatch(-1, m) #else #define FREE_PKT(m) m_freem(m) #endif #endif /* !FREE_PKT */ /* * Function definitions. */ int ipfw_chk(struct ip_fw_args *args); struct mbuf *ipfw_send_pkt(struct mbuf *, struct ipfw_flow_id *, u_int32_t, u_int32_t, int); /* attach (arg = 1) or detach (arg = 0) hooks */ int ipfw_attach_hooks(int); #ifdef NOTYET void ipfw_nat_destroy(void); #endif /* In ip_fw_log.c */ struct ip; struct ip_fw_chain; void ipfw_bpf_init(int); void ipfw_bpf_uninit(int); void ipfw_bpf_mtap2(void *, u_int, struct mbuf *); void ipfw_log(struct ip_fw_chain *chain, struct ip_fw *f, u_int hlen, struct ip_fw_args *args, struct mbuf *m, struct ifnet *oif, u_short offset, uint32_t tablearg, struct ip *ip); VNET_DECLARE(u_int64_t, norule_counter); #define V_norule_counter VNET(norule_counter) VNET_DECLARE(int, verbose_limit); #define V_verbose_limit VNET(verbose_limit) /* In ip_fw_dynamic.c */ struct sockopt_data; enum { /* result for matching dynamic rules */ MATCH_REVERSE = 0, MATCH_FORWARD, MATCH_NONE, MATCH_UNKNOWN, }; /* * Macro to determine that we need to do or redo dynamic state lookup. * direction == MATCH_UNKNOWN means that this is first lookup, then we need * to do lookup. * Otherwise check the state name, if previous lookup was for "any" name, * this means there is no state with specific name. Thus no need to do * lookup. If previous name was not "any", redo lookup for specific name. */ #define DYN_LOOKUP_NEEDED(p, cmd) \ ((p)->direction == MATCH_UNKNOWN || \ ((p)->kidx != 0 && (p)->kidx != (cmd)->arg1)) #define DYN_INFO_INIT(p) do { \ (p)->direction = MATCH_UNKNOWN; \ (p)->kidx = 0; \ } while (0) struct ipfw_dyn_info { uint16_t direction; /* match direction */ uint16_t kidx; /* state name kidx */ uint32_t hashval; /* hash value */ uint32_t version; /* bucket version */ uint32_t f_pos; }; int ipfw_dyn_install_state(struct ip_fw_chain *chain, struct ip_fw *rule, const ipfw_insn_limit *cmd, const struct ip_fw_args *args, const void *ulp, int pktlen, struct ipfw_dyn_info *info, uint32_t tablearg); struct ip_fw *ipfw_dyn_lookup_state(const struct ip_fw_args *args, const void *ulp, int pktlen, const ipfw_insn *cmd, struct ipfw_dyn_info *info); int ipfw_is_dyn_rule(struct ip_fw *rule); void ipfw_expire_dyn_states(struct ip_fw_chain *, ipfw_range_tlv *); void ipfw_get_dynamic(struct ip_fw_chain *chain, char **bp, const char *ep); int ipfw_dump_states(struct ip_fw_chain *chain, struct sockopt_data *sd); void ipfw_dyn_init(struct ip_fw_chain *); /* per-vnet initialization */ void ipfw_dyn_uninit(int); /* per-vnet deinitialization */ int ipfw_dyn_len(void); uint32_t ipfw_dyn_get_count(uint32_t *, int *); void ipfw_dyn_reset_eaction(struct ip_fw_chain *ch, uint16_t eaction_id, uint16_t default_id, uint16_t instance_id); /* common variables */ VNET_DECLARE(int, fw_one_pass); #define V_fw_one_pass VNET(fw_one_pass) VNET_DECLARE(int, fw_verbose); #define V_fw_verbose VNET(fw_verbose) VNET_DECLARE(struct ip_fw_chain, layer3_chain); #define V_layer3_chain VNET(layer3_chain) VNET_DECLARE(int, ipfw_vnet_ready); #define V_ipfw_vnet_ready VNET(ipfw_vnet_ready) VNET_DECLARE(u_int32_t, set_disable); #define V_set_disable VNET(set_disable) VNET_DECLARE(int, autoinc_step); #define V_autoinc_step VNET(autoinc_step) VNET_DECLARE(unsigned int, fw_tables_max); #define V_fw_tables_max VNET(fw_tables_max) VNET_DECLARE(unsigned int, fw_tables_sets); #define V_fw_tables_sets VNET(fw_tables_sets) struct tables_config; #ifdef _KERNEL /* * Here we have the structure representing an ipfw rule. * * It starts with a general area * followed by an array of one or more instructions, which the code * accesses as an array of 32-bit values. * * Given a rule pointer r: * * r->cmd is the start of the first instruction. * ACTION_PTR(r) is the start of the first action (things to do * once a rule matched). */ struct ip_fw { uint16_t act_ofs; /* offset of action in 32-bit units */ uint16_t cmd_len; /* # of 32-bit words in cmd */ uint16_t rulenum; /* rule number */ uint8_t set; /* rule set (0..31) */ uint8_t flags; /* currently unused */ counter_u64_t cntr; /* Pointer to rule counters */ uint32_t timestamp; /* tv_sec of last match */ uint32_t id; /* rule id */ uint32_t cached_id; /* used by jump_fast */ uint32_t cached_pos; /* used by jump_fast */ uint32_t refcnt; /* number of references */ struct ip_fw *next; /* linked list of deleted rules */ ipfw_insn cmd[1]; /* storage for commands */ }; #define IPFW_RULE_CNTR_SIZE (2 * sizeof(uint64_t)) #endif struct ip_fw_chain { struct ip_fw **map; /* array of rule ptrs to ease lookup */ uint32_t id; /* ruleset id */ int n_rules; /* number of static rules */ void *tablestate; /* runtime table info */ void *valuestate; /* runtime table value info */ int *idxmap; /* skipto array of rules */ void **srvstate; /* runtime service mappings */ #if defined( __linux__ ) || defined( _WIN32 ) spinlock_t rwmtx; #else struct rmlock rwmtx; #endif int static_len; /* total len of static rules (v0) */ uint32_t gencnt; /* NAT generation count */ LIST_HEAD(nat_list, cfg_nat) nat; /* list of nat entries */ struct ip_fw *default_rule; struct tables_config *tblcfg; /* tables module data */ void *ifcfg; /* interface module data */ int *idxmap_back; /* standby skipto array of rules */ struct namedobj_instance *srvmap; /* cfg name->number mappings */ #if defined( __linux__ ) || defined( _WIN32 ) spinlock_t uh_lock; #else struct rwlock uh_lock; /* lock for upper half */ #endif }; /* 64-byte structure representing multi-field table value */ struct table_value { uint32_t tag; /* O_TAG/O_TAGGED */ uint32_t pipe; /* O_PIPE/O_QUEUE */ uint16_t divert; /* O_DIVERT/O_TEE */ uint16_t skipto; /* skipto, CALLRET */ uint32_t netgraph; /* O_NETGRAPH/O_NGTEE */ uint32_t fib; /* O_SETFIB */ uint32_t nat; /* O_NAT */ uint32_t nh4; uint8_t dscp; uint8_t spare0; uint16_t spare1; /* -- 32 bytes -- */ struct in6_addr nh6; uint32_t limit; /* O_LIMIT */ uint32_t zoneid; /* scope zone id for nh6 */ uint64_t refcnt; /* Number of references */ }; struct named_object { TAILQ_ENTRY(named_object) nn_next; /* namehash */ TAILQ_ENTRY(named_object) nv_next; /* valuehash */ char *name; /* object name */ uint16_t etlv; /* Export TLV id */ uint8_t subtype;/* object subtype within class */ uint8_t set; /* set object belongs to */ uint16_t kidx; /* object kernel index */ uint16_t spare; uint32_t ocnt; /* object counter for internal use */ uint32_t refcnt; /* number of references */ }; TAILQ_HEAD(namedobjects_head, named_object); struct sockopt; /* used by tcp_var.h */ struct sockopt_data { caddr_t kbuf; /* allocated buffer */ size_t ksize; /* given buffer size */ size_t koff; /* data already used */ size_t kavail; /* number of bytes available */ size_t ktotal; /* total bytes pushed */ struct sockopt *sopt; /* socket data */ caddr_t sopt_val; /* sopt user buffer */ size_t valsize; /* original data size */ }; struct ipfw_ifc; typedef void (ipfw_ifc_cb)(struct ip_fw_chain *ch, void *cbdata, uint16_t ifindex); struct ipfw_iface { struct named_object no; char ifname[64]; int resolved; uint16_t ifindex; uint16_t spare; uint64_t gencnt; TAILQ_HEAD(, ipfw_ifc) consumers; }; struct ipfw_ifc { TAILQ_ENTRY(ipfw_ifc) next; struct ipfw_iface *iface; ipfw_ifc_cb *cb; void *cbdata; }; /* Macro for working with various counters */ #define IPFW_INC_RULE_COUNTER(_cntr, _bytes) do { \ counter_u64_add((_cntr)->cntr, 1); \ counter_u64_add((_cntr)->cntr + 1, _bytes); \ if ((_cntr)->timestamp != time_uptime) \ (_cntr)->timestamp = time_uptime; \ } while (0) #define IPFW_INC_DYN_COUNTER(_cntr, _bytes) do { \ (_cntr)->pcnt++; \ (_cntr)->bcnt += _bytes; \ } while (0) #define IPFW_ZERO_RULE_COUNTER(_cntr) do { \ counter_u64_zero((_cntr)->cntr); \ counter_u64_zero((_cntr)->cntr + 1); \ (_cntr)->timestamp = 0; \ } while (0) #define IPFW_ZERO_DYN_COUNTER(_cntr) do { \ (_cntr)->pcnt = 0; \ (_cntr)->bcnt = 0; \ } while (0) #define TARG_VAL(ch, k, f) ((struct table_value *)((ch)->valuestate))[k].f #define IP_FW_ARG_TABLEARG(ch, a, f) \ (((a) == IP_FW_TARG) ? TARG_VAL(ch, tablearg, f) : (a)) /* * The lock is heavily used by ip_fw2.c (the main file) and ip_fw_nat.c * so the variable and the macros must be here. */ #if defined( __linux__ ) || defined( _WIN32 ) #define IPFW_LOCK_INIT(_chain) do { \ rw_init(&(_chain)->rwmtx, "IPFW static rules"); \ rw_init(&(_chain)->uh_lock, "IPFW UH lock"); \ } while (0) #define IPFW_LOCK_DESTROY(_chain) do { \ rw_destroy(&(_chain)->rwmtx); \ rw_destroy(&(_chain)->uh_lock); \ } while (0) #define IPFW_RLOCK_ASSERT(_chain) rw_assert(&(_chain)->rwmtx, RA_RLOCKED) #define IPFW_WLOCK_ASSERT(_chain) rw_assert(&(_chain)->rwmtx, RA_WLOCKED) #define IPFW_RLOCK_TRACKER #define IPFW_RLOCK(p) rw_rlock(&(p)->rwmtx) #define IPFW_RUNLOCK(p) rw_runlock(&(p)->rwmtx) #define IPFW_WLOCK(p) rw_wlock(&(p)->rwmtx) #define IPFW_WUNLOCK(p) rw_wunlock(&(p)->rwmtx) #define IPFW_PF_RLOCK(p) IPFW_RLOCK(p) #define IPFW_PF_RUNLOCK(p) IPFW_RUNLOCK(p) #else /* FreeBSD */ #define IPFW_LOCK_INIT(_chain) do { \ rm_init_flags(&(_chain)->rwmtx, "IPFW static rules", RM_RECURSE); \ rw_init(&(_chain)->uh_lock, "IPFW UH lock"); \ } while (0) #define IPFW_LOCK_DESTROY(_chain) do { \ rm_destroy(&(_chain)->rwmtx); \ rw_destroy(&(_chain)->uh_lock); \ } while (0) #define IPFW_RLOCK_ASSERT(_chain) rm_assert(&(_chain)->rwmtx, RA_RLOCKED) #define IPFW_WLOCK_ASSERT(_chain) rm_assert(&(_chain)->rwmtx, RA_WLOCKED) #define IPFW_RLOCK_TRACKER struct rm_priotracker _tracker #define IPFW_RLOCK(p) rm_rlock(&(p)->rwmtx, &_tracker) #define IPFW_RUNLOCK(p) rm_runlock(&(p)->rwmtx, &_tracker) #define IPFW_WLOCK(p) rm_wlock(&(p)->rwmtx) #define IPFW_WUNLOCK(p) rm_wunlock(&(p)->rwmtx) #define IPFW_PF_RLOCK(p) IPFW_RLOCK(p) #define IPFW_PF_RUNLOCK(p) IPFW_RUNLOCK(p) #endif #define IPFW_UH_RLOCK_ASSERT(_chain) rw_assert(&(_chain)->uh_lock, RA_RLOCKED) #define IPFW_UH_WLOCK_ASSERT(_chain) rw_assert(&(_chain)->uh_lock, RA_WLOCKED) #define IPFW_UH_UNLOCK_ASSERT(_chain) rw_assert(&(_chain)->uh_lock, RA_UNLOCKED) #define IPFW_UH_RLOCK(p) rw_rlock(&(p)->uh_lock) #define IPFW_UH_RUNLOCK(p) rw_runlock(&(p)->uh_lock) #define IPFW_UH_WLOCK(p) rw_wlock(&(p)->uh_lock) #define IPFW_UH_WUNLOCK(p) rw_wunlock(&(p)->uh_lock) struct obj_idx { uint16_t uidx; /* internal index supplied by userland */ uint16_t kidx; /* kernel object index */ uint16_t off; /* tlv offset from rule end in 4-byte words */ uint8_t spare; uint8_t type; /* object type within its category */ }; struct rule_check_info { uint16_t flags; /* rule-specific check flags */ uint16_t object_opcodes; /* num of opcodes referencing objects */ uint16_t urule_numoff; /* offset of rulenum in bytes */ uint8_t version; /* rule version */ uint8_t spare; ipfw_obj_ctlv *ctlv; /* name TLV containter */ struct ip_fw *krule; /* resulting rule pointer */ caddr_t urule; /* original rule pointer */ struct obj_idx obuf[8]; /* table references storage */ }; /* Legacy interface support */ /* * FreeBSD 8 export rule format */ struct ip_fw_rule0 { struct ip_fw *x_next; /* linked list of rules */ struct ip_fw *next_rule; /* ptr to next [skipto] rule */ /* 'next_rule' is used to pass up 'set_disable' status */ uint16_t act_ofs; /* offset of action in 32-bit units */ uint16_t cmd_len; /* # of 32-bit words in cmd */ uint16_t rulenum; /* rule number */ uint8_t set; /* rule set (0..31) */ uint8_t _pad; /* padding */ uint32_t id; /* rule id */ /* These fields are present in all rules. */ uint64_t pcnt; /* Packet counter */ uint64_t bcnt; /* Byte counter */ uint32_t timestamp; /* tv_sec of last match */ ipfw_insn cmd[1]; /* storage for commands */ }; struct ip_fw_bcounter0 { uint64_t pcnt; /* Packet counter */ uint64_t bcnt; /* Byte counter */ uint32_t timestamp; /* tv_sec of last match */ }; /* Kernel rule length */ /* * RULE _K_ SIZE _V_ -> * get kernel size from userland rool version _V_. * RULE _U_ SIZE _V_ -> * get user size version _V_ from kernel rule * RULESIZE _V_ -> * get user size rule length */ /* FreeBSD8 <> current kernel format */ #define RULEUSIZE0(r) (sizeof(struct ip_fw_rule0) + (r)->cmd_len * 4 - 4) #define RULEKSIZE0(r) roundup2((sizeof(struct ip_fw) + (r)->cmd_len*4 - 4), 8) /* FreeBSD11 <> current kernel format */ #define RULEUSIZE1(r) (roundup2(sizeof(struct ip_fw_rule) + \ (r)->cmd_len * 4 - 4, 8)) #define RULEKSIZE1(r) roundup2((sizeof(struct ip_fw) + (r)->cmd_len*4 - 4), 8) /* * Tables/Objects index rewriting code */ /* Default and maximum number of ipfw tables/objects. */ #define IPFW_TABLES_MAX 65536 #define IPFW_TABLES_DEFAULT 128 #define IPFW_OBJECTS_MAX 65536 #define IPFW_OBJECTS_DEFAULT 1024 #define CHAIN_TO_SRV(ch) ((ch)->srvmap) #define SRV_OBJECT(ch, idx) ((ch)->srvstate[(idx)]) struct tid_info { uint32_t set; /* table set */ uint16_t uidx; /* table index */ uint8_t type; /* table type */ uint8_t atype; uint8_t spare; int tlen; /* Total TLV size block */ void *tlvs; /* Pointer to first TLV */ }; /* * Classifier callback. Checks if @cmd opcode contains kernel object reference. * If true, returns its index and type. * Returns 0 if match is found, 1 overwise. */ typedef int (ipfw_obj_rw_cl)(ipfw_insn *cmd, uint16_t *puidx, uint8_t *ptype); /* * Updater callback. Sets kernel object reference index to @puidx */ typedef void (ipfw_obj_rw_upd)(ipfw_insn *cmd, uint16_t puidx); /* * Finder callback. Tries to find named object by name (specified via @ti). * Stores found named object pointer in @pno. * If object was not found, NULL is stored. * * Return 0 if input data was valid. */ typedef int (ipfw_obj_fname_cb)(struct ip_fw_chain *ch, struct tid_info *ti, struct named_object **pno); /* * Another finder callback. Tries to findex named object by kernel index. * * Returns pointer to named object or NULL. */ typedef struct named_object *(ipfw_obj_fidx_cb)(struct ip_fw_chain *ch, uint16_t kidx); /* * Object creator callback. Tries to create object specified by @ti. * Stores newly-allocated object index in @pkidx. * * Returns 0 on success. */ typedef int (ipfw_obj_create_cb)(struct ip_fw_chain *ch, struct tid_info *ti, uint16_t *pkidx); /* * Object destroy callback. Intended to free resources allocated by * create_object callback. */ typedef void (ipfw_obj_destroy_cb)(struct ip_fw_chain *ch, struct named_object *no); /* * Sets handler callback. Handles moving and swaping set of named object. * SWAP_ALL moves all named objects from set `set' to `new_set' and vise versa; * TEST_ALL checks that there aren't any named object with conflicting names; * MOVE_ALL moves all named objects from set `set' to `new_set'; * COUNT_ONE used to count number of references used by object with kidx `set'; * TEST_ONE checks that named object with kidx `set' can be moved to `new_set`; * MOVE_ONE moves named object with kidx `set' to set `new_set'. */ enum ipfw_sets_cmd { SWAP_ALL = 0, TEST_ALL, MOVE_ALL, COUNT_ONE, TEST_ONE, MOVE_ONE }; typedef int (ipfw_obj_sets_cb)(struct ip_fw_chain *ch, uint16_t set, uint8_t new_set, enum ipfw_sets_cmd cmd); struct opcode_obj_rewrite { uint32_t opcode; /* Opcode to act upon */ uint32_t etlv; /* Relevant export TLV id */ ipfw_obj_rw_cl *classifier; /* Check if rewrite is needed */ ipfw_obj_rw_upd *update; /* update cmd with new value */ ipfw_obj_fname_cb *find_byname; /* Find named object by name */ ipfw_obj_fidx_cb *find_bykidx; /* Find named object by kidx */ ipfw_obj_create_cb *create_object; /* Create named object */ ipfw_obj_destroy_cb *destroy_object;/* Destroy named object */ ipfw_obj_sets_cb *manage_sets; /* Swap or move sets */ }; #define IPFW_ADD_OBJ_REWRITER(f, c) do { \ if ((f) != 0) \ ipfw_add_obj_rewriter(c, \ sizeof(c) / sizeof(c[0])); \ } while(0) #define IPFW_DEL_OBJ_REWRITER(l, c) do { \ if ((l) != 0) \ ipfw_del_obj_rewriter(c, \ sizeof(c) / sizeof(c[0])); \ } while(0) /* In ip_fw_iface.c */ int ipfw_iface_init(void); void ipfw_iface_destroy(void); void vnet_ipfw_iface_destroy(struct ip_fw_chain *ch); int ipfw_iface_ref(struct ip_fw_chain *ch, char *name, struct ipfw_ifc *ic); void ipfw_iface_unref(struct ip_fw_chain *ch, struct ipfw_ifc *ic); void ipfw_iface_add_notify(struct ip_fw_chain *ch, struct ipfw_ifc *ic); void ipfw_iface_del_notify(struct ip_fw_chain *ch, struct ipfw_ifc *ic); /* In ip_fw_sockopt.c */ void ipfw_init_skipto_cache(struct ip_fw_chain *chain); void ipfw_destroy_skipto_cache(struct ip_fw_chain *chain); int ipfw_find_rule(struct ip_fw_chain *chain, uint32_t key, uint32_t id); int ipfw_ctl3(struct sockopt *sopt); int ipfw_add_protected_rule(struct ip_fw_chain *chain, struct ip_fw *rule, int locked); void ipfw_reap_add(struct ip_fw_chain *chain, struct ip_fw **head, struct ip_fw *rule); void ipfw_reap_rules(struct ip_fw *head); void ipfw_init_counters(void); void ipfw_destroy_counters(void); struct ip_fw *ipfw_alloc_rule(struct ip_fw_chain *chain, size_t rulesize); void ipfw_free_rule(struct ip_fw *rule); int ipfw_match_range(struct ip_fw *rule, ipfw_range_tlv *rt); int ipfw_mark_object_kidx(uint32_t *bmask, uint16_t etlv, uint16_t kidx); typedef int (sopt_handler_f)(struct ip_fw_chain *ch, ip_fw3_opheader *op3, struct sockopt_data *sd); struct ipfw_sopt_handler { uint16_t opcode; uint8_t version; uint8_t dir; sopt_handler_f *handler; uint64_t refcnt; }; #define HDIR_SET 0x01 /* Handler is used to set some data */ #define HDIR_GET 0x02 /* Handler is used to retrieve data */ #define HDIR_BOTH HDIR_GET|HDIR_SET void ipfw_init_sopt_handler(void); void ipfw_destroy_sopt_handler(void); void ipfw_add_sopt_handler(struct ipfw_sopt_handler *sh, size_t count); int ipfw_del_sopt_handler(struct ipfw_sopt_handler *sh, size_t count); caddr_t ipfw_get_sopt_space(struct sockopt_data *sd, size_t needed); caddr_t ipfw_get_sopt_header(struct sockopt_data *sd, size_t needed); #define IPFW_ADD_SOPT_HANDLER(f, c) do { \ if ((f) != 0) \ ipfw_add_sopt_handler(c, \ sizeof(c) / sizeof(c[0])); \ } while(0) #define IPFW_DEL_SOPT_HANDLER(l, c) do { \ if ((l) != 0) \ ipfw_del_sopt_handler(c, \ sizeof(c) / sizeof(c[0])); \ } while(0) struct namedobj_instance; typedef int (objhash_cb_t)(struct namedobj_instance *ni, struct named_object *, void *arg); typedef uint32_t (objhash_hash_f)(struct namedobj_instance *ni, const void *key, uint32_t kopt); typedef int (objhash_cmp_f)(struct named_object *no, const void *key, uint32_t kopt); struct namedobj_instance *ipfw_objhash_create(uint32_t items); void ipfw_objhash_destroy(struct namedobj_instance *); void ipfw_objhash_bitmap_alloc(uint32_t items, void **idx, int *pblocks); void ipfw_objhash_bitmap_merge(struct namedobj_instance *ni, void **idx, int *blocks); void ipfw_objhash_bitmap_swap(struct namedobj_instance *ni, void **idx, int *blocks); void ipfw_objhash_bitmap_free(void *idx, int blocks); void ipfw_objhash_set_hashf(struct namedobj_instance *ni, objhash_hash_f *f); struct named_object *ipfw_objhash_lookup_name(struct namedobj_instance *ni, uint32_t set, char *name); struct named_object *ipfw_objhash_lookup_name_type(struct namedobj_instance *ni, uint32_t set, uint32_t type, const char *name); struct named_object *ipfw_objhash_lookup_kidx(struct namedobj_instance *ni, uint16_t idx); int ipfw_objhash_same_name(struct namedobj_instance *ni, struct named_object *a, struct named_object *b); void ipfw_objhash_add(struct namedobj_instance *ni, struct named_object *no); void ipfw_objhash_del(struct namedobj_instance *ni, struct named_object *no); uint32_t ipfw_objhash_count(struct namedobj_instance *ni); uint32_t ipfw_objhash_count_type(struct namedobj_instance *ni, uint16_t type); int ipfw_objhash_foreach(struct namedobj_instance *ni, objhash_cb_t *f, void *arg); int ipfw_objhash_foreach_type(struct namedobj_instance *ni, objhash_cb_t *f, void *arg, uint16_t type); int ipfw_objhash_free_idx(struct namedobj_instance *ni, uint16_t idx); int ipfw_objhash_alloc_idx(void *n, uint16_t *pidx); void ipfw_objhash_set_funcs(struct namedobj_instance *ni, objhash_hash_f *hash_f, objhash_cmp_f *cmp_f); int ipfw_objhash_find_type(struct namedobj_instance *ni, struct tid_info *ti, uint32_t etlv, struct named_object **pno); void ipfw_export_obj_ntlv(struct named_object *no, ipfw_obj_ntlv *ntlv); ipfw_obj_ntlv *ipfw_find_name_tlv_type(void *tlvs, int len, uint16_t uidx, uint32_t etlv); void ipfw_init_obj_rewriter(void); void ipfw_destroy_obj_rewriter(void); void ipfw_add_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count); int ipfw_del_obj_rewriter(struct opcode_obj_rewrite *rw, size_t count); int create_objects_compat(struct ip_fw_chain *ch, ipfw_insn *cmd, struct obj_idx *oib, struct obj_idx *pidx, struct tid_info *ti); void update_opcode_kidx(ipfw_insn *cmd, uint16_t idx); int classify_opcode_kidx(ipfw_insn *cmd, uint16_t *puidx); void ipfw_init_srv(struct ip_fw_chain *ch); void ipfw_destroy_srv(struct ip_fw_chain *ch); int ipfw_check_object_name_generic(const char *name); int ipfw_obj_manage_sets(struct namedobj_instance *ni, uint16_t type, uint16_t set, uint8_t new_set, enum ipfw_sets_cmd cmd); /* In ip_fw_eaction.c */ typedef int (ipfw_eaction_t)(struct ip_fw_chain *ch, struct ip_fw_args *args, ipfw_insn *cmd, int *done); int ipfw_eaction_init(struct ip_fw_chain *ch, int first); void ipfw_eaction_uninit(struct ip_fw_chain *ch, int last); uint16_t ipfw_add_eaction(struct ip_fw_chain *ch, ipfw_eaction_t handler, const char *name); int ipfw_del_eaction(struct ip_fw_chain *ch, uint16_t eaction_id); int ipfw_run_eaction(struct ip_fw_chain *ch, struct ip_fw_args *args, ipfw_insn *cmd, int *done); int ipfw_reset_eaction(struct ip_fw_chain *ch, struct ip_fw *rule, uint16_t eaction_id, uint16_t default_id, uint16_t instance_id); int ipfw_reset_eaction_instance(struct ip_fw_chain *ch, uint16_t eaction_id, uint16_t instance_id); /* In ip_fw_table.c */ struct table_info; typedef int (table_lookup_t)(struct table_info *ti, void *key, uint32_t keylen, uint32_t *val); int ipfw_lookup_table(struct ip_fw_chain *ch, uint16_t tbl, uint16_t plen, void *paddr, uint32_t *val); struct named_object *ipfw_objhash_lookup_table_kidx(struct ip_fw_chain *ch, uint16_t kidx); int ipfw_ref_table(struct ip_fw_chain *ch, ipfw_obj_ntlv *ntlv, uint16_t *kidx); void ipfw_unref_table(struct ip_fw_chain *ch, uint16_t kidx); int ipfw_init_tables(struct ip_fw_chain *ch, int first); int ipfw_resize_tables(struct ip_fw_chain *ch, unsigned int ntables); int ipfw_switch_tables_namespace(struct ip_fw_chain *ch, unsigned int nsets); void ipfw_destroy_tables(struct ip_fw_chain *ch, int last); /* In ip_fw_nat.c -- XXX to be moved to ip_var.h */ extern struct cfg_nat *(*lookup_nat_ptr)(struct nat_list *, int); typedef int ipfw_nat_t(struct ip_fw_args *, struct cfg_nat *, struct mbuf *); typedef int ipfw_nat_cfg_t(struct sockopt *); VNET_DECLARE(int, ipfw_nat_ready); #define V_ipfw_nat_ready VNET(ipfw_nat_ready) #define IPFW_NAT_LOADED (V_ipfw_nat_ready) extern ipfw_nat_t *ipfw_nat_ptr; extern ipfw_nat_cfg_t *ipfw_nat_cfg_ptr; extern ipfw_nat_cfg_t *ipfw_nat_del_ptr; extern ipfw_nat_cfg_t *ipfw_nat_get_cfg_ptr; extern ipfw_nat_cfg_t *ipfw_nat_get_log_ptr; /* Helper functions for IP checksum adjustment */ static __inline uint16_t cksum_add(uint16_t sum, uint16_t a) { uint16_t res; res = sum + a; return (res + (res < a)); } static __inline uint16_t cksum_adjust(uint16_t oldsum, uint16_t old, uint16_t new) { return (~cksum_add(cksum_add(~oldsum, ~old), new)); } #endif /* _KERNEL */ #endif /* _IPFW2_PRIVATE_H */ Index: projects/import-googletest-1.8.1/sys/netpfil/ipfw/nat64/nat64_translate.c =================================================================== --- projects/import-googletest-1.8.1/sys/netpfil/ipfw/nat64/nat64_translate.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/netpfil/ipfw/nat64/nat64_translate.c (revision 345026) @@ -1,1629 +1,1631 @@ /*- * Copyright (c) 2015-2018 Yandex LLC * Copyright (c) 2015-2018 Andrey V. Elsukov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ip_fw_nat64.h" #include "nat64_translate.h" typedef int (*nat64_output_t)(struct ifnet *, struct mbuf *, struct sockaddr *, struct nat64_counters *, void *); typedef int (*nat64_output_one_t)(struct mbuf *, struct nat64_counters *, void *); static int nat64_find_route4(struct nhop4_basic *, struct sockaddr_in *, struct mbuf *); static int nat64_find_route6(struct nhop6_basic *, struct sockaddr_in6 *, struct mbuf *); static int nat64_output_one(struct mbuf *, struct nat64_counters *, void *); static int nat64_output(struct ifnet *, struct mbuf *, struct sockaddr *, struct nat64_counters *, void *); static int nat64_direct_output_one(struct mbuf *, struct nat64_counters *, void *); static int nat64_direct_output(struct ifnet *, struct mbuf *, struct sockaddr *, struct nat64_counters *, void *); struct nat64_methods { nat64_output_t output; nat64_output_one_t output_one; }; static const struct nat64_methods nat64_netisr = { .output = nat64_output, .output_one = nat64_output_one }; static const struct nat64_methods nat64_direct = { .output = nat64_direct_output, .output_one = nat64_direct_output_one }; VNET_DEFINE_STATIC(const struct nat64_methods *, nat64out) = &nat64_netisr; #define V_nat64out VNET(nat64out) void nat64_set_output_method(int direct) { V_nat64out = direct != 0 ? &nat64_direct: &nat64_netisr; } int nat64_get_output_method(void) { return (V_nat64out == &nat64_direct ? 1: 0); } static void nat64_log(struct pfloghdr *logdata, struct mbuf *m, sa_family_t family) { logdata->dir = PF_OUT; logdata->af = family; ipfw_bpf_mtap2(logdata, PFLOG_HDRLEN, m); } static int nat64_direct_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct nat64_counters *stats, void *logdata) { int error; if (logdata != NULL) nat64_log(logdata, m, dst->sa_family); error = (*ifp->if_output)(ifp, m, dst, NULL); if (error != 0) NAT64STAT_INC(stats, oerrors); return (error); } static int nat64_direct_output_one(struct mbuf *m, struct nat64_counters *stats, void *logdata) { struct nhop6_basic nh6; struct nhop4_basic nh4; struct sockaddr_in6 dst6; struct sockaddr_in dst4; struct sockaddr *dst; struct ip6_hdr *ip6; struct ip *ip4; struct ifnet *ifp; int error; ip4 = mtod(m, struct ip *); switch (ip4->ip_v) { case IPVERSION: dst4.sin_addr = ip4->ip_dst; error = nat64_find_route4(&nh4, &dst4, m); if (error != 0) NAT64STAT_INC(stats, noroute4); else { ifp = nh4.nh_ifp; dst = (struct sockaddr *)&dst4; } break; case (IPV6_VERSION >> 4): ip6 = mtod(m, struct ip6_hdr *); dst6.sin6_addr = ip6->ip6_dst; error = nat64_find_route6(&nh6, &dst6, m); if (error != 0) NAT64STAT_INC(stats, noroute6); else { ifp = nh6.nh_ifp; dst = (struct sockaddr *)&dst6; } break; default: m_freem(m); NAT64STAT_INC(stats, dropped); DPRINTF(DP_DROPS, "dropped due to unknown IP version"); return (EAFNOSUPPORT); } if (error != 0) { m_freem(m); return (EHOSTUNREACH); } if (logdata != NULL) nat64_log(logdata, m, dst->sa_family); error = (*ifp->if_output)(ifp, m, dst, NULL); if (error != 0) NAT64STAT_INC(stats, oerrors); return (error); } static int nat64_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct nat64_counters *stats, void *logdata) { struct ip *ip4; int ret, af; ip4 = mtod(m, struct ip *); switch (ip4->ip_v) { case IPVERSION: af = AF_INET; ret = NETISR_IP; break; case (IPV6_VERSION >> 4): af = AF_INET6; ret = NETISR_IPV6; break; default: m_freem(m); NAT64STAT_INC(stats, dropped); DPRINTF(DP_DROPS, "unknown IP version"); return (EAFNOSUPPORT); } if (logdata != NULL) nat64_log(logdata, m, af); + if (m->m_pkthdr.rcvif == NULL) + m->m_pkthdr.rcvif = V_loif; ret = netisr_queue(ret, m); if (ret != 0) NAT64STAT_INC(stats, oerrors); return (ret); } static int nat64_output_one(struct mbuf *m, struct nat64_counters *stats, void *logdata) { return (nat64_output(NULL, m, NULL, stats, logdata)); } /* * Check the given IPv6 prefix and length according to RFC6052: * The prefixes can only have one of the following lengths: * 32, 40, 48, 56, 64, or 96 (The Well-Known Prefix is 96 bits long). * Returns zero on success, otherwise EINVAL. */ int nat64_check_prefix6(const struct in6_addr *prefix, int length) { switch (length) { case 32: case 40: case 48: case 56: case 64: /* Well-known prefix has 96 prefix length */ if (IN6_IS_ADDR_WKPFX(prefix)) return (EINVAL); /* FALLTHROUGH */ case 96: /* Bits 64 to 71 must be set to zero */ if (prefix->__u6_addr.__u6_addr8[8] != 0) return (EINVAL); /* Some extra checks */ if (IN6_IS_ADDR_MULTICAST(prefix) || IN6_IS_ADDR_UNSPECIFIED(prefix) || IN6_IS_ADDR_LOOPBACK(prefix)) return (EINVAL); return (0); } return (EINVAL); } int nat64_check_private_ip4(const struct nat64_config *cfg, in_addr_t ia) { if (V_nat64_allow_private) return (0); /* WKPFX must not be used to represent non-global IPv4 addresses */ if (cfg->flags & NAT64_WKPFX) { /* IN_PRIVATE */ if ((ia & htonl(0xff000000)) == htonl(0x0a000000) || (ia & htonl(0xfff00000)) == htonl(0xac100000) || (ia & htonl(0xffff0000)) == htonl(0xc0a80000)) return (1); /* * RFC 5735: * 192.0.0.0/24 - reserved for IETF protocol assignments * 192.88.99.0/24 - for use as 6to4 relay anycast addresses * 198.18.0.0/15 - for use in benchmark tests * 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24 - for use * in documentation and example code */ if ((ia & htonl(0xffffff00)) == htonl(0xc0000000) || (ia & htonl(0xffffff00)) == htonl(0xc0586300) || (ia & htonl(0xfffffe00)) == htonl(0xc6120000) || (ia & htonl(0xffffff00)) == htonl(0xc0000200) || (ia & htonl(0xfffffe00)) == htonl(0xc6336400) || (ia & htonl(0xffffff00)) == htonl(0xcb007100)) return (1); } return (0); } void nat64_embed_ip4(const struct nat64_config *cfg, in_addr_t ia, struct in6_addr *ip6) { /* assume the prefix6 is properly filled with zeros */ bcopy(&cfg->prefix6, ip6, sizeof(*ip6)); switch (cfg->plen6) { case 32: case 96: ip6->s6_addr32[cfg->plen6 / 32] = ia; break; case 40: case 48: case 56: #if BYTE_ORDER == BIG_ENDIAN ip6->s6_addr32[1] = cfg->prefix6.s6_addr32[1] | (ia >> (cfg->plen6 % 32)); ip6->s6_addr32[2] = ia << (24 - cfg->plen6 % 32); #elif BYTE_ORDER == LITTLE_ENDIAN ip6->s6_addr32[1] = cfg->prefix6.s6_addr32[1] | (ia << (cfg->plen6 % 32)); ip6->s6_addr32[2] = ia >> (24 - cfg->plen6 % 32); #endif break; case 64: #if BYTE_ORDER == BIG_ENDIAN ip6->s6_addr32[2] = ia >> 8; ip6->s6_addr32[3] = ia << 24; #elif BYTE_ORDER == LITTLE_ENDIAN ip6->s6_addr32[2] = ia << 8; ip6->s6_addr32[3] = ia >> 24; #endif break; default: panic("Wrong plen6"); }; ip6->s6_addr8[8] = 0; } in_addr_t nat64_extract_ip4(const struct nat64_config *cfg, const struct in6_addr *ip6) { in_addr_t ia; /* * According to RFC 6052 p2.2: * IPv4-embedded IPv6 addresses are composed of a variable-length * prefix, the embedded IPv4 address, and a variable length suffix. * The suffix bits are reserved for future extensions and SHOULD * be set to zero. */ switch (cfg->plen6) { case 32: if (ip6->s6_addr32[3] != 0 || ip6->s6_addr32[2] != 0) goto badip6; break; case 40: if (ip6->s6_addr32[3] != 0 || (ip6->s6_addr32[2] & htonl(0xff00ffff)) != 0) goto badip6; break; case 48: if (ip6->s6_addr32[3] != 0 || (ip6->s6_addr32[2] & htonl(0xff0000ff)) != 0) goto badip6; break; case 56: if (ip6->s6_addr32[3] != 0 || ip6->s6_addr8[8] != 0) goto badip6; break; case 64: if (ip6->s6_addr8[8] != 0 || (ip6->s6_addr32[3] & htonl(0x00ffffff)) != 0) goto badip6; }; switch (cfg->plen6) { case 32: case 96: ia = ip6->s6_addr32[cfg->plen6 / 32]; break; case 40: case 48: case 56: #if BYTE_ORDER == BIG_ENDIAN ia = (ip6->s6_addr32[1] << (cfg->plen6 % 32)) | (ip6->s6_addr32[2] >> (24 - cfg->plen6 % 32)); #elif BYTE_ORDER == LITTLE_ENDIAN ia = (ip6->s6_addr32[1] >> (cfg->plen6 % 32)) | (ip6->s6_addr32[2] << (24 - cfg->plen6 % 32)); #endif break; case 64: #if BYTE_ORDER == BIG_ENDIAN ia = (ip6->s6_addr32[2] << 8) | (ip6->s6_addr32[3] >> 24); #elif BYTE_ORDER == LITTLE_ENDIAN ia = (ip6->s6_addr32[2] >> 8) | (ip6->s6_addr32[3] << 24); #endif break; default: return (0); }; if (nat64_check_ip4(ia) != 0 || nat64_check_private_ip4(cfg, ia) != 0) goto badip4; return (ia); badip4: DPRINTF(DP_GENERIC | DP_DROPS, "invalid destination address: %08x", ia); return (0); badip6: DPRINTF(DP_GENERIC | DP_DROPS, "invalid IPv4-embedded IPv6 address"); return (0); } /* * According to RFC 1624 the equation for incremental checksum update is: * HC' = ~(~HC + ~m + m') -- [Eqn. 3] * HC' = HC - ~m - m' -- [Eqn. 4] * So, when we are replacing IPv4 addresses to IPv6, we * can assume, that new bytes previously were zeros, and vise versa - * when we replacing IPv6 addresses to IPv4, now unused bytes become * zeros. The payload length in pseudo header has bigger size, but one * half of it should be zero. Using the equation 4 we get: * HC' = HC - (~m0 + m0') -- m0 is first changed word * HC' = (HC - (~m0 + m0')) - (~m1 + m1') -- m1 is second changed word * HC' = HC - ~m0 - m0' - ~m1 - m1' - ... = * = HC - sum(~m[i] + m'[i]) * * The function result should be used as follows: * IPv6 to IPv4: HC' = cksum_add(HC, result) * IPv4 to IPv6: HC' = cksum_add(HC, ~result) */ static NAT64NOINLINE uint16_t nat64_cksum_convert(struct ip6_hdr *ip6, struct ip *ip) { uint32_t sum; uint16_t *p; sum = ~ip->ip_src.s_addr >> 16; sum += ~ip->ip_src.s_addr & 0xffff; sum += ~ip->ip_dst.s_addr >> 16; sum += ~ip->ip_dst.s_addr & 0xffff; for (p = (uint16_t *)&ip6->ip6_src; p < (uint16_t *)(&ip6->ip6_src + 2); p++) sum += *p; while (sum >> 16) sum = (sum & 0xffff) + (sum >> 16); return (sum); } static NAT64NOINLINE void nat64_init_ip4hdr(const struct ip6_hdr *ip6, const struct ip6_frag *frag, uint16_t plen, uint8_t proto, struct ip *ip) { /* assume addresses are already initialized */ ip->ip_v = IPVERSION; ip->ip_hl = sizeof(*ip) >> 2; ip->ip_tos = (ntohl(ip6->ip6_flow) >> 20) & 0xff; ip->ip_len = htons(sizeof(*ip) + plen); ip->ip_ttl = ip6->ip6_hlim; /* Forwarding code will decrement TTL for netisr based output. */ if (V_nat64out == &nat64_direct) ip->ip_ttl -= IPV6_HLIMDEC; ip->ip_sum = 0; ip->ip_p = (proto == IPPROTO_ICMPV6) ? IPPROTO_ICMP: proto; ip_fillid(ip); if (frag != NULL) { ip->ip_off = htons(ntohs(frag->ip6f_offlg) >> 3); if (frag->ip6f_offlg & IP6F_MORE_FRAG) ip->ip_off |= htons(IP_MF); } else { ip->ip_off = htons(IP_DF); } ip->ip_sum = in_cksum_hdr(ip); } #define FRAGSZ(mtu) ((mtu) - sizeof(struct ip6_hdr) - sizeof(struct ip6_frag)) static NAT64NOINLINE int nat64_fragment6(struct nat64_counters *stats, struct ip6_hdr *ip6, struct mbufq *mq, struct mbuf *m, uint32_t mtu, uint16_t ip_id, uint16_t ip_off) { struct ip6_frag ip6f; struct mbuf *n; uint16_t hlen, len, offset; int plen; plen = ntohs(ip6->ip6_plen); hlen = sizeof(struct ip6_hdr); /* Fragmentation isn't needed */ if (ip_off == 0 && plen <= mtu - hlen) { M_PREPEND(m, hlen, M_NOWAIT); if (m == NULL) { NAT64STAT_INC(stats, nomem); return (ENOMEM); } bcopy(ip6, mtod(m, void *), hlen); if (mbufq_enqueue(mq, m) != 0) { m_freem(m); NAT64STAT_INC(stats, dropped); DPRINTF(DP_DROPS, "dropped due to mbufq overflow"); return (ENOBUFS); } return (0); } hlen += sizeof(struct ip6_frag); ip6f.ip6f_reserved = 0; ip6f.ip6f_nxt = ip6->ip6_nxt; ip6->ip6_nxt = IPPROTO_FRAGMENT; if (ip_off != 0) { /* * We have got an IPv4 fragment. * Use offset value and ip_id from original fragment. */ ip6f.ip6f_ident = htonl(ntohs(ip_id)); offset = (ntohs(ip_off) & IP_OFFMASK) << 3; NAT64STAT_INC(stats, ifrags); } else { /* The packet size exceeds interface MTU */ ip6f.ip6f_ident = htonl(ip6_randomid()); offset = 0; /* First fragment*/ } while (plen > 0 && m != NULL) { n = NULL; len = FRAGSZ(mtu) & ~7; if (len > plen) len = plen; ip6->ip6_plen = htons(len + sizeof(ip6f)); ip6f.ip6f_offlg = ntohs(offset); if (len < plen || (ip_off & htons(IP_MF)) != 0) ip6f.ip6f_offlg |= IP6F_MORE_FRAG; offset += len; plen -= len; if (plen > 0) { n = m_split(m, len, M_NOWAIT); if (n == NULL) goto fail; } M_PREPEND(m, hlen, M_NOWAIT); if (m == NULL) goto fail; bcopy(ip6, mtod(m, void *), sizeof(struct ip6_hdr)); bcopy(&ip6f, mtodo(m, sizeof(struct ip6_hdr)), sizeof(struct ip6_frag)); if (mbufq_enqueue(mq, m) != 0) goto fail; m = n; } NAT64STAT_ADD(stats, ofrags, mbufq_len(mq)); return (0); fail: if (m != NULL) m_freem(m); if (n != NULL) m_freem(n); mbufq_drain(mq); NAT64STAT_INC(stats, nomem); return (ENOMEM); } static NAT64NOINLINE int nat64_find_route6(struct nhop6_basic *pnh, struct sockaddr_in6 *dst, struct mbuf *m) { if (fib6_lookup_nh_basic(M_GETFIB(m), &dst->sin6_addr, 0, 0, 0, pnh) != 0) return (EHOSTUNREACH); if (pnh->nh_flags & (NHF_BLACKHOLE | NHF_REJECT)) return (EHOSTUNREACH); /* * XXX: we need to use destination address with embedded scope * zone id, because LLTABLE uses such form of addresses for lookup. */ dst->sin6_family = AF_INET6; dst->sin6_len = sizeof(*dst); dst->sin6_addr = pnh->nh_addr; if (IN6_IS_SCOPE_LINKLOCAL(&dst->sin6_addr)) dst->sin6_addr.s6_addr16[1] = htons(pnh->nh_ifp->if_index & 0xffff); dst->sin6_port = 0; dst->sin6_scope_id = 0; dst->sin6_flowinfo = 0; return (0); } #define NAT64_ICMP6_PLEN 64 static NAT64NOINLINE void nat64_icmp6_reflect(struct mbuf *m, uint8_t type, uint8_t code, uint32_t mtu, struct nat64_counters *stats, void *logdata) { struct icmp6_hdr *icmp6; struct ip6_hdr *ip6, *oip6; struct mbuf *n; int len, plen; len = 0; plen = nat64_getlasthdr(m, &len); if (plen < 0) { DPRINTF(DP_DROPS, "mbuf isn't contigious"); goto freeit; } /* * Do not send ICMPv6 in reply to ICMPv6 errors. */ if (plen == IPPROTO_ICMPV6) { if (m->m_len < len + sizeof(*icmp6)) { DPRINTF(DP_DROPS, "mbuf isn't contigious"); goto freeit; } icmp6 = mtodo(m, len); if (icmp6->icmp6_type < ICMP6_ECHO_REQUEST || icmp6->icmp6_type == ND_REDIRECT) { DPRINTF(DP_DROPS, "do not send ICMPv6 in reply to " "ICMPv6 errors"); goto freeit; } } /* if (icmp6_ratelimit(&ip6->ip6_src, type, code)) goto freeit; */ ip6 = mtod(m, struct ip6_hdr *); switch (type) { case ICMP6_DST_UNREACH: case ICMP6_PACKET_TOO_BIG: case ICMP6_TIME_EXCEEDED: case ICMP6_PARAM_PROB: break; default: goto freeit; } /* Calculate length of ICMPv6 payload */ len = (m->m_pkthdr.len > NAT64_ICMP6_PLEN) ? NAT64_ICMP6_PLEN: m->m_pkthdr.len; /* Create new ICMPv6 datagram */ plen = len + sizeof(struct icmp6_hdr); n = m_get2(sizeof(struct ip6_hdr) + plen + max_hdr, M_NOWAIT, MT_HEADER, M_PKTHDR); if (n == NULL) { NAT64STAT_INC(stats, nomem); m_freem(m); return; } /* * Move pkthdr from original mbuf. We should have initialized some * fields, because we can reinject this mbuf to netisr and it will * go trough input path (it requires at least rcvif should be set). * Also do M_ALIGN() to reduce chances of need to allocate new mbuf * in the chain, when we will do M_PREPEND() or make some type of * tunneling. */ m_move_pkthdr(n, m); M_ALIGN(n, sizeof(struct ip6_hdr) + plen + max_hdr); n->m_len = n->m_pkthdr.len = sizeof(struct ip6_hdr) + plen; oip6 = mtod(n, struct ip6_hdr *); oip6->ip6_src = ip6->ip6_dst; oip6->ip6_dst = ip6->ip6_src; oip6->ip6_nxt = IPPROTO_ICMPV6; oip6->ip6_flow = 0; oip6->ip6_vfc |= IPV6_VERSION; oip6->ip6_hlim = V_ip6_defhlim; oip6->ip6_plen = htons(plen); icmp6 = mtodo(n, sizeof(struct ip6_hdr)); icmp6->icmp6_cksum = 0; icmp6->icmp6_type = type; icmp6->icmp6_code = code; icmp6->icmp6_mtu = htonl(mtu); m_copydata(m, 0, len, mtodo(n, sizeof(struct ip6_hdr) + sizeof(struct icmp6_hdr))); icmp6->icmp6_cksum = in6_cksum(n, IPPROTO_ICMPV6, sizeof(struct ip6_hdr), plen); m_freem(m); V_nat64out->output_one(n, stats, logdata); return; freeit: NAT64STAT_INC(stats, dropped); m_freem(m); } static NAT64NOINLINE int nat64_find_route4(struct nhop4_basic *pnh, struct sockaddr_in *dst, struct mbuf *m) { if (fib4_lookup_nh_basic(M_GETFIB(m), dst->sin_addr, 0, 0, pnh) != 0) return (EHOSTUNREACH); if (pnh->nh_flags & (NHF_BLACKHOLE | NHF_BROADCAST | NHF_REJECT)) return (EHOSTUNREACH); dst->sin_family = AF_INET; dst->sin_len = sizeof(*dst); dst->sin_addr = pnh->nh_addr; dst->sin_port = 0; return (0); } #define NAT64_ICMP_PLEN 64 static NAT64NOINLINE void nat64_icmp_reflect(struct mbuf *m, uint8_t type, uint8_t code, uint16_t mtu, struct nat64_counters *stats, void *logdata) { struct icmp *icmp; struct ip *ip, *oip; struct mbuf *n; int len, plen; ip = mtod(m, struct ip *); /* Do not send ICMP error if packet is not the first fragment */ if (ip->ip_off & ~ntohs(IP_MF|IP_DF)) { DPRINTF(DP_DROPS, "not first fragment"); goto freeit; } /* Do not send ICMP in reply to ICMP errors */ if (ip->ip_p == IPPROTO_ICMP) { if (m->m_len < (ip->ip_hl << 2)) { DPRINTF(DP_DROPS, "mbuf isn't contigious"); goto freeit; } icmp = mtodo(m, ip->ip_hl << 2); if (!ICMP_INFOTYPE(icmp->icmp_type)) { DPRINTF(DP_DROPS, "do not send ICMP in reply to " "ICMP errors"); goto freeit; } } switch (type) { case ICMP_UNREACH: case ICMP_TIMXCEED: case ICMP_PARAMPROB: break; default: goto freeit; } /* Calculate length of ICMP payload */ len = (m->m_pkthdr.len > NAT64_ICMP_PLEN) ? (ip->ip_hl << 2) + 8: m->m_pkthdr.len; /* Create new ICMPv4 datagram */ plen = len + sizeof(struct icmphdr) + sizeof(uint32_t); n = m_get2(sizeof(struct ip) + plen + max_hdr, M_NOWAIT, MT_HEADER, M_PKTHDR); if (n == NULL) { NAT64STAT_INC(stats, nomem); m_freem(m); return; } m_move_pkthdr(n, m); M_ALIGN(n, sizeof(struct ip) + plen + max_hdr); n->m_len = n->m_pkthdr.len = sizeof(struct ip) + plen; oip = mtod(n, struct ip *); oip->ip_v = IPVERSION; oip->ip_hl = sizeof(struct ip) >> 2; oip->ip_tos = 0; oip->ip_len = htons(n->m_pkthdr.len); oip->ip_ttl = V_ip_defttl; oip->ip_p = IPPROTO_ICMP; ip_fillid(oip); oip->ip_off = htons(IP_DF); oip->ip_src = ip->ip_dst; oip->ip_dst = ip->ip_src; oip->ip_sum = 0; oip->ip_sum = in_cksum_hdr(oip); icmp = mtodo(n, sizeof(struct ip)); icmp->icmp_type = type; icmp->icmp_code = code; icmp->icmp_cksum = 0; icmp->icmp_pmvoid = 0; icmp->icmp_nextmtu = htons(mtu); m_copydata(m, 0, len, mtodo(n, sizeof(struct ip) + sizeof(struct icmphdr) + sizeof(uint32_t))); icmp->icmp_cksum = in_cksum_skip(n, sizeof(struct ip) + plen, sizeof(struct ip)); m_freem(m); V_nat64out->output_one(n, stats, logdata); return; freeit: NAT64STAT_INC(stats, dropped); m_freem(m); } /* Translate ICMP echo request/reply into ICMPv6 */ static void nat64_icmp_handle_echo(struct ip6_hdr *ip6, struct icmp6_hdr *icmp6, uint16_t id, uint8_t type) { uint16_t old; old = *(uint16_t *)icmp6; /* save type+code in one word */ icmp6->icmp6_type = type; /* Reflect ICMPv6 -> ICMPv4 type translation in the cksum */ icmp6->icmp6_cksum = cksum_adjust(icmp6->icmp6_cksum, old, *(uint16_t *)icmp6); if (id != 0) { old = icmp6->icmp6_id; icmp6->icmp6_id = id; /* Reflect ICMP id translation in the cksum */ icmp6->icmp6_cksum = cksum_adjust(icmp6->icmp6_cksum, old, id); } /* Reflect IPv6 pseudo header in the cksum */ icmp6->icmp6_cksum = ~in6_cksum_pseudo(ip6, ntohs(ip6->ip6_plen), IPPROTO_ICMPV6, ~icmp6->icmp6_cksum); } static NAT64NOINLINE struct mbuf * nat64_icmp_translate(struct mbuf *m, struct ip6_hdr *ip6, uint16_t icmpid, int offset, struct nat64_config *cfg) { struct ip ip; struct icmp *icmp; struct tcphdr *tcp; struct udphdr *udp; struct ip6_hdr *eip6; struct mbuf *n; uint32_t mtu; int len, hlen, plen; uint8_t type, code; if (m->m_len < offset + ICMP_MINLEN) m = m_pullup(m, offset + ICMP_MINLEN); if (m == NULL) { NAT64STAT_INC(&cfg->stats, nomem); return (m); } mtu = 0; icmp = mtodo(m, offset); /* RFC 7915 p4.2 */ switch (icmp->icmp_type) { case ICMP_ECHOREPLY: type = ICMP6_ECHO_REPLY; code = 0; break; case ICMP_UNREACH: type = ICMP6_DST_UNREACH; switch (icmp->icmp_code) { case ICMP_UNREACH_NET: case ICMP_UNREACH_HOST: case ICMP_UNREACH_SRCFAIL: case ICMP_UNREACH_NET_UNKNOWN: case ICMP_UNREACH_HOST_UNKNOWN: case ICMP_UNREACH_TOSNET: case ICMP_UNREACH_TOSHOST: code = ICMP6_DST_UNREACH_NOROUTE; break; case ICMP_UNREACH_PROTOCOL: type = ICMP6_PARAM_PROB; code = ICMP6_PARAMPROB_NEXTHEADER; break; case ICMP_UNREACH_PORT: code = ICMP6_DST_UNREACH_NOPORT; break; case ICMP_UNREACH_NEEDFRAG: type = ICMP6_PACKET_TOO_BIG; code = 0; /* XXX: needs an additional look */ mtu = max(IPV6_MMTU, ntohs(icmp->icmp_nextmtu) + 20); break; case ICMP_UNREACH_NET_PROHIB: case ICMP_UNREACH_HOST_PROHIB: case ICMP_UNREACH_FILTER_PROHIB: case ICMP_UNREACH_PRECEDENCE_CUTOFF: code = ICMP6_DST_UNREACH_ADMIN; break; default: DPRINTF(DP_DROPS, "Unsupported ICMP type %d, code %d", icmp->icmp_type, icmp->icmp_code); goto freeit; } break; case ICMP_TIMXCEED: type = ICMP6_TIME_EXCEEDED; code = icmp->icmp_code; break; case ICMP_ECHO: type = ICMP6_ECHO_REQUEST; code = 0; break; case ICMP_PARAMPROB: type = ICMP6_PARAM_PROB; switch (icmp->icmp_code) { case ICMP_PARAMPROB_ERRATPTR: case ICMP_PARAMPROB_LENGTH: code = ICMP6_PARAMPROB_HEADER; switch (icmp->icmp_pptr) { case 0: /* Version/IHL */ case 1: /* Type Of Service */ mtu = icmp->icmp_pptr; break; case 2: /* Total Length */ case 3: mtu = 4; /* Payload Length */ break; case 8: /* Time to Live */ mtu = 7; /* Hop Limit */ break; case 9: /* Protocol */ mtu = 6; /* Next Header */ break; case 12: /* Source address */ case 13: case 14: case 15: mtu = 8; break; case 16: /* Destination address */ case 17: case 18: case 19: mtu = 24; break; default: /* Silently drop */ DPRINTF(DP_DROPS, "Unsupported ICMP type %d," " code %d, pptr %d", icmp->icmp_type, icmp->icmp_code, icmp->icmp_pptr); goto freeit; } break; default: DPRINTF(DP_DROPS, "Unsupported ICMP type %d," " code %d, pptr %d", icmp->icmp_type, icmp->icmp_code, icmp->icmp_pptr); goto freeit; } break; default: DPRINTF(DP_DROPS, "Unsupported ICMP type %d, code %d", icmp->icmp_type, icmp->icmp_code); goto freeit; } /* * For echo request/reply we can use original payload, * but we need adjust icmp_cksum, because ICMPv6 cksum covers * IPv6 pseudo header and ICMPv6 types differs from ICMPv4. */ if (type == ICMP6_ECHO_REQUEST || type == ICMP6_ECHO_REPLY) { nat64_icmp_handle_echo(ip6, ICMP6(icmp), icmpid, type); return (m); } /* * For other types of ICMP messages we need to translate inner * IPv4 header to IPv6 header. * Assume ICMP src is the same as payload dst * E.g. we have ( GWsrc1 , NATIP1 ) in outer header * and ( NATIP1, Hostdst1 ) in ICMP copy header. * In that case, we already have map for NATIP1 and GWsrc1. * The only thing we need is to copy IPv6 map prefix to * Hostdst1. */ hlen = offset + ICMP_MINLEN; if (m->m_pkthdr.len < hlen + sizeof(struct ip) + ICMP_MINLEN) { DPRINTF(DP_DROPS, "Message is too short %d", m->m_pkthdr.len); goto freeit; } m_copydata(m, hlen, sizeof(struct ip), (char *)&ip); if (ip.ip_v != IPVERSION) { DPRINTF(DP_DROPS, "Wrong IP version %d", ip.ip_v); goto freeit; } hlen += ip.ip_hl << 2; /* Skip inner IP header */ if (nat64_check_ip4(ip.ip_src.s_addr) != 0 || nat64_check_ip4(ip.ip_dst.s_addr) != 0 || nat64_check_private_ip4(cfg, ip.ip_src.s_addr) != 0 || nat64_check_private_ip4(cfg, ip.ip_dst.s_addr) != 0) { DPRINTF(DP_DROPS, "IP addresses checks failed %04x -> %04x", ntohl(ip.ip_src.s_addr), ntohl(ip.ip_dst.s_addr)); goto freeit; } if (m->m_pkthdr.len < hlen + ICMP_MINLEN) { DPRINTF(DP_DROPS, "Message is too short %d", m->m_pkthdr.len); goto freeit; } #if 0 /* * Check that inner source matches the outer destination. * XXX: We need some method to convert IPv4 into IPv6 address here, * and compare IPv6 addresses. */ if (ip.ip_src.s_addr != nat64_get_ip4(&ip6->ip6_dst)) { DPRINTF(DP_GENERIC, "Inner source doesn't match destination ", "%04x vs %04x", ip.ip_src.s_addr, nat64_get_ip4(&ip6->ip6_dst)); goto freeit; } #endif /* * Create new mbuf for ICMPv6 datagram. * NOTE: len is data length just after inner IP header. */ len = m->m_pkthdr.len - hlen; if (sizeof(struct ip6_hdr) + sizeof(struct icmp6_hdr) + len > NAT64_ICMP6_PLEN) len = NAT64_ICMP6_PLEN - sizeof(struct icmp6_hdr) - sizeof(struct ip6_hdr); plen = sizeof(struct icmp6_hdr) + sizeof(struct ip6_hdr) + len; n = m_get2(offset + plen + max_hdr, M_NOWAIT, MT_HEADER, M_PKTHDR); if (n == NULL) { NAT64STAT_INC(&cfg->stats, nomem); m_freem(m); return (NULL); } m_move_pkthdr(n, m); M_ALIGN(n, offset + plen + max_hdr); n->m_len = n->m_pkthdr.len = offset + plen; /* Adjust ip6_plen in outer header */ ip6->ip6_plen = htons(plen); /* Construct new inner IPv6 header */ eip6 = mtodo(n, offset + sizeof(struct icmp6_hdr)); eip6->ip6_src = ip6->ip6_dst; /* Use the fact that we have single /96 prefix for IPv4 map */ eip6->ip6_dst = ip6->ip6_src; nat64_embed_ip4(cfg, ip.ip_dst.s_addr, &eip6->ip6_dst); eip6->ip6_flow = htonl(ip.ip_tos << 20); eip6->ip6_vfc |= IPV6_VERSION; eip6->ip6_hlim = ip.ip_ttl; eip6->ip6_plen = htons(ntohs(ip.ip_len) - (ip.ip_hl << 2)); eip6->ip6_nxt = (ip.ip_p == IPPROTO_ICMP) ? IPPROTO_ICMPV6: ip.ip_p; m_copydata(m, hlen, len, (char *)(eip6 + 1)); /* * We need to translate source port in the inner ULP header, * and adjust ULP checksum. */ switch (ip.ip_p) { case IPPROTO_TCP: if (len < offsetof(struct tcphdr, th_sum)) break; tcp = TCP(eip6 + 1); if (icmpid != 0) { tcp->th_sum = cksum_adjust(tcp->th_sum, tcp->th_sport, icmpid); tcp->th_sport = icmpid; } tcp->th_sum = cksum_add(tcp->th_sum, ~nat64_cksum_convert(eip6, &ip)); break; case IPPROTO_UDP: if (len < offsetof(struct udphdr, uh_sum)) break; udp = UDP(eip6 + 1); if (icmpid != 0) { udp->uh_sum = cksum_adjust(udp->uh_sum, udp->uh_sport, icmpid); udp->uh_sport = icmpid; } udp->uh_sum = cksum_add(udp->uh_sum, ~nat64_cksum_convert(eip6, &ip)); break; case IPPROTO_ICMP: /* * Check if this is an ICMP error message for echo request * that we sent. I.e. ULP in the data containing invoking * packet is IPPROTO_ICMP and its type is ICMP_ECHO. */ icmp = (struct icmp *)(eip6 + 1); if (icmp->icmp_type != ICMP_ECHO) { m_freem(n); goto freeit; } /* * For our client this original datagram should looks * like it was ICMPv6 datagram with type ICMP6_ECHO_REQUEST. * Thus we need adjust icmp_cksum and convert type from * ICMP_ECHO to ICMP6_ECHO_REQUEST. */ nat64_icmp_handle_echo(eip6, ICMP6(icmp), icmpid, ICMP6_ECHO_REQUEST); } m_freem(m); /* Convert ICMPv4 into ICMPv6 header */ icmp = mtodo(n, offset); ICMP6(icmp)->icmp6_type = type; ICMP6(icmp)->icmp6_code = code; ICMP6(icmp)->icmp6_mtu = htonl(mtu); ICMP6(icmp)->icmp6_cksum = 0; ICMP6(icmp)->icmp6_cksum = cksum_add( ~in6_cksum_pseudo(ip6, plen, IPPROTO_ICMPV6, 0), in_cksum_skip(n, n->m_pkthdr.len, offset)); return (n); freeit: m_freem(m); NAT64STAT_INC(&cfg->stats, dropped); return (NULL); } int nat64_getlasthdr(struct mbuf *m, int *offset) { struct ip6_hdr *ip6; struct ip6_hbh *hbh; int proto, hlen; if (offset != NULL) hlen = *offset; else hlen = 0; if (m->m_len < hlen + sizeof(*ip6)) return (-1); ip6 = mtodo(m, hlen); hlen += sizeof(*ip6); proto = ip6->ip6_nxt; /* Skip extension headers */ while (proto == IPPROTO_HOPOPTS || proto == IPPROTO_ROUTING || proto == IPPROTO_DSTOPTS) { hbh = mtodo(m, hlen); /* * We expect mbuf has contigious data up to * upper level header. */ if (m->m_len < hlen) return (-1); /* * We doesn't support Jumbo payload option, * so return error. */ if (proto == IPPROTO_HOPOPTS && ip6->ip6_plen == 0) return (-1); proto = hbh->ip6h_nxt; hlen += (hbh->ip6h_len + 1) << 3; } if (offset != NULL) *offset = hlen; return (proto); } int nat64_do_handle_ip4(struct mbuf *m, struct in6_addr *saddr, struct in6_addr *daddr, uint16_t lport, struct nat64_config *cfg, void *logdata) { struct nhop6_basic nh; struct ip6_hdr ip6; struct sockaddr_in6 dst; struct ip *ip; struct mbufq mq; uint16_t ip_id, ip_off; uint16_t *csum; int plen, hlen; uint8_t proto; ip = mtod(m, struct ip*); if (ip->ip_ttl <= IPTTLDEC) { nat64_icmp_reflect(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, 0, &cfg->stats, logdata); return (NAT64RETURN); } ip6.ip6_dst = *daddr; ip6.ip6_src = *saddr; hlen = ip->ip_hl << 2; plen = ntohs(ip->ip_len) - hlen; proto = ip->ip_p; /* Save ip_id and ip_off, both are in network byte order */ ip_id = ip->ip_id; ip_off = ip->ip_off & htons(IP_OFFMASK | IP_MF); /* Fragment length must be multiple of 8 octets */ if ((ip->ip_off & htons(IP_MF)) != 0 && (plen & 0x7) != 0) { nat64_icmp_reflect(m, ICMP_PARAMPROB, ICMP_PARAMPROB_LENGTH, 0, &cfg->stats, logdata); return (NAT64RETURN); } /* Fragmented ICMP is unsupported */ if (proto == IPPROTO_ICMP && ip_off != 0) { DPRINTF(DP_DROPS, "dropped due to fragmented ICMP"); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } dst.sin6_addr = ip6.ip6_dst; if (nat64_find_route6(&nh, &dst, m) != 0) { NAT64STAT_INC(&cfg->stats, noroute6); nat64_icmp_reflect(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, &cfg->stats, logdata); return (NAT64RETURN); } if (nh.nh_mtu < plen + sizeof(ip6) && (ip->ip_off & htons(IP_DF)) != 0) { nat64_icmp_reflect(m, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, FRAGSZ(nh.nh_mtu) + sizeof(struct ip), &cfg->stats, logdata); return (NAT64RETURN); } ip6.ip6_flow = htonl(ip->ip_tos << 20); ip6.ip6_vfc |= IPV6_VERSION; ip6.ip6_hlim = ip->ip_ttl; /* Forwarding code will decrement TTL for netisr based output. */ if (V_nat64out == &nat64_direct) ip6.ip6_hlim -= IPTTLDEC; ip6.ip6_plen = htons(plen); ip6.ip6_nxt = (proto == IPPROTO_ICMP) ? IPPROTO_ICMPV6: proto; /* Convert checksums. */ switch (proto) { case IPPROTO_TCP: csum = &TCP(mtodo(m, hlen))->th_sum; if (lport != 0) { struct tcphdr *tcp = TCP(mtodo(m, hlen)); *csum = cksum_adjust(*csum, tcp->th_dport, lport); tcp->th_dport = lport; } *csum = cksum_add(*csum, ~nat64_cksum_convert(&ip6, ip)); break; case IPPROTO_UDP: csum = &UDP(mtodo(m, hlen))->uh_sum; if (lport != 0) { struct udphdr *udp = UDP(mtodo(m, hlen)); *csum = cksum_adjust(*csum, udp->uh_dport, lport); udp->uh_dport = lport; } *csum = cksum_add(*csum, ~nat64_cksum_convert(&ip6, ip)); break; case IPPROTO_ICMP: m = nat64_icmp_translate(m, &ip6, lport, hlen, cfg); if (m == NULL) /* stats already accounted */ return (NAT64RETURN); } m_adj(m, hlen); mbufq_init(&mq, 255); nat64_fragment6(&cfg->stats, &ip6, &mq, m, nh.nh_mtu, ip_id, ip_off); while ((m = mbufq_dequeue(&mq)) != NULL) { if (V_nat64out->output(nh.nh_ifp, m, (struct sockaddr *)&dst, &cfg->stats, logdata) != 0) break; NAT64STAT_INC(&cfg->stats, opcnt46); } mbufq_drain(&mq); return (NAT64RETURN); } int nat64_handle_icmp6(struct mbuf *m, int hlen, uint32_t aaddr, uint16_t aport, struct nat64_config *cfg, void *logdata) { struct ip ip; struct icmp6_hdr *icmp6; struct ip6_frag *ip6f; struct ip6_hdr *ip6, *ip6i; uint32_t mtu; int plen, proto; uint8_t type, code; if (hlen == 0) { ip6 = mtod(m, struct ip6_hdr *); if (nat64_check_ip6(&ip6->ip6_src) != 0 || nat64_check_ip6(&ip6->ip6_dst) != 0) return (NAT64SKIP); proto = nat64_getlasthdr(m, &hlen); if (proto != IPPROTO_ICMPV6) { DPRINTF(DP_DROPS, "dropped due to mbuf isn't contigious"); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } } /* * Translate ICMPv6 type and code to ICMPv4 (RFC7915). * NOTE: ICMPv6 echo handled by nat64_do_handle_ip6(). */ icmp6 = mtodo(m, hlen); mtu = 0; switch (icmp6->icmp6_type) { case ICMP6_DST_UNREACH: type = ICMP_UNREACH; switch (icmp6->icmp6_code) { case ICMP6_DST_UNREACH_NOROUTE: case ICMP6_DST_UNREACH_BEYONDSCOPE: case ICMP6_DST_UNREACH_ADDR: code = ICMP_UNREACH_HOST; break; case ICMP6_DST_UNREACH_ADMIN: code = ICMP_UNREACH_HOST_PROHIB; break; case ICMP6_DST_UNREACH_NOPORT: code = ICMP_UNREACH_PORT; break; default: DPRINTF(DP_DROPS, "Unsupported ICMPv6 type %d," " code %d", icmp6->icmp6_type, icmp6->icmp6_code); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } break; case ICMP6_PACKET_TOO_BIG: type = ICMP_UNREACH; code = ICMP_UNREACH_NEEDFRAG; mtu = ntohl(icmp6->icmp6_mtu); if (mtu < IPV6_MMTU) { DPRINTF(DP_DROPS, "Wrong MTU %d in ICMPv6 type %d," " code %d", mtu, icmp6->icmp6_type, icmp6->icmp6_code); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } /* * Adjust MTU to reflect difference between * IPv6 an IPv4 headers. */ mtu -= sizeof(struct ip6_hdr) - sizeof(struct ip); break; case ICMP6_TIME_EXCEEDED: type = ICMP_TIMXCEED; code = icmp6->icmp6_code; break; case ICMP6_PARAM_PROB: switch (icmp6->icmp6_code) { case ICMP6_PARAMPROB_HEADER: type = ICMP_PARAMPROB; code = ICMP_PARAMPROB_ERRATPTR; mtu = ntohl(icmp6->icmp6_pptr); switch (mtu) { case 0: /* Version/Traffic Class */ case 1: /* Traffic Class/Flow Label */ break; case 4: /* Payload Length */ case 5: mtu = 2; break; case 6: /* Next Header */ mtu = 9; break; case 7: /* Hop Limit */ mtu = 8; break; default: if (mtu >= 8 && mtu <= 23) { mtu = 12; /* Source address */ break; } if (mtu >= 24 && mtu <= 39) { mtu = 16; /* Destination address */ break; } DPRINTF(DP_DROPS, "Unsupported ICMPv6 type %d," " code %d, pptr %d", icmp6->icmp6_type, icmp6->icmp6_code, mtu); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } case ICMP6_PARAMPROB_NEXTHEADER: type = ICMP_UNREACH; code = ICMP_UNREACH_PROTOCOL; break; default: DPRINTF(DP_DROPS, "Unsupported ICMPv6 type %d," " code %d, pptr %d", icmp6->icmp6_type, icmp6->icmp6_code, ntohl(icmp6->icmp6_pptr)); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } break; default: DPRINTF(DP_DROPS, "Unsupported ICMPv6 type %d, code %d", icmp6->icmp6_type, icmp6->icmp6_code); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } hlen += sizeof(struct icmp6_hdr); if (m->m_pkthdr.len < hlen + sizeof(struct ip6_hdr) + ICMP_MINLEN) { NAT64STAT_INC(&cfg->stats, dropped); DPRINTF(DP_DROPS, "Message is too short %d", m->m_pkthdr.len); return (NAT64MFREE); } /* * We need at least ICMP_MINLEN bytes of original datagram payload * to generate ICMP message. It is nice that ICMP_MINLEN is equal * to sizeof(struct ip6_frag). So, if embedded datagram had a fragment * header we will not have to do m_pullup() again. * * What we have here: * Outer header: (IPv6iGW, v4mapPRefix+v4exthost) * Inner header: (v4mapPRefix+v4host, IPv6iHost) [sport, dport] * We need to translate it to: * * Outer header: (alias_host, v4exthost) * Inner header: (v4exthost, alias_host) [sport, alias_port] * * Assume caller function has checked if v4mapPRefix+v4host * matches configured prefix. * The only two things we should be provided with are mapping between * IPv6iHost <> alias_host and between dport and alias_port. */ if (m->m_len < hlen + sizeof(struct ip6_hdr) + ICMP_MINLEN) m = m_pullup(m, hlen + sizeof(struct ip6_hdr) + ICMP_MINLEN); if (m == NULL) { NAT64STAT_INC(&cfg->stats, nomem); return (NAT64RETURN); } ip6 = mtod(m, struct ip6_hdr *); ip6i = mtodo(m, hlen); ip6f = NULL; proto = ip6i->ip6_nxt; plen = ntohs(ip6i->ip6_plen); hlen += sizeof(struct ip6_hdr); if (proto == IPPROTO_FRAGMENT) { if (m->m_pkthdr.len < hlen + sizeof(struct ip6_frag) + ICMP_MINLEN) goto fail; ip6f = mtodo(m, hlen); proto = ip6f->ip6f_nxt; plen -= sizeof(struct ip6_frag); hlen += sizeof(struct ip6_frag); /* Ajust MTU to reflect frag header size */ if (type == ICMP_UNREACH && code == ICMP_UNREACH_NEEDFRAG) mtu -= sizeof(struct ip6_frag); } if (proto != IPPROTO_TCP && proto != IPPROTO_UDP) { DPRINTF(DP_DROPS, "Unsupported proto %d in the inner header", proto); goto fail; } if (nat64_check_ip6(&ip6i->ip6_src) != 0 || nat64_check_ip6(&ip6i->ip6_dst) != 0) { DPRINTF(DP_DROPS, "Inner addresses do not passes the check"); goto fail; } /* Check if outer dst is the same as inner src */ if (!IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &ip6i->ip6_src)) { DPRINTF(DP_DROPS, "Inner src doesn't match outer dst"); goto fail; } /* Now we need to make a fake IPv4 packet to generate ICMP message */ ip.ip_dst.s_addr = aaddr; ip.ip_src.s_addr = nat64_extract_ip4(cfg, &ip6i->ip6_src); /* XXX: Make fake ulp header */ if (V_nat64out == &nat64_direct) /* init_ip4hdr will decrement it */ ip6i->ip6_hlim += IPV6_HLIMDEC; nat64_init_ip4hdr(ip6i, ip6f, plen, proto, &ip); m_adj(m, hlen - sizeof(struct ip)); bcopy(&ip, mtod(m, void *), sizeof(ip)); nat64_icmp_reflect(m, type, code, (uint16_t)mtu, &cfg->stats, logdata); return (NAT64RETURN); fail: /* * We must call m_freem() because mbuf pointer could be * changed with m_pullup(). */ m_freem(m); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64RETURN); } int nat64_do_handle_ip6(struct mbuf *m, uint32_t aaddr, uint16_t aport, struct nat64_config *cfg, void *logdata) { struct ip ip; struct nhop4_basic nh; struct sockaddr_in dst; struct ip6_frag *frag; struct ip6_hdr *ip6; struct icmp6_hdr *icmp6; uint16_t *csum; int plen, hlen, proto; /* * XXX: we expect ipfw_chk() did m_pullup() up to upper level * protocol's headers. Also we skip some checks, that ip6_input(), * ip6_forward(), ip6_fastfwd() and ipfw_chk() already did. */ ip6 = mtod(m, struct ip6_hdr *); if (nat64_check_ip6(&ip6->ip6_src) != 0 || nat64_check_ip6(&ip6->ip6_dst) != 0) { return (NAT64SKIP); } /* Starting from this point we must not return zero */ ip.ip_src.s_addr = aaddr; if (nat64_check_ip4(ip.ip_src.s_addr) != 0) { DPRINTF(DP_GENERIC | DP_DROPS, "invalid source address: %08x", ip.ip_src.s_addr); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } ip.ip_dst.s_addr = nat64_extract_ip4(cfg, &ip6->ip6_dst); if (ip.ip_dst.s_addr == 0) { NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } if (ip6->ip6_hlim <= IPV6_HLIMDEC) { nat64_icmp6_reflect(m, ICMP6_TIME_EXCEEDED, ICMP6_TIME_EXCEED_TRANSIT, 0, &cfg->stats, logdata); return (NAT64RETURN); } hlen = 0; plen = ntohs(ip6->ip6_plen); proto = nat64_getlasthdr(m, &hlen); if (proto < 0) { DPRINTF(DP_DROPS, "dropped due to mbuf isn't contigious"); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } frag = NULL; if (proto == IPPROTO_FRAGMENT) { /* ipfw_chk should m_pullup up to frag header */ if (m->m_len < hlen + sizeof(*frag)) { DPRINTF(DP_DROPS, "dropped due to mbuf isn't contigious"); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } frag = mtodo(m, hlen); proto = frag->ip6f_nxt; hlen += sizeof(*frag); /* Fragmented ICMPv6 is unsupported */ if (proto == IPPROTO_ICMPV6) { DPRINTF(DP_DROPS, "dropped due to fragmented ICMPv6"); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } /* Fragment length must be multiple of 8 octets */ if ((frag->ip6f_offlg & IP6F_MORE_FRAG) != 0 && ((plen + sizeof(struct ip6_hdr) - hlen) & 0x7) != 0) { nat64_icmp6_reflect(m, ICMP6_PARAM_PROB, ICMP6_PARAMPROB_HEADER, offsetof(struct ip6_hdr, ip6_plen), &cfg->stats, logdata); return (NAT64RETURN); } } plen -= hlen - sizeof(struct ip6_hdr); if (plen < 0 || m->m_pkthdr.len < plen + hlen) { DPRINTF(DP_DROPS, "plen %d, pkthdr.len %d, hlen %d", plen, m->m_pkthdr.len, hlen); NAT64STAT_INC(&cfg->stats, dropped); return (NAT64MFREE); } icmp6 = NULL; /* Make gcc happy */ if (proto == IPPROTO_ICMPV6) { icmp6 = mtodo(m, hlen); if (icmp6->icmp6_type != ICMP6_ECHO_REQUEST && icmp6->icmp6_type != ICMP6_ECHO_REPLY) return (nat64_handle_icmp6(m, hlen, aaddr, aport, cfg, logdata)); } dst.sin_addr.s_addr = ip.ip_dst.s_addr; if (nat64_find_route4(&nh, &dst, m) != 0) { NAT64STAT_INC(&cfg->stats, noroute4); nat64_icmp6_reflect(m, ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOROUTE, 0, &cfg->stats, logdata); return (NAT64RETURN); } if (nh.nh_mtu < plen + sizeof(ip)) { nat64_icmp6_reflect(m, ICMP6_PACKET_TOO_BIG, 0, nh.nh_mtu, &cfg->stats, logdata); return (NAT64RETURN); } nat64_init_ip4hdr(ip6, frag, plen, proto, &ip); /* Convert checksums. */ switch (proto) { case IPPROTO_TCP: csum = &TCP(mtodo(m, hlen))->th_sum; if (aport != 0) { struct tcphdr *tcp = TCP(mtodo(m, hlen)); *csum = cksum_adjust(*csum, tcp->th_sport, aport); tcp->th_sport = aport; } *csum = cksum_add(*csum, nat64_cksum_convert(ip6, &ip)); break; case IPPROTO_UDP: csum = &UDP(mtodo(m, hlen))->uh_sum; if (aport != 0) { struct udphdr *udp = UDP(mtodo(m, hlen)); *csum = cksum_adjust(*csum, udp->uh_sport, aport); udp->uh_sport = aport; } *csum = cksum_add(*csum, nat64_cksum_convert(ip6, &ip)); break; case IPPROTO_ICMPV6: /* Checksum in ICMPv6 covers pseudo header */ csum = &icmp6->icmp6_cksum; *csum = cksum_add(*csum, in6_cksum_pseudo(ip6, plen, IPPROTO_ICMPV6, 0)); /* Convert ICMPv6 types to ICMP */ proto = *(uint16_t *)icmp6; /* save old word for cksum_adjust */ if (icmp6->icmp6_type == ICMP6_ECHO_REQUEST) icmp6->icmp6_type = ICMP_ECHO; else /* ICMP6_ECHO_REPLY */ icmp6->icmp6_type = ICMP_ECHOREPLY; *csum = cksum_adjust(*csum, (uint16_t)proto, *(uint16_t *)icmp6); if (aport != 0) { uint16_t old_id = icmp6->icmp6_id; icmp6->icmp6_id = aport; *csum = cksum_adjust(*csum, old_id, aport); } break; }; m_adj(m, hlen - sizeof(ip)); bcopy(&ip, mtod(m, void *), sizeof(ip)); if (V_nat64out->output(nh.nh_ifp, m, (struct sockaddr *)&dst, &cfg->stats, logdata) == 0) NAT64STAT_INC(&cfg->stats, opcnt64); return (NAT64RETURN); } Index: projects/import-googletest-1.8.1/sys/ufs/ufs/ufs_vnops.c =================================================================== --- projects/import-googletest-1.8.1/sys/ufs/ufs/ufs_vnops.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/ufs/ufs/ufs_vnops.c (revision 345026) @@ -1,2795 +1,2785 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1986, 1989, 1993, 1995 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)ufs_vnops.c 8.27 (Berkeley) 5/27/95 */ #include __FBSDID("$FreeBSD$"); #include "opt_quota.h" #include "opt_suiddir.h" #include "opt_ufs.h" #include "opt_ffs.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* XXX */ #include #include #include #include #include #include #include #include #include #ifdef UFS_DIRHASH #include #endif #ifdef UFS_GJOURNAL #include FEATURE(ufs_gjournal, "Journaling support through GEOM for UFS"); #endif #ifdef QUOTA FEATURE(ufs_quota, "UFS disk quotas support"); FEATURE(ufs_quota64, "64bit UFS disk quotas support"); #endif #ifdef SUIDDIR FEATURE(suiddir, "Give all new files in directory the same ownership as the directory"); #endif #include static vop_accessx_t ufs_accessx; static int ufs_chmod(struct vnode *, int, struct ucred *, struct thread *); static int ufs_chown(struct vnode *, uid_t, gid_t, struct ucred *, struct thread *); static vop_close_t ufs_close; static vop_create_t ufs_create; static vop_getattr_t ufs_getattr; static vop_ioctl_t ufs_ioctl; static vop_link_t ufs_link; static int ufs_makeinode(int mode, struct vnode *, struct vnode **, struct componentname *, const char *); static vop_markatime_t ufs_markatime; static vop_mkdir_t ufs_mkdir; static vop_mknod_t ufs_mknod; static vop_open_t ufs_open; static vop_pathconf_t ufs_pathconf; static vop_print_t ufs_print; static vop_readlink_t ufs_readlink; static vop_remove_t ufs_remove; static vop_rename_t ufs_rename; static vop_rmdir_t ufs_rmdir; static vop_setattr_t ufs_setattr; static vop_strategy_t ufs_strategy; static vop_symlink_t ufs_symlink; static vop_whiteout_t ufs_whiteout; static vop_close_t ufsfifo_close; static vop_kqfilter_t ufsfifo_kqfilter; SYSCTL_NODE(_vfs, OID_AUTO, ufs, CTLFLAG_RD, 0, "UFS filesystem"); /* * A virgin directory (no blushing please). */ static struct dirtemplate mastertemplate = { 0, 12, DT_DIR, 1, ".", 0, DIRBLKSIZ - 12, DT_DIR, 2, ".." }; static struct odirtemplate omastertemplate = { 0, 12, 1, ".", 0, DIRBLKSIZ - 12, 2, ".." }; static void ufs_itimes_locked(struct vnode *vp) { struct inode *ip; struct timespec ts; ASSERT_VI_LOCKED(vp, __func__); ip = VTOI(vp); if (UFS_RDONLY(ip)) goto out; if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_UPDATE)) == 0) return; if ((vp->v_type == VBLK || vp->v_type == VCHR) && !DOINGSOFTDEP(vp)) ip->i_flag |= IN_LAZYMOD; else if (((vp->v_mount->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND)) == 0) || (ip->i_flag & (IN_CHANGE | IN_UPDATE))) ip->i_flag |= IN_MODIFIED; else if (ip->i_flag & IN_ACCESS) ip->i_flag |= IN_LAZYACCESS; vfs_timestamp(&ts); if (ip->i_flag & IN_ACCESS) { DIP_SET(ip, i_atime, ts.tv_sec); DIP_SET(ip, i_atimensec, ts.tv_nsec); } if (ip->i_flag & IN_UPDATE) { DIP_SET(ip, i_mtime, ts.tv_sec); DIP_SET(ip, i_mtimensec, ts.tv_nsec); } if (ip->i_flag & IN_CHANGE) { DIP_SET(ip, i_ctime, ts.tv_sec); DIP_SET(ip, i_ctimensec, ts.tv_nsec); DIP_SET(ip, i_modrev, DIP(ip, i_modrev) + 1); } out: ip->i_flag &= ~(IN_ACCESS | IN_CHANGE | IN_UPDATE); } void ufs_itimes(struct vnode *vp) { VI_LOCK(vp); ufs_itimes_locked(vp); VI_UNLOCK(vp); } /* * Create a regular file */ static int ufs_create(ap) struct vop_create_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; } */ *ap; { int error; error = ufs_makeinode(MAKEIMODE(ap->a_vap->va_type, ap->a_vap->va_mode), ap->a_dvp, ap->a_vpp, ap->a_cnp, "ufs_create"); if (error != 0) return (error); if ((ap->a_cnp->cn_flags & MAKEENTRY) != 0) cache_enter(ap->a_dvp, *ap->a_vpp, ap->a_cnp); return (0); } /* * Mknod vnode call */ /* ARGSUSED */ static int ufs_mknod(ap) struct vop_mknod_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; } */ *ap; { struct vattr *vap = ap->a_vap; struct vnode **vpp = ap->a_vpp; struct inode *ip; ino_t ino; int error; error = ufs_makeinode(MAKEIMODE(vap->va_type, vap->va_mode), ap->a_dvp, vpp, ap->a_cnp, "ufs_mknod"); if (error) return (error); ip = VTOI(*vpp); ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE; if (vap->va_rdev != VNOVAL) { /* * Want to be able to use this to make badblock * inodes, so don't truncate the dev number. */ DIP_SET(ip, i_rdev, vap->va_rdev); } /* * Remove inode, then reload it through VFS_VGET so it is * checked to see if it is an alias of an existing entry in * the inode cache. XXX I don't believe this is necessary now. */ (*vpp)->v_type = VNON; ino = ip->i_number; /* Save this before vgone() invalidates ip. */ vgone(*vpp); vput(*vpp); error = VFS_VGET(ap->a_dvp->v_mount, ino, LK_EXCLUSIVE, vpp); if (error) { *vpp = NULL; return (error); } return (0); } /* * Open called. */ /* ARGSUSED */ static int ufs_open(struct vop_open_args *ap) { struct vnode *vp = ap->a_vp; struct inode *ip; if (vp->v_type == VCHR || vp->v_type == VBLK) return (EOPNOTSUPP); ip = VTOI(vp); /* * Files marked append-only must be opened for appending. */ if ((ip->i_flags & APPEND) && (ap->a_mode & (FWRITE | O_APPEND)) == FWRITE) return (EPERM); vnode_create_vobject(vp, DIP(ip, i_size), ap->a_td); return (0); } /* * Close called. * * Update the times on the inode. */ /* ARGSUSED */ static int ufs_close(ap) struct vop_close_args /* { struct vnode *a_vp; int a_fflag; struct ucred *a_cred; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; int usecount; VI_LOCK(vp); usecount = vp->v_usecount; if (usecount > 1) ufs_itimes_locked(vp); VI_UNLOCK(vp); return (0); } static int ufs_accessx(ap) struct vop_accessx_args /* { struct vnode *a_vp; accmode_t a_accmode; struct ucred *a_cred; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); accmode_t accmode = ap->a_accmode; int error; #ifdef UFS_ACL struct acl *acl; acl_type_t type; #endif /* * Disallow write attempts on read-only filesystems; * unless the file is a socket, fifo, or a block or * character device resident on the filesystem. */ if (accmode & VMODIFY_PERMS) { switch (vp->v_type) { case VDIR: case VLNK: case VREG: if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); #ifdef QUOTA /* * Inode is accounted in the quotas only if struct * dquot is attached to it. VOP_ACCESS() is called * from vn_open_cred() and provides a convenient * point to call getinoquota(). The lock mode is * exclusive when the file is opening for write. */ if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) { error = getinoquota(ip); if (error != 0) return (error); } #endif break; default: break; } } /* * If immutable bit set, nobody gets to write it. "& ~VADMIN_PERMS" * permits the owner of the file to remove the IMMUTABLE flag. */ if ((accmode & (VMODIFY_PERMS & ~VADMIN_PERMS)) && (ip->i_flags & (IMMUTABLE | SF_SNAPSHOT))) return (EPERM); #ifdef UFS_ACL if ((vp->v_mount->mnt_flag & (MNT_ACLS | MNT_NFS4ACLS)) != 0) { if (vp->v_mount->mnt_flag & MNT_NFS4ACLS) type = ACL_TYPE_NFS4; else type = ACL_TYPE_ACCESS; acl = acl_alloc(M_WAITOK); if (type == ACL_TYPE_NFS4) error = ufs_getacl_nfs4_internal(vp, acl, ap->a_td); else error = VOP_GETACL(vp, type, acl, ap->a_cred, ap->a_td); switch (error) { case 0: if (type == ACL_TYPE_NFS4) { error = vaccess_acl_nfs4(vp->v_type, ip->i_uid, ip->i_gid, acl, accmode, ap->a_cred, NULL); } else { error = vfs_unixify_accmode(&accmode); if (error == 0) error = vaccess_acl_posix1e(vp->v_type, ip->i_uid, ip->i_gid, acl, accmode, ap->a_cred, NULL); } break; default: if (error != EOPNOTSUPP) printf( "ufs_accessx(): Error retrieving ACL on object (%d).\n", error); /* * XXX: Fall back until debugged. Should * eventually possibly log an error, and return * EPERM for safety. */ error = vfs_unixify_accmode(&accmode); if (error == 0) error = vaccess(vp->v_type, ip->i_mode, ip->i_uid, ip->i_gid, accmode, ap->a_cred, NULL); } acl_free(acl); return (error); } #endif /* !UFS_ACL */ error = vfs_unixify_accmode(&accmode); if (error == 0) error = vaccess(vp->v_type, ip->i_mode, ip->i_uid, ip->i_gid, accmode, ap->a_cred, NULL); return (error); } /* ARGSUSED */ static int ufs_getattr(ap) struct vop_getattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); struct vattr *vap = ap->a_vap; VI_LOCK(vp); ufs_itimes_locked(vp); if (I_IS_UFS1(ip)) { vap->va_atime.tv_sec = ip->i_din1->di_atime; vap->va_atime.tv_nsec = ip->i_din1->di_atimensec; } else { vap->va_atime.tv_sec = ip->i_din2->di_atime; vap->va_atime.tv_nsec = ip->i_din2->di_atimensec; } VI_UNLOCK(vp); /* * Copy from inode table */ vap->va_fsid = dev2udev(ITOUMP(ip)->um_dev); vap->va_fileid = ip->i_number; vap->va_mode = ip->i_mode & ~IFMT; vap->va_nlink = ip->i_effnlink; vap->va_uid = ip->i_uid; vap->va_gid = ip->i_gid; if (I_IS_UFS1(ip)) { vap->va_rdev = ip->i_din1->di_rdev; vap->va_size = ip->i_din1->di_size; vap->va_mtime.tv_sec = ip->i_din1->di_mtime; vap->va_mtime.tv_nsec = ip->i_din1->di_mtimensec; vap->va_ctime.tv_sec = ip->i_din1->di_ctime; vap->va_ctime.tv_nsec = ip->i_din1->di_ctimensec; vap->va_bytes = dbtob((u_quad_t)ip->i_din1->di_blocks); vap->va_filerev = ip->i_din1->di_modrev; } else { vap->va_rdev = ip->i_din2->di_rdev; vap->va_size = ip->i_din2->di_size; vap->va_mtime.tv_sec = ip->i_din2->di_mtime; vap->va_mtime.tv_nsec = ip->i_din2->di_mtimensec; vap->va_ctime.tv_sec = ip->i_din2->di_ctime; vap->va_ctime.tv_nsec = ip->i_din2->di_ctimensec; vap->va_birthtime.tv_sec = ip->i_din2->di_birthtime; vap->va_birthtime.tv_nsec = ip->i_din2->di_birthnsec; vap->va_bytes = dbtob((u_quad_t)ip->i_din2->di_blocks); vap->va_filerev = ip->i_din2->di_modrev; } vap->va_flags = ip->i_flags; vap->va_gen = ip->i_gen; vap->va_blocksize = vp->v_mount->mnt_stat.f_iosize; vap->va_type = IFTOVT(ip->i_mode); return (0); } /* * Set attribute vnode op. called from several syscalls */ static int ufs_setattr(ap) struct vop_setattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { struct vattr *vap = ap->a_vap; struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); struct ucred *cred = ap->a_cred; struct thread *td = curthread; int error; /* * Check for unsettable attributes. */ if ((vap->va_type != VNON) || (vap->va_nlink != VNOVAL) || (vap->va_fsid != VNOVAL) || (vap->va_fileid != VNOVAL) || (vap->va_blocksize != VNOVAL) || (vap->va_rdev != VNOVAL) || ((int)vap->va_bytes != VNOVAL) || (vap->va_gen != VNOVAL)) { return (EINVAL); } if (vap->va_flags != VNOVAL) { if ((vap->va_flags & ~(SF_APPEND | SF_ARCHIVED | SF_IMMUTABLE | SF_NOUNLINK | SF_SNAPSHOT | UF_APPEND | UF_ARCHIVE | UF_HIDDEN | UF_IMMUTABLE | UF_NODUMP | UF_NOUNLINK | UF_OFFLINE | UF_OPAQUE | UF_READONLY | UF_REPARSE | UF_SPARSE | UF_SYSTEM)) != 0) return (EOPNOTSUPP); if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); /* * Callers may only modify the file flags on objects they * have VADMIN rights for. */ if ((error = VOP_ACCESS(vp, VADMIN, cred, td))) return (error); /* * Unprivileged processes are not permitted to unset system * flags, or modify flags if any system flags are set. * Privileged non-jail processes may not modify system flags * if securelevel > 0 and any existing system flags are set. * Privileged jail processes behave like privileged non-jail * processes if the PR_ALLOW_CHFLAGS permission bit is set; * otherwise, they behave like unprivileged processes. */ if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS)) { if (ip->i_flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) { error = securelevel_gt(cred, 0); if (error) return (error); } /* The snapshot flag cannot be toggled. */ if ((vap->va_flags ^ ip->i_flags) & SF_SNAPSHOT) return (EPERM); } else { if (ip->i_flags & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) || ((vap->va_flags ^ ip->i_flags) & SF_SETTABLE)) return (EPERM); } ip->i_flags = vap->va_flags; DIP_SET(ip, i_flags, vap->va_flags); ip->i_flag |= IN_CHANGE; error = UFS_UPDATE(vp, 0); if (ip->i_flags & (IMMUTABLE | APPEND)) return (error); } /* * If immutable or append, no one can change any of its attributes * except the ones already handled (in some cases, file flags * including the immutability flags themselves for the superuser). */ if (ip->i_flags & (IMMUTABLE | APPEND)) return (EPERM); /* * Go through the fields and update iff not VNOVAL. */ if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if ((error = ufs_chown(vp, vap->va_uid, vap->va_gid, cred, td)) != 0) return (error); } if (vap->va_size != VNOVAL) { /* * XXX most of the following special cases should be in * callers instead of in N filesystems. The VDIR check * mostly already is. */ switch (vp->v_type) { case VDIR: return (EISDIR); case VLNK: case VREG: /* * Truncation should have an effect in these cases. * Disallow it if the filesystem is read-only or * the file is being snapshotted. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if ((ip->i_flags & SF_SNAPSHOT) != 0) return (EPERM); break; default: /* * According to POSIX, the result is unspecified * for file types other than regular files, * directories and shared memory objects. We * don't support shared memory objects in the file * system, and have dubious support for truncating * symlinks. Just ignore the request in other cases. */ return (0); } if ((error = UFS_TRUNCATE(vp, vap->va_size, IO_NORMAL | ((vap->va_vaflags & VA_SYNC) != 0 ? IO_SYNC : 0), cred)) != 0) return (error); } if (vap->va_atime.tv_sec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || vap->va_birthtime.tv_sec != VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if ((ip->i_flags & SF_SNAPSHOT) != 0) return (EPERM); error = vn_utimes_perm(vp, vap, cred, td); if (error != 0) return (error); ip->i_flag |= IN_CHANGE | IN_MODIFIED; if (vap->va_atime.tv_sec != VNOVAL) { ip->i_flag &= ~IN_ACCESS; DIP_SET(ip, i_atime, vap->va_atime.tv_sec); DIP_SET(ip, i_atimensec, vap->va_atime.tv_nsec); } if (vap->va_mtime.tv_sec != VNOVAL) { ip->i_flag &= ~IN_UPDATE; DIP_SET(ip, i_mtime, vap->va_mtime.tv_sec); DIP_SET(ip, i_mtimensec, vap->va_mtime.tv_nsec); } if (vap->va_birthtime.tv_sec != VNOVAL && I_IS_UFS2(ip)) { ip->i_din2->di_birthtime = vap->va_birthtime.tv_sec; ip->i_din2->di_birthnsec = vap->va_birthtime.tv_nsec; } error = UFS_UPDATE(vp, 0); if (error) return (error); } error = 0; if (vap->va_mode != (mode_t)VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if ((ip->i_flags & SF_SNAPSHOT) != 0 && (vap->va_mode & (S_IXUSR | S_IWUSR | S_IXGRP | S_IWGRP | S_IXOTH | S_IWOTH))) return (EPERM); error = ufs_chmod(vp, (int)vap->va_mode, cred, td); } return (error); } #ifdef UFS_ACL static int ufs_update_nfs4_acl_after_mode_change(struct vnode *vp, int mode, int file_owner_id, struct ucred *cred, struct thread *td) { int error; struct acl *aclp; aclp = acl_alloc(M_WAITOK); error = ufs_getacl_nfs4_internal(vp, aclp, td); /* * We don't have to handle EOPNOTSUPP here, as the filesystem claims * it supports ACLs. */ if (error) goto out; acl_nfs4_sync_acl_from_mode(aclp, mode, file_owner_id); error = ufs_setacl_nfs4_internal(vp, aclp, td); out: acl_free(aclp); return (error); } #endif /* UFS_ACL */ /* * Mark this file's access time for update for vfs_mark_atime(). This * is called from execve() and mmap(). */ static int ufs_markatime(ap) struct vop_markatime_args /* { struct vnode *a_vp; } */ *ap; { struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); VI_LOCK(vp); ip->i_flag |= IN_ACCESS; VI_UNLOCK(vp); /* * XXXKIB No UFS_UPDATE(ap->a_vp, 0) there. */ return (0); } /* * Change the mode on a file. * Inode must be locked before calling. */ static int ufs_chmod(vp, mode, cred, td) struct vnode *vp; int mode; struct ucred *cred; struct thread *td; { struct inode *ip = VTOI(vp); int error; /* * To modify the permissions on a file, must possess VADMIN * for that file. */ if ((error = VOP_ACCESSX(vp, VWRITE_ACL, cred, td))) return (error); /* * Privileged processes may set the sticky bit on non-directories, * as well as set the setgid bit on a file with a group that the * process is not a member of. Both of these are allowed in * jail(8). */ if (vp->v_type != VDIR && (mode & S_ISTXT)) { if (priv_check_cred(cred, PRIV_VFS_STICKYFILE)) return (EFTYPE); } if (!groupmember(ip->i_gid, cred) && (mode & ISGID)) { error = priv_check_cred(cred, PRIV_VFS_SETGID); if (error) return (error); } /* * Deny setting setuid if we are not the file owner. */ if ((mode & ISUID) && ip->i_uid != cred->cr_uid) { error = priv_check_cred(cred, PRIV_VFS_ADMIN); if (error) return (error); } ip->i_mode &= ~ALLPERMS; ip->i_mode |= (mode & ALLPERMS); DIP_SET(ip, i_mode, ip->i_mode); ip->i_flag |= IN_CHANGE; #ifdef UFS_ACL if ((vp->v_mount->mnt_flag & MNT_NFS4ACLS) != 0) error = ufs_update_nfs4_acl_after_mode_change(vp, mode, ip->i_uid, cred, td); #endif if (error == 0 && (ip->i_flag & IN_CHANGE) != 0) error = UFS_UPDATE(vp, 0); return (error); } /* * Perform chown operation on inode ip; * inode must be locked prior to call. */ static int ufs_chown(vp, uid, gid, cred, td) struct vnode *vp; uid_t uid; gid_t gid; struct ucred *cred; struct thread *td; { struct inode *ip = VTOI(vp); uid_t ouid; gid_t ogid; int error = 0; #ifdef QUOTA int i; ufs2_daddr_t change; #endif if (uid == (uid_t)VNOVAL) uid = ip->i_uid; if (gid == (gid_t)VNOVAL) gid = ip->i_gid; /* * To modify the ownership of a file, must possess VADMIN for that * file. */ if ((error = VOP_ACCESSX(vp, VWRITE_OWNER, cred, td))) return (error); /* * To change the owner of a file, or change the group of a file to a * group of which we are not a member, the caller must have * privilege. */ if (((uid != ip->i_uid && uid != cred->cr_uid) || (gid != ip->i_gid && !groupmember(gid, cred))) && (error = priv_check_cred(cred, PRIV_VFS_CHOWN))) return (error); ogid = ip->i_gid; ouid = ip->i_uid; #ifdef QUOTA if ((error = getinoquota(ip)) != 0) return (error); if (ouid == uid) { dqrele(vp, ip->i_dquot[USRQUOTA]); ip->i_dquot[USRQUOTA] = NODQUOT; } if (ogid == gid) { dqrele(vp, ip->i_dquot[GRPQUOTA]); ip->i_dquot[GRPQUOTA] = NODQUOT; } change = DIP(ip, i_blocks); (void) chkdq(ip, -change, cred, CHOWN); (void) chkiq(ip, -1, cred, CHOWN); for (i = 0; i < MAXQUOTAS; i++) { dqrele(vp, ip->i_dquot[i]); ip->i_dquot[i] = NODQUOT; } #endif ip->i_gid = gid; DIP_SET(ip, i_gid, gid); ip->i_uid = uid; DIP_SET(ip, i_uid, uid); #ifdef QUOTA if ((error = getinoquota(ip)) == 0) { if (ouid == uid) { dqrele(vp, ip->i_dquot[USRQUOTA]); ip->i_dquot[USRQUOTA] = NODQUOT; } if (ogid == gid) { dqrele(vp, ip->i_dquot[GRPQUOTA]); ip->i_dquot[GRPQUOTA] = NODQUOT; } if ((error = chkdq(ip, change, cred, CHOWN)) == 0) { if ((error = chkiq(ip, 1, cred, CHOWN)) == 0) goto good; else (void) chkdq(ip, -change, cred, CHOWN|FORCE); } for (i = 0; i < MAXQUOTAS; i++) { dqrele(vp, ip->i_dquot[i]); ip->i_dquot[i] = NODQUOT; } } ip->i_gid = ogid; DIP_SET(ip, i_gid, ogid); ip->i_uid = ouid; DIP_SET(ip, i_uid, ouid); if (getinoquota(ip) == 0) { if (ouid == uid) { dqrele(vp, ip->i_dquot[USRQUOTA]); ip->i_dquot[USRQUOTA] = NODQUOT; } if (ogid == gid) { dqrele(vp, ip->i_dquot[GRPQUOTA]); ip->i_dquot[GRPQUOTA] = NODQUOT; } (void) chkdq(ip, change, cred, FORCE|CHOWN); (void) chkiq(ip, 1, cred, FORCE|CHOWN); (void) getinoquota(ip); } return (error); good: if (getinoquota(ip)) panic("ufs_chown: lost quota"); #endif /* QUOTA */ ip->i_flag |= IN_CHANGE; if ((ip->i_mode & (ISUID | ISGID)) && (ouid != uid || ogid != gid)) { if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID)) { ip->i_mode &= ~(ISUID | ISGID); DIP_SET(ip, i_mode, ip->i_mode); } } error = UFS_UPDATE(vp, 0); return (error); } static int ufs_remove(ap) struct vop_remove_args /* { struct vnode *a_dvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { struct inode *ip; struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; int error; struct thread *td; td = curthread; ip = VTOI(vp); if ((ip->i_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (VTOI(dvp)->i_flags & APPEND)) { error = EPERM; goto out; } #ifdef UFS_GJOURNAL ufs_gjournal_orphan(vp); #endif error = ufs_dirremove(dvp, ip, ap->a_cnp->cn_flags, 0); if (ip->i_nlink <= 0) vp->v_vflag |= VV_NOSYNC; if ((ip->i_flags & SF_SNAPSHOT) != 0) { /* * Avoid deadlock where another thread is trying to * update the inodeblock for dvp and is waiting on * snaplk. Temporary unlock the vnode lock for the * unlinked file and sync the directory. This should * allow vput() of the directory to not block later on * while holding the snapshot vnode locked, assuming * that the directory hasn't been unlinked too. */ VOP_UNLOCK(vp, 0); (void) VOP_FSYNC(dvp, MNT_WAIT, td); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } out: return (error); } static void print_bad_link_count(const char *funcname, struct vnode *dvp) { struct inode *dip; dip = VTOI(dvp); uprintf("%s: Bad link count %d on parent inode %jd in file system %s\n", funcname, dip->i_effnlink, (intmax_t)dip->i_number, dvp->v_mount->mnt_stat.f_mntonname); } /* * link vnode call */ static int ufs_link(ap) struct vop_link_args /* { struct vnode *a_tdvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { struct vnode *vp = ap->a_vp; struct vnode *tdvp = ap->a_tdvp; struct componentname *cnp = ap->a_cnp; struct inode *ip; struct direct newdir; int error; #ifdef INVARIANTS if ((cnp->cn_flags & HASBUF) == 0) panic("ufs_link: no name"); #endif if (VTOI(tdvp)->i_effnlink < 2) { print_bad_link_count("ufs_link", tdvp); error = EINVAL; goto out; } ip = VTOI(vp); if (ip->i_nlink >= UFS_LINK_MAX) { error = EMLINK; goto out; } /* * The file may have been removed after namei droped the original * lock. */ if (ip->i_effnlink == 0) { error = ENOENT; goto out; } if (ip->i_flags & (IMMUTABLE | APPEND)) { error = EPERM; goto out; } ip->i_effnlink++; ip->i_nlink++; DIP_SET(ip, i_nlink, ip->i_nlink); ip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(vp)) softdep_setup_link(VTOI(tdvp), ip); error = UFS_UPDATE(vp, !DOINGSOFTDEP(vp) && !DOINGASYNC(vp)); if (!error) { ufs_makedirentry(ip, cnp, &newdir); error = ufs_direnter(tdvp, vp, &newdir, cnp, NULL, 0); } if (error) { ip->i_effnlink--; ip->i_nlink--; DIP_SET(ip, i_nlink, ip->i_nlink); ip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(vp)) softdep_revert_link(VTOI(tdvp), ip); } out: return (error); } /* * whiteout vnode call */ static int ufs_whiteout(ap) struct vop_whiteout_args /* { struct vnode *a_dvp; struct componentname *a_cnp; int a_flags; } */ *ap; { struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct direct newdir; int error = 0; switch (ap->a_flags) { case LOOKUP: /* 4.4 format directories support whiteout operations */ if (dvp->v_mount->mnt_maxsymlinklen > 0) return (0); return (EOPNOTSUPP); case CREATE: /* create a new directory whiteout */ #ifdef INVARIANTS if ((cnp->cn_flags & SAVENAME) == 0) panic("ufs_whiteout: missing name"); if (dvp->v_mount->mnt_maxsymlinklen <= 0) panic("ufs_whiteout: old format filesystem"); #endif newdir.d_ino = UFS_WINO; newdir.d_namlen = cnp->cn_namelen; bcopy(cnp->cn_nameptr, newdir.d_name, (unsigned)cnp->cn_namelen + 1); newdir.d_type = DT_WHT; error = ufs_direnter(dvp, NULL, &newdir, cnp, NULL, 0); break; case DELETE: /* remove an existing directory whiteout */ #ifdef INVARIANTS if (dvp->v_mount->mnt_maxsymlinklen <= 0) panic("ufs_whiteout: old format filesystem"); #endif cnp->cn_flags &= ~DOWHITEOUT; error = ufs_dirremove(dvp, NULL, cnp->cn_flags, 0); break; default: panic("ufs_whiteout: unknown op"); } return (error); } static volatile int rename_restarts; SYSCTL_INT(_vfs_ufs, OID_AUTO, rename_restarts, CTLFLAG_RD, __DEVOLATILE(int *, &rename_restarts), 0, "Times rename had to restart due to lock contention"); /* * Rename system call. * rename("foo", "bar"); * is essentially * unlink("bar"); * link("foo", "bar"); * unlink("foo"); * but ``atomically''. Can't do full commit without saving state in the * inode on disk which isn't feasible at this time. Best we can do is * always guarantee the target exists. * * Basic algorithm is: * * 1) Bump link count on source while we're linking it to the * target. This also ensure the inode won't be deleted out * from underneath us while we work (it may be truncated by * a concurrent `trunc' or `open' for creation). * 2) Link source to destination. If destination already exists, * delete it first. * 3) Unlink source reference to inode if still around. If a * directory was moved and the parent of the destination * is different from the source, patch the ".." entry in the * directory. */ static int ufs_rename(ap) struct vop_rename_args /* { struct vnode *a_fdvp; struct vnode *a_fvp; struct componentname *a_fcnp; struct vnode *a_tdvp; struct vnode *a_tvp; struct componentname *a_tcnp; } */ *ap; { struct vnode *tvp = ap->a_tvp; struct vnode *tdvp = ap->a_tdvp; struct vnode *fvp = ap->a_fvp; struct vnode *fdvp = ap->a_fdvp; struct vnode *nvp; struct componentname *tcnp = ap->a_tcnp; struct componentname *fcnp = ap->a_fcnp; struct thread *td = fcnp->cn_thread; struct inode *fip, *tip, *tdp, *fdp; struct direct newdir; off_t endoff; int doingdirectory, newparent; int error = 0; struct mount *mp; ino_t ino; #ifdef INVARIANTS if ((tcnp->cn_flags & HASBUF) == 0 || (fcnp->cn_flags & HASBUF) == 0) panic("ufs_rename: no name"); #endif endoff = 0; mp = tdvp->v_mount; VOP_UNLOCK(tdvp, 0); if (tvp && tvp != tdvp) VOP_UNLOCK(tvp, 0); /* * Check for cross-device rename. */ if ((fvp->v_mount != tdvp->v_mount) || (tvp && (fvp->v_mount != tvp->v_mount))) { error = EXDEV; mp = NULL; goto releout; } relock: /* * We need to acquire 2 to 4 locks depending on whether tvp is NULL * and fdvp and tdvp are the same directory. Subsequently we need * to double-check all paths and in the directory rename case we * need to verify that we are not creating a directory loop. To * handle this we acquire all but fdvp using non-blocking * acquisitions. If we fail to acquire any lock in the path we will * drop all held locks, acquire the new lock in a blocking fashion, * and then release it and restart the rename. This acquire/release * step ensures that we do not spin on a lock waiting for release. */ error = vn_lock(fdvp, LK_EXCLUSIVE); if (error) goto releout; if (vn_lock(tdvp, LK_EXCLUSIVE | LK_NOWAIT) != 0) { VOP_UNLOCK(fdvp, 0); error = vn_lock(tdvp, LK_EXCLUSIVE); if (error) goto releout; VOP_UNLOCK(tdvp, 0); atomic_add_int(&rename_restarts, 1); goto relock; } /* * Re-resolve fvp to be certain it still exists and fetch the * correct vnode. */ error = ufs_lookup_ino(fdvp, NULL, fcnp, &ino); if (error) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); goto releout; } error = VFS_VGET(mp, ino, LK_EXCLUSIVE | LK_NOWAIT, &nvp); if (error) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); if (error != EBUSY) goto releout; error = VFS_VGET(mp, ino, LK_EXCLUSIVE, &nvp); if (error != 0) goto releout; VOP_UNLOCK(nvp, 0); vrele(fvp); fvp = nvp; atomic_add_int(&rename_restarts, 1); goto relock; } vrele(fvp); fvp = nvp; /* * Re-resolve tvp and acquire the vnode lock if present. */ error = ufs_lookup_ino(tdvp, NULL, tcnp, &ino); if (error != 0 && error != EJUSTRETURN) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); VOP_UNLOCK(fvp, 0); goto releout; } /* * If tvp disappeared we just carry on. */ if (error == EJUSTRETURN && tvp != NULL) { vrele(tvp); tvp = NULL; } /* * Get the tvp ino if the lookup succeeded. We may have to restart * if the non-blocking acquire fails. */ if (error == 0) { nvp = NULL; error = VFS_VGET(mp, ino, LK_EXCLUSIVE | LK_NOWAIT, &nvp); if (tvp) vrele(tvp); tvp = nvp; if (error) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(tdvp, 0); VOP_UNLOCK(fvp, 0); if (error != EBUSY) goto releout; error = VFS_VGET(mp, ino, LK_EXCLUSIVE, &nvp); if (error != 0) goto releout; vput(nvp); atomic_add_int(&rename_restarts, 1); goto relock; } } fdp = VTOI(fdvp); fip = VTOI(fvp); tdp = VTOI(tdvp); tip = NULL; if (tvp) tip = VTOI(tvp); if (tvp && ((VTOI(tvp)->i_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (VTOI(tdvp)->i_flags & APPEND))) { error = EPERM; goto unlockout; } /* * Renaming a file to itself has no effect. The upper layers should * not call us in that case. However, things could change after * we drop the locks above. */ if (fvp == tvp) { error = 0; goto unlockout; } doingdirectory = 0; newparent = 0; ino = fip->i_number; if (fip->i_nlink >= UFS_LINK_MAX) { error = EMLINK; goto unlockout; } if ((fip->i_flags & (NOUNLINK | IMMUTABLE | APPEND)) || (fdp->i_flags & APPEND)) { error = EPERM; goto unlockout; } if ((fip->i_mode & IFMT) == IFDIR) { /* * Avoid ".", "..", and aliases of "." for obvious reasons. */ if ((fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.') || fdp == fip || (fcnp->cn_flags | tcnp->cn_flags) & ISDOTDOT) { error = EINVAL; goto unlockout; } if (fdp->i_number != tdp->i_number) newparent = tdp->i_number; doingdirectory = 1; } if ((fvp->v_type == VDIR && fvp->v_mountedhere != NULL) || (tvp != NULL && tvp->v_type == VDIR && tvp->v_mountedhere != NULL)) { error = EXDEV; goto unlockout; } /* * If ".." must be changed (ie the directory gets a new * parent) then the source directory must not be in the * directory hierarchy above the target, as this would * orphan everything below the source directory. Also * the user must have write permission in the source so * as to be able to change "..". */ if (doingdirectory && newparent) { error = VOP_ACCESS(fvp, VWRITE, tcnp->cn_cred, tcnp->cn_thread); if (error) goto unlockout; error = ufs_checkpath(ino, fdp->i_number, tdp, tcnp->cn_cred, &ino); /* * We encountered a lock that we have to wait for. Unlock * everything else and VGET before restarting. */ if (ino) { VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(fvp, 0); VOP_UNLOCK(tdvp, 0); if (tvp) VOP_UNLOCK(tvp, 0); error = VFS_VGET(mp, ino, LK_SHARED, &nvp); if (error == 0) vput(nvp); atomic_add_int(&rename_restarts, 1); goto relock; } if (error) goto unlockout; if ((tcnp->cn_flags & SAVESTART) == 0) panic("ufs_rename: lost to startdir"); } if (fip->i_effnlink == 0 || fdp->i_effnlink == 0 || tdp->i_effnlink == 0) panic("Bad effnlink fip %p, fdp %p, tdp %p", fip, fdp, tdp); /* * 1) Bump link count while we're moving stuff * around. If we crash somewhere before * completing our work, the link count * may be wrong, but correctable. */ fip->i_effnlink++; fip->i_nlink++; DIP_SET(fip, i_nlink, fip->i_nlink); fip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(fvp)) softdep_setup_link(tdp, fip); error = UFS_UPDATE(fvp, !DOINGSOFTDEP(fvp) && !DOINGASYNC(fvp)); if (error) goto bad; /* * 2) If target doesn't exist, link the target * to the source and unlink the source. * Otherwise, rewrite the target directory * entry to reference the source inode and * expunge the original entry's existence. */ if (tip == NULL) { if (ITODEV(tdp) != ITODEV(fip)) panic("ufs_rename: EXDEV"); if (doingdirectory && newparent) { /* * Account for ".." in new directory. * When source and destination have the same * parent we don't adjust the link count. The * actual link modification is completed when * .. is rewritten below. */ if (tdp->i_nlink >= UFS_LINK_MAX) { error = EMLINK; goto bad; } } ufs_makedirentry(fip, tcnp, &newdir); error = ufs_direnter(tdvp, NULL, &newdir, tcnp, NULL, 1); if (error) goto bad; /* Setup tdvp for directory compaction if needed. */ if (tdp->i_count && tdp->i_endoff && tdp->i_endoff < tdp->i_size) endoff = tdp->i_endoff; } else { if (ITODEV(tip) != ITODEV(tdp) || ITODEV(tip) != ITODEV(fip)) panic("ufs_rename: EXDEV"); /* * Short circuit rename(foo, foo). */ if (tip->i_number == fip->i_number) panic("ufs_rename: same file"); /* * If the parent directory is "sticky", then the caller * must possess VADMIN for the parent directory, or the * destination of the rename. This implements append-only * directories. */ if ((tdp->i_mode & S_ISTXT) && VOP_ACCESS(tdvp, VADMIN, tcnp->cn_cred, td) && VOP_ACCESS(tvp, VADMIN, tcnp->cn_cred, td)) { error = EPERM; goto bad; } /* * Target must be empty if a directory and have no links * to it. Also, ensure source and target are compatible * (both directories, or both not directories). */ if ((tip->i_mode & IFMT) == IFDIR) { if ((tip->i_effnlink > 2) || !ufs_dirempty(tip, tdp->i_number, tcnp->cn_cred)) { error = ENOTEMPTY; goto bad; } if (!doingdirectory) { error = ENOTDIR; goto bad; } cache_purge(tdvp); } else if (doingdirectory) { error = EISDIR; goto bad; } if (doingdirectory) { if (!newparent) { tdp->i_effnlink--; if (DOINGSOFTDEP(tdvp)) softdep_change_linkcnt(tdp); } tip->i_effnlink--; if (DOINGSOFTDEP(tvp)) softdep_change_linkcnt(tip); } error = ufs_dirrewrite(tdp, tip, fip->i_number, IFTODT(fip->i_mode), (doingdirectory && newparent) ? newparent : doingdirectory); if (error) { if (doingdirectory) { if (!newparent) { tdp->i_effnlink++; if (DOINGSOFTDEP(tdvp)) softdep_change_linkcnt(tdp); } tip->i_effnlink++; if (DOINGSOFTDEP(tvp)) softdep_change_linkcnt(tip); } } if (doingdirectory && !DOINGSOFTDEP(tvp)) { /* * The only stuff left in the directory is "." * and "..". The "." reference is inconsequential * since we are quashing it. We have removed the "." * reference and the reference in the parent directory, * but there may be other hard links. The soft * dependency code will arrange to do these operations * after the parent directory entry has been deleted on * disk, so when running with that code we avoid doing * them now. */ if (!newparent) { tdp->i_nlink--; DIP_SET(tdp, i_nlink, tdp->i_nlink); tdp->i_flag |= IN_CHANGE; } tip->i_nlink--; DIP_SET(tip, i_nlink, tip->i_nlink); tip->i_flag |= IN_CHANGE; } } /* * 3) Unlink the source. We have to resolve the path again to * fixup the directory offset and count for ufs_dirremove. */ if (fdvp == tdvp) { error = ufs_lookup_ino(fdvp, NULL, fcnp, &ino); if (error) panic("ufs_rename: from entry went away!"); if (ino != fip->i_number) panic("ufs_rename: ino mismatch %ju != %ju\n", (uintmax_t)ino, (uintmax_t)fip->i_number); } /* * If the source is a directory with a * new parent, the link count of the old * parent directory must be decremented * and ".." set to point to the new parent. */ if (doingdirectory && newparent) { /* * If tip exists we simply use its link, otherwise we must * add a new one. */ if (tip == NULL) { tdp->i_effnlink++; tdp->i_nlink++; DIP_SET(tdp, i_nlink, tdp->i_nlink); tdp->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(tdvp)) softdep_setup_dotdot_link(tdp, fip); error = UFS_UPDATE(tdvp, !DOINGSOFTDEP(tdvp) && !DOINGASYNC(tdvp)); /* Don't go to bad here as the new link exists. */ if (error) goto unlockout; } else if (DOINGSUJ(tdvp)) /* Journal must account for each new link. */ softdep_setup_dotdot_link(tdp, fip); fip->i_offset = mastertemplate.dot_reclen; ufs_dirrewrite(fip, fdp, newparent, DT_DIR, 0); cache_purge(fdvp); } error = ufs_dirremove(fdvp, fip, fcnp->cn_flags, 0); /* * The kern_renameat() looks up the fvp using the DELETE flag, which * causes the removal of the name cache entry for fvp. * As the relookup of the fvp is done in two steps: * ufs_lookup_ino() and then VFS_VGET(), another thread might do a * normal lookup of the from name just before the VFS_VGET() call, * causing the cache entry to be re-instantiated. * * The same issue also applies to tvp if it exists as * otherwise we may have a stale name cache entry for the new * name that references the old i-node if it has other links * or open file descriptors. */ cache_purge(fvp); if (tvp) cache_purge(tvp); cache_purge_negative(tdvp); unlockout: vput(fdvp); vput(fvp); if (tvp) vput(tvp); /* * If compaction or fsync was requested do it now that other locks * are no longer needed. */ if (error == 0 && endoff != 0) { error = UFS_TRUNCATE(tdvp, endoff, IO_NORMAL | (DOINGASYNC(tdvp) ? 0 : IO_SYNC), tcnp->cn_cred); if (error != 0) vn_printf(tdvp, "ufs_rename: failed to truncate, error %d\n", error); #ifdef UFS_DIRHASH else if (tdp->i_dirhash != NULL) ufsdirhash_dirtrunc(tdp, endoff); #endif /* * Even if the directory compaction failed, rename was * succesful. Do not propagate a UFS_TRUNCATE() error * to the caller. */ error = 0; } if (error == 0 && tdp->i_flag & IN_NEEDSYNC) error = VOP_FSYNC(tdvp, MNT_WAIT, td); vput(tdvp); return (error); bad: fip->i_effnlink--; fip->i_nlink--; DIP_SET(fip, i_nlink, fip->i_nlink); fip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(fvp)) softdep_revert_link(tdp, fip); goto unlockout; releout: vrele(fdvp); vrele(fvp); vrele(tdvp); if (tvp) vrele(tvp); return (error); } #ifdef UFS_ACL static int ufs_do_posix1e_acl_inheritance_dir(struct vnode *dvp, struct vnode *tvp, mode_t dmode, struct ucred *cred, struct thread *td) { int error; struct inode *ip = VTOI(tvp); struct acl *dacl, *acl; acl = acl_alloc(M_WAITOK); dacl = acl_alloc(M_WAITOK); /* * Retrieve default ACL from parent, if any. */ error = VOP_GETACL(dvp, ACL_TYPE_DEFAULT, acl, cred, td); switch (error) { case 0: /* * Retrieved a default ACL, so merge mode and ACL if * necessary. If the ACL is empty, fall through to * the "not defined or available" case. */ if (acl->acl_cnt != 0) { dmode = acl_posix1e_newfilemode(dmode, acl); ip->i_mode = dmode; DIP_SET(ip, i_mode, dmode); *dacl = *acl; ufs_sync_acl_from_inode(ip, acl); break; } /* FALLTHROUGH */ case EOPNOTSUPP: /* * Just use the mode as-is. */ ip->i_mode = dmode; DIP_SET(ip, i_mode, dmode); error = 0; goto out; default: goto out; } /* * XXX: If we abort now, will Soft Updates notify the extattr * code that the EAs for the file need to be released? */ error = VOP_SETACL(tvp, ACL_TYPE_ACCESS, acl, cred, td); if (error == 0) error = VOP_SETACL(tvp, ACL_TYPE_DEFAULT, dacl, cred, td); switch (error) { case 0: break; case EOPNOTSUPP: /* * XXX: This should not happen, as EOPNOTSUPP above * was supposed to free acl. */ printf("ufs_mkdir: VOP_GETACL() but no VOP_SETACL()\n"); /* panic("ufs_mkdir: VOP_GETACL() but no VOP_SETACL()"); */ break; default: goto out; } out: acl_free(acl); acl_free(dacl); return (error); } static int ufs_do_posix1e_acl_inheritance_file(struct vnode *dvp, struct vnode *tvp, mode_t mode, struct ucred *cred, struct thread *td) { int error; struct inode *ip = VTOI(tvp); struct acl *acl; acl = acl_alloc(M_WAITOK); /* * Retrieve default ACL for parent, if any. */ error = VOP_GETACL(dvp, ACL_TYPE_DEFAULT, acl, cred, td); switch (error) { case 0: /* * Retrieved a default ACL, so merge mode and ACL if * necessary. */ if (acl->acl_cnt != 0) { /* * Two possible ways for default ACL to not * be present. First, the EA can be * undefined, or second, the default ACL can * be blank. If it's blank, fall through to * the it's not defined case. */ mode = acl_posix1e_newfilemode(mode, acl); ip->i_mode = mode; DIP_SET(ip, i_mode, mode); ufs_sync_acl_from_inode(ip, acl); break; } /* FALLTHROUGH */ case EOPNOTSUPP: /* * Just use the mode as-is. */ ip->i_mode = mode; DIP_SET(ip, i_mode, mode); error = 0; goto out; default: goto out; } /* * XXX: If we abort now, will Soft Updates notify the extattr * code that the EAs for the file need to be released? */ error = VOP_SETACL(tvp, ACL_TYPE_ACCESS, acl, cred, td); switch (error) { case 0: break; case EOPNOTSUPP: /* * XXX: This should not happen, as EOPNOTSUPP above was * supposed to free acl. */ printf("ufs_do_posix1e_acl_inheritance_file: VOP_GETACL() " "but no VOP_SETACL()\n"); /* panic("ufs_do_posix1e_acl_inheritance_file: VOP_GETACL() " "but no VOP_SETACL()"); */ break; default: goto out; } out: acl_free(acl); return (error); } static int ufs_do_nfs4_acl_inheritance(struct vnode *dvp, struct vnode *tvp, mode_t child_mode, struct ucred *cred, struct thread *td) { int error; struct acl *parent_aclp, *child_aclp; parent_aclp = acl_alloc(M_WAITOK); child_aclp = acl_alloc(M_WAITOK | M_ZERO); error = ufs_getacl_nfs4_internal(dvp, parent_aclp, td); if (error) goto out; acl_nfs4_compute_inherited_acl(parent_aclp, child_aclp, child_mode, VTOI(tvp)->i_uid, tvp->v_type == VDIR); error = ufs_setacl_nfs4_internal(tvp, child_aclp, td); if (error) goto out; out: acl_free(parent_aclp); acl_free(child_aclp); return (error); } #endif /* * Mkdir system call */ static int ufs_mkdir(ap) struct vop_mkdir_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; } */ *ap; { struct vnode *dvp = ap->a_dvp; struct vattr *vap = ap->a_vap; struct componentname *cnp = ap->a_cnp; struct inode *ip, *dp; struct vnode *tvp; struct buf *bp; struct dirtemplate dirtemplate, *dtp; struct direct newdir; int error, dmode; long blkoff; #ifdef INVARIANTS if ((cnp->cn_flags & HASBUF) == 0) panic("ufs_mkdir: no name"); #endif dp = VTOI(dvp); if (dp->i_nlink >= UFS_LINK_MAX) { error = EMLINK; goto out; } dmode = vap->va_mode & 0777; dmode |= IFDIR; /* * Must simulate part of ufs_makeinode here to acquire the inode, * but not have it entered in the parent directory. The entry is * made later after writing "." and ".." entries. */ if (dp->i_effnlink < 2) { print_bad_link_count("ufs_mkdir", dvp); error = EINVAL; goto out; } error = UFS_VALLOC(dvp, dmode, cnp->cn_cred, &tvp); if (error) goto out; ip = VTOI(tvp); ip->i_gid = dp->i_gid; DIP_SET(ip, i_gid, dp->i_gid); #ifdef SUIDDIR { #ifdef QUOTA struct ucred ucred, *ucp; gid_t ucred_group; ucp = cnp->cn_cred; #endif /* * If we are hacking owners here, (only do this where told to) * and we are not giving it TO root, (would subvert quotas) * then go ahead and give it to the other user. * The new directory also inherits the SUID bit. * If user's UID and dir UID are the same, * 'give it away' so that the SUID is still forced on. */ if ((dvp->v_mount->mnt_flag & MNT_SUIDDIR) && (dp->i_mode & ISUID) && dp->i_uid) { dmode |= ISUID; ip->i_uid = dp->i_uid; DIP_SET(ip, i_uid, dp->i_uid); #ifdef QUOTA if (dp->i_uid != cnp->cn_cred->cr_uid) { /* * Make sure the correct user gets charged * for the space. * Make a dummy credential for the victim. * XXX This seems to never be accessed out of * our context so a stack variable is ok. */ refcount_init(&ucred.cr_ref, 1); ucred.cr_uid = ip->i_uid; ucred.cr_ngroups = 1; ucred.cr_groups = &ucred_group; ucred.cr_groups[0] = dp->i_gid; ucp = &ucred; } #endif } else { ip->i_uid = cnp->cn_cred->cr_uid; DIP_SET(ip, i_uid, ip->i_uid); } #ifdef QUOTA if ((error = getinoquota(ip)) || (error = chkiq(ip, 1, ucp, 0))) { if (DOINGSOFTDEP(tvp)) softdep_revert_link(dp, ip); UFS_VFREE(tvp, ip->i_number, dmode); vput(tvp); return (error); } #endif } #else /* !SUIDDIR */ ip->i_uid = cnp->cn_cred->cr_uid; DIP_SET(ip, i_uid, ip->i_uid); #ifdef QUOTA if ((error = getinoquota(ip)) || (error = chkiq(ip, 1, cnp->cn_cred, 0))) { if (DOINGSOFTDEP(tvp)) softdep_revert_link(dp, ip); UFS_VFREE(tvp, ip->i_number, dmode); vput(tvp); return (error); } #endif #endif /* !SUIDDIR */ ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE; ip->i_mode = dmode; DIP_SET(ip, i_mode, dmode); tvp->v_type = VDIR; /* Rest init'd in getnewvnode(). */ ip->i_effnlink = 2; ip->i_nlink = 2; DIP_SET(ip, i_nlink, 2); if (cnp->cn_flags & ISWHITEOUT) { ip->i_flags |= UF_OPAQUE; DIP_SET(ip, i_flags, ip->i_flags); } /* * Bump link count in parent directory to reflect work done below. * Should be done before reference is created so cleanup is * possible if we crash. */ dp->i_effnlink++; dp->i_nlink++; DIP_SET(dp, i_nlink, dp->i_nlink); dp->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(dvp)) softdep_setup_mkdir(dp, ip); error = UFS_UPDATE(dvp, !DOINGSOFTDEP(dvp) && !DOINGASYNC(dvp)); if (error) goto bad; #ifdef MAC if (dvp->v_mount->mnt_flag & MNT_MULTILABEL) { error = mac_vnode_create_extattr(cnp->cn_cred, dvp->v_mount, dvp, tvp, cnp); if (error) goto bad; } #endif #ifdef UFS_ACL if (dvp->v_mount->mnt_flag & MNT_ACLS) { error = ufs_do_posix1e_acl_inheritance_dir(dvp, tvp, dmode, cnp->cn_cred, cnp->cn_thread); if (error) goto bad; } else if (dvp->v_mount->mnt_flag & MNT_NFS4ACLS) { error = ufs_do_nfs4_acl_inheritance(dvp, tvp, dmode, cnp->cn_cred, cnp->cn_thread); if (error) goto bad; } #endif /* !UFS_ACL */ /* * Initialize directory with "." and ".." from static template. */ if (dvp->v_mount->mnt_maxsymlinklen > 0) dtp = &mastertemplate; else dtp = (struct dirtemplate *)&omastertemplate; dirtemplate = *dtp; dirtemplate.dot_ino = ip->i_number; dirtemplate.dotdot_ino = dp->i_number; vnode_pager_setsize(tvp, DIRBLKSIZ); if ((error = UFS_BALLOC(tvp, (off_t)0, DIRBLKSIZ, cnp->cn_cred, BA_CLRBUF, &bp)) != 0) goto bad; ip->i_size = DIRBLKSIZ; DIP_SET(ip, i_size, DIRBLKSIZ); ip->i_flag |= IN_CHANGE | IN_UPDATE; bcopy((caddr_t)&dirtemplate, (caddr_t)bp->b_data, sizeof dirtemplate); if (DOINGSOFTDEP(tvp)) { /* * Ensure that the entire newly allocated block is a * valid directory so that future growth within the * block does not have to ensure that the block is * written before the inode. */ blkoff = DIRBLKSIZ; while (blkoff < bp->b_bcount) { ((struct direct *) (bp->b_data + blkoff))->d_reclen = DIRBLKSIZ; blkoff += DIRBLKSIZ; } } if ((error = UFS_UPDATE(tvp, !DOINGSOFTDEP(tvp) && !DOINGASYNC(tvp))) != 0) { (void)bwrite(bp); goto bad; } /* * Directory set up, now install its entry in the parent directory. * * If we are not doing soft dependencies, then we must write out the * buffer containing the new directory body before entering the new * name in the parent. If we are doing soft dependencies, then the * buffer containing the new directory body will be passed to and * released in the soft dependency code after the code has attached * an appropriate ordering dependency to the buffer which ensures that * the buffer is written before the new name is written in the parent. */ if (DOINGASYNC(dvp)) bdwrite(bp); else if (!DOINGSOFTDEP(dvp) && ((error = bwrite(bp)))) goto bad; ufs_makedirentry(ip, cnp, &newdir); error = ufs_direnter(dvp, tvp, &newdir, cnp, bp, 0); bad: if (error == 0) { *ap->a_vpp = tvp; } else { dp->i_effnlink--; dp->i_nlink--; DIP_SET(dp, i_nlink, dp->i_nlink); dp->i_flag |= IN_CHANGE; /* * No need to do an explicit VOP_TRUNCATE here, vrele will * do this for us because we set the link count to 0. */ ip->i_effnlink = 0; ip->i_nlink = 0; DIP_SET(ip, i_nlink, 0); ip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(tvp)) softdep_revert_mkdir(dp, ip); vput(tvp); } out: return (error); } /* * Rmdir system call. */ static int ufs_rmdir(ap) struct vop_rmdir_args /* { struct vnode *a_dvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct inode *ip, *dp; int error; ip = VTOI(vp); dp = VTOI(dvp); /* * Do not remove a directory that is in the process of being renamed. * Verify the directory is empty (and valid). Rmdir ".." will not be * valid since ".." will contain a reference to the current directory * and thus be non-empty. Do not allow the removal of mounted on * directories (this can happen when an NFS exported filesystem * tries to remove a locally mounted on directory). */ error = 0; if (dp->i_effnlink <= 2) { if (dp->i_effnlink == 2) print_bad_link_count("ufs_rmdir", dvp); error = EINVAL; goto out; } if (!ufs_dirempty(ip, dp->i_number, cnp->cn_cred)) { error = ENOTEMPTY; goto out; } if ((dp->i_flags & APPEND) || (ip->i_flags & (NOUNLINK | IMMUTABLE | APPEND))) { error = EPERM; goto out; } if (vp->v_mountedhere != 0) { error = EINVAL; goto out; } #ifdef UFS_GJOURNAL ufs_gjournal_orphan(vp); #endif /* * Delete reference to directory before purging * inode. If we crash in between, the directory * will be reattached to lost+found, */ dp->i_effnlink--; ip->i_effnlink--; if (DOINGSOFTDEP(vp)) softdep_setup_rmdir(dp, ip); error = ufs_dirremove(dvp, ip, cnp->cn_flags, 1); if (error) { dp->i_effnlink++; ip->i_effnlink++; if (DOINGSOFTDEP(vp)) softdep_revert_rmdir(dp, ip); goto out; } cache_purge(dvp); /* * The only stuff left in the directory is "." and "..". The "." * reference is inconsequential since we are quashing it. The soft * dependency code will arrange to do these operations after * the parent directory entry has been deleted on disk, so * when running with that code we avoid doing them now. */ if (!DOINGSOFTDEP(vp)) { dp->i_nlink--; DIP_SET(dp, i_nlink, dp->i_nlink); dp->i_flag |= IN_CHANGE; error = UFS_UPDATE(dvp, 0); ip->i_nlink--; DIP_SET(ip, i_nlink, ip->i_nlink); ip->i_flag |= IN_CHANGE; } cache_purge(vp); #ifdef UFS_DIRHASH /* Kill any active hash; i_effnlink == 0, so it will not come back. */ if (ip->i_dirhash != NULL) ufsdirhash_free(ip); #endif out: return (error); } /* * symlink -- make a symbolic link */ static int ufs_symlink(ap) struct vop_symlink_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; const char *a_target; } */ *ap; { struct vnode *vp, **vpp = ap->a_vpp; struct inode *ip; int len, error; error = ufs_makeinode(IFLNK | ap->a_vap->va_mode, ap->a_dvp, vpp, ap->a_cnp, "ufs_symlink"); if (error) return (error); vp = *vpp; len = strlen(ap->a_target); if (len < vp->v_mount->mnt_maxsymlinklen) { ip = VTOI(vp); bcopy(ap->a_target, SHORTLINK(ip), len); ip->i_size = len; DIP_SET(ip, i_size, len); ip->i_flag |= IN_CHANGE | IN_UPDATE; error = UFS_UPDATE(vp, 0); } else error = vn_rdwr(UIO_WRITE, vp, __DECONST(void *, ap->a_target), len, (off_t)0, UIO_SYSSPACE, IO_NODELOCKED | IO_NOMACCHECK, ap->a_cnp->cn_cred, NOCRED, NULL, NULL); if (error) vput(vp); return (error); } /* * Vnode op for reading directories. */ int ufs_readdir(ap) struct vop_readdir_args /* { struct vnode *a_vp; struct uio *a_uio; struct ucred *a_cred; int *a_eofflag; int *a_ncookies; u_long **a_cookies; } */ *ap; { struct vnode *vp = ap->a_vp; struct uio *uio = ap->a_uio; struct buf *bp; struct inode *ip; struct direct *dp, *edp; u_long *cookies; struct dirent dstdp; off_t offset, startoffset; size_t readcnt, skipcnt; ssize_t startresid; u_int ncookies; int error; if (uio->uio_offset < 0) return (EINVAL); ip = VTOI(vp); if (ip->i_effnlink == 0) return (0); if (ap->a_ncookies != NULL) { if (uio->uio_resid < 0) ncookies = 0; else ncookies = uio->uio_resid; if (uio->uio_offset >= ip->i_size) ncookies = 0; else if (ip->i_size - uio->uio_offset < ncookies) ncookies = ip->i_size - uio->uio_offset; ncookies = ncookies / (offsetof(struct direct, d_name) + 4) + 1; cookies = malloc(ncookies * sizeof(*cookies), M_TEMP, M_WAITOK); *ap->a_ncookies = ncookies; *ap->a_cookies = cookies; } else { ncookies = 0; cookies = NULL; } offset = startoffset = uio->uio_offset; startresid = uio->uio_resid; error = 0; while (error == 0 && uio->uio_resid > 0 && uio->uio_offset < ip->i_size) { error = ffs_blkatoff(vp, uio->uio_offset, NULL, &bp); if (error) break; if (bp->b_offset + bp->b_bcount > ip->i_size) readcnt = ip->i_size - bp->b_offset; else readcnt = bp->b_bcount; skipcnt = (size_t)(uio->uio_offset - bp->b_offset) & ~(size_t)(DIRBLKSIZ - 1); offset = bp->b_offset + skipcnt; dp = (struct direct *)&bp->b_data[skipcnt]; edp = (struct direct *)&bp->b_data[readcnt]; while (error == 0 && uio->uio_resid > 0 && dp < edp) { if (dp->d_reclen <= offsetof(struct direct, d_name) || (caddr_t)dp + dp->d_reclen > (caddr_t)edp) { error = EIO; break; } #if BYTE_ORDER == LITTLE_ENDIAN /* Old filesystem format. */ if (vp->v_mount->mnt_maxsymlinklen <= 0) { dstdp.d_namlen = dp->d_type; dstdp.d_type = dp->d_namlen; } else #endif { dstdp.d_namlen = dp->d_namlen; dstdp.d_type = dp->d_type; } if (offsetof(struct direct, d_name) + dstdp.d_namlen > dp->d_reclen) { error = EIO; break; } if (offset < startoffset || dp->d_ino == 0) goto nextentry; dstdp.d_fileno = dp->d_ino; dstdp.d_reclen = GENERIC_DIRSIZ(&dstdp); bcopy(dp->d_name, dstdp.d_name, dstdp.d_namlen); /* NOTE: d_off is the offset of the *next* entry. */ dstdp.d_off = offset + dp->d_reclen; dirent_terminate(&dstdp); if (dstdp.d_reclen > uio->uio_resid) { if (uio->uio_resid == startresid) error = EINVAL; else error = EJUSTRETURN; break; } /* Advance dp. */ error = uiomove((caddr_t)&dstdp, dstdp.d_reclen, uio); if (error) break; if (cookies != NULL) { KASSERT(ncookies > 0, ("ufs_readdir: cookies buffer too small")); *cookies = offset + dp->d_reclen; cookies++; ncookies--; } nextentry: offset += dp->d_reclen; dp = (struct direct *)((caddr_t)dp + dp->d_reclen); } bqrelse(bp); uio->uio_offset = offset; } /* We need to correct uio_offset. */ uio->uio_offset = offset; if (error == EJUSTRETURN) error = 0; if (ap->a_ncookies != NULL) { if (error == 0) { ap->a_ncookies -= ncookies; } else { free(*ap->a_cookies, M_TEMP); *ap->a_ncookies = 0; *ap->a_cookies = NULL; } } if (error == 0 && ap->a_eofflag) *ap->a_eofflag = ip->i_size <= uio->uio_offset; return (error); } /* * Return target name of a symbolic link */ static int ufs_readlink(ap) struct vop_readlink_args /* { struct vnode *a_vp; struct uio *a_uio; struct ucred *a_cred; } */ *ap; { struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); doff_t isize; isize = ip->i_size; if ((isize < vp->v_mount->mnt_maxsymlinklen) || DIP(ip, i_blocks) == 0) { /* XXX - for old fastlink support */ return (uiomove(SHORTLINK(ip), isize, ap->a_uio)); } return (VOP_READ(vp, ap->a_uio, 0, ap->a_cred)); } /* * Calculate the logical to physical mapping if not done already, * then call the device strategy routine. * * In order to be able to swap to a file, the ufs_bmaparray() operation may not * deadlock on memory. See ufs_bmap() for details. */ static int ufs_strategy(ap) struct vop_strategy_args /* { struct vnode *a_vp; struct buf *a_bp; } */ *ap; { struct buf *bp = ap->a_bp; struct vnode *vp = ap->a_vp; ufs2_daddr_t blkno; int error; if (bp->b_blkno == bp->b_lblkno) { error = ufs_bmaparray(vp, bp->b_lblkno, &blkno, bp, NULL, NULL); bp->b_blkno = blkno; if (error) { bp->b_error = error; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return (0); } if ((long)bp->b_blkno == -1) vfs_bio_clrbuf(bp); } if ((long)bp->b_blkno == -1) { bufdone(bp); return (0); } bp->b_iooffset = dbtob(bp->b_blkno); BO_STRATEGY(VFSTOUFS(vp->v_mount)->um_bo, bp); return (0); } /* * Print out the contents of an inode. */ static int ufs_print(ap) struct vop_print_args /* { struct vnode *a_vp; } */ *ap; { struct vnode *vp = ap->a_vp; struct inode *ip = VTOI(vp); printf("\tino %lu, on dev %s", (u_long)ip->i_number, devtoname(ITODEV(ip))); if (vp->v_type == VFIFO) fifo_printinfo(vp); printf("\n"); return (0); } /* * Close wrapper for fifos. * * Update the times on the inode then do device close. */ static int ufsfifo_close(ap) struct vop_close_args /* { struct vnode *a_vp; int a_fflag; struct ucred *a_cred; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; int usecount; VI_LOCK(vp); usecount = vp->v_usecount; if (usecount > 1) ufs_itimes_locked(vp); VI_UNLOCK(vp); return (fifo_specops.vop_close(ap)); } /* * Kqfilter wrapper for fifos. * * Fall through to ufs kqfilter routines if needed */ static int ufsfifo_kqfilter(ap) struct vop_kqfilter_args *ap; { int error; error = fifo_specops.vop_kqfilter(ap); if (error) error = vfs_kqfilter(ap); return (error); } /* * Return POSIX pathconf information applicable to ufs filesystems. */ static int ufs_pathconf(ap) struct vop_pathconf_args /* { struct vnode *a_vp; int a_name; int *a_retval; } */ *ap; { int error; error = 0; switch (ap->a_name) { case _PC_LINK_MAX: *ap->a_retval = UFS_LINK_MAX; break; case _PC_NAME_MAX: *ap->a_retval = UFS_MAXNAMLEN; break; case _PC_PIPE_BUF: if (ap->a_vp->v_type == VDIR || ap->a_vp->v_type == VFIFO) *ap->a_retval = PIPE_BUF; else error = EINVAL; break; case _PC_CHOWN_RESTRICTED: *ap->a_retval = 1; break; case _PC_NO_TRUNC: *ap->a_retval = 1; break; - case _PC_ACL_EXTENDED: #ifdef UFS_ACL + case _PC_ACL_EXTENDED: if (ap->a_vp->v_mount->mnt_flag & MNT_ACLS) *ap->a_retval = 1; else *ap->a_retval = 0; -#else - *ap->a_retval = 0; -#endif break; - case _PC_ACL_NFS4: -#ifdef UFS_ACL if (ap->a_vp->v_mount->mnt_flag & MNT_NFS4ACLS) *ap->a_retval = 1; else *ap->a_retval = 0; -#else - *ap->a_retval = 0; -#endif break; - +#endif case _PC_ACL_PATH_MAX: #ifdef UFS_ACL if (ap->a_vp->v_mount->mnt_flag & (MNT_ACLS | MNT_NFS4ACLS)) *ap->a_retval = ACL_MAX_ENTRIES; else *ap->a_retval = 3; #else *ap->a_retval = 3; #endif break; - case _PC_MAC_PRESENT: #ifdef MAC + case _PC_MAC_PRESENT: if (ap->a_vp->v_mount->mnt_flag & MNT_MULTILABEL) *ap->a_retval = 1; else *ap->a_retval = 0; -#else - *ap->a_retval = 0; -#endif break; +#endif case _PC_MIN_HOLE_SIZE: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize; break; case _PC_PRIO_IO: *ap->a_retval = 0; break; case _PC_SYNC_IO: *ap->a_retval = 0; break; case _PC_ALLOC_SIZE_MIN: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_bsize; break; case _PC_FILESIZEBITS: *ap->a_retval = 64; break; case _PC_REC_INCR_XFER_SIZE: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize; break; case _PC_REC_MAX_XFER_SIZE: *ap->a_retval = -1; /* means ``unlimited'' */ break; case _PC_REC_MIN_XFER_SIZE: *ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize; break; case _PC_REC_XFER_ALIGN: *ap->a_retval = PAGE_SIZE; break; case _PC_SYMLINK_MAX: *ap->a_retval = MAXPATHLEN; break; default: error = vop_stdpathconf(ap); break; } return (error); } /* * Initialize the vnode associated with a new inode, handle aliased * vnodes. */ int ufs_vinit(mntp, fifoops, vpp) struct mount *mntp; struct vop_vector *fifoops; struct vnode **vpp; { struct inode *ip; struct vnode *vp; vp = *vpp; ASSERT_VOP_LOCKED(vp, "ufs_vinit"); ip = VTOI(vp); vp->v_type = IFTOVT(ip->i_mode); /* * Only unallocated inodes should be of type VNON. */ if (ip->i_mode != 0 && vp->v_type == VNON) return (EINVAL); if (vp->v_type == VFIFO) vp->v_op = fifoops; if (ip->i_number == UFS_ROOTINO) vp->v_vflag |= VV_ROOT; *vpp = vp; return (0); } /* * Allocate a new inode. * Vnode dvp must be locked. */ static int ufs_makeinode(mode, dvp, vpp, cnp, callfunc) int mode; struct vnode *dvp; struct vnode **vpp; struct componentname *cnp; const char *callfunc; { struct inode *ip, *pdir; struct direct newdir; struct vnode *tvp; int error; pdir = VTOI(dvp); #ifdef INVARIANTS if ((cnp->cn_flags & HASBUF) == 0) panic("%s: no name", callfunc); #endif *vpp = NULL; if ((mode & IFMT) == 0) mode |= IFREG; if (pdir->i_effnlink < 2) { print_bad_link_count(callfunc, dvp); return (EINVAL); } error = UFS_VALLOC(dvp, mode, cnp->cn_cred, &tvp); if (error) return (error); ip = VTOI(tvp); ip->i_gid = pdir->i_gid; DIP_SET(ip, i_gid, pdir->i_gid); #ifdef SUIDDIR { #ifdef QUOTA struct ucred ucred, *ucp; gid_t ucred_group; ucp = cnp->cn_cred; #endif /* * If we are not the owner of the directory, * and we are hacking owners here, (only do this where told to) * and we are not giving it TO root, (would subvert quotas) * then go ahead and give it to the other user. * Note that this drops off the execute bits for security. */ if ((dvp->v_mount->mnt_flag & MNT_SUIDDIR) && (pdir->i_mode & ISUID) && (pdir->i_uid != cnp->cn_cred->cr_uid) && pdir->i_uid) { ip->i_uid = pdir->i_uid; DIP_SET(ip, i_uid, ip->i_uid); mode &= ~07111; #ifdef QUOTA /* * Make sure the correct user gets charged * for the space. * Quickly knock up a dummy credential for the victim. * XXX This seems to never be accessed out of our * context so a stack variable is ok. */ refcount_init(&ucred.cr_ref, 1); ucred.cr_uid = ip->i_uid; ucred.cr_ngroups = 1; ucred.cr_groups = &ucred_group; ucred.cr_groups[0] = pdir->i_gid; ucp = &ucred; #endif } else { ip->i_uid = cnp->cn_cred->cr_uid; DIP_SET(ip, i_uid, ip->i_uid); } #ifdef QUOTA if ((error = getinoquota(ip)) || (error = chkiq(ip, 1, ucp, 0))) { if (DOINGSOFTDEP(tvp)) softdep_revert_link(pdir, ip); UFS_VFREE(tvp, ip->i_number, mode); vput(tvp); return (error); } #endif } #else /* !SUIDDIR */ ip->i_uid = cnp->cn_cred->cr_uid; DIP_SET(ip, i_uid, ip->i_uid); #ifdef QUOTA if ((error = getinoquota(ip)) || (error = chkiq(ip, 1, cnp->cn_cred, 0))) { if (DOINGSOFTDEP(tvp)) softdep_revert_link(pdir, ip); UFS_VFREE(tvp, ip->i_number, mode); vput(tvp); return (error); } #endif #endif /* !SUIDDIR */ ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE; ip->i_mode = mode; DIP_SET(ip, i_mode, mode); tvp->v_type = IFTOVT(mode); /* Rest init'd in getnewvnode(). */ ip->i_effnlink = 1; ip->i_nlink = 1; DIP_SET(ip, i_nlink, 1); if (DOINGSOFTDEP(tvp)) softdep_setup_create(VTOI(dvp), ip); if ((ip->i_mode & ISGID) && !groupmember(ip->i_gid, cnp->cn_cred) && priv_check_cred(cnp->cn_cred, PRIV_VFS_SETGID)) { ip->i_mode &= ~ISGID; DIP_SET(ip, i_mode, ip->i_mode); } if (cnp->cn_flags & ISWHITEOUT) { ip->i_flags |= UF_OPAQUE; DIP_SET(ip, i_flags, ip->i_flags); } /* * Make sure inode goes to disk before directory entry. */ error = UFS_UPDATE(tvp, !DOINGSOFTDEP(tvp) && !DOINGASYNC(tvp)); if (error) goto bad; #ifdef MAC if (dvp->v_mount->mnt_flag & MNT_MULTILABEL) { error = mac_vnode_create_extattr(cnp->cn_cred, dvp->v_mount, dvp, tvp, cnp); if (error) goto bad; } #endif #ifdef UFS_ACL if (dvp->v_mount->mnt_flag & MNT_ACLS) { error = ufs_do_posix1e_acl_inheritance_file(dvp, tvp, mode, cnp->cn_cred, cnp->cn_thread); if (error) goto bad; } else if (dvp->v_mount->mnt_flag & MNT_NFS4ACLS) { error = ufs_do_nfs4_acl_inheritance(dvp, tvp, mode, cnp->cn_cred, cnp->cn_thread); if (error) goto bad; } #endif /* !UFS_ACL */ ufs_makedirentry(ip, cnp, &newdir); error = ufs_direnter(dvp, tvp, &newdir, cnp, NULL, 0); if (error) goto bad; *vpp = tvp; return (0); bad: /* * Write error occurred trying to update the inode * or the directory so must deallocate the inode. */ ip->i_effnlink = 0; ip->i_nlink = 0; DIP_SET(ip, i_nlink, 0); ip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(tvp)) softdep_revert_create(VTOI(dvp), ip); vput(tvp); return (error); } static int ufs_ioctl(struct vop_ioctl_args *ap) { switch (ap->a_command) { case FIOSEEKDATA: case FIOSEEKHOLE: return (vn_bmap_seekhole(ap->a_vp, ap->a_command, (off_t *)ap->a_data, ap->a_cred)); default: return (ENOTTY); } } /* Global vfs data structures for ufs. */ struct vop_vector ufs_vnodeops = { .vop_default = &default_vnodeops, .vop_fsync = VOP_PANIC, .vop_read = VOP_PANIC, .vop_reallocblks = VOP_PANIC, .vop_write = VOP_PANIC, .vop_accessx = ufs_accessx, .vop_bmap = ufs_bmap, .vop_cachedlookup = ufs_lookup, .vop_close = ufs_close, .vop_create = ufs_create, .vop_getattr = ufs_getattr, .vop_inactive = ufs_inactive, .vop_ioctl = ufs_ioctl, .vop_link = ufs_link, .vop_lookup = vfs_cache_lookup, .vop_markatime = ufs_markatime, .vop_mkdir = ufs_mkdir, .vop_mknod = ufs_mknod, .vop_open = ufs_open, .vop_pathconf = ufs_pathconf, .vop_poll = vop_stdpoll, .vop_print = ufs_print, .vop_readdir = ufs_readdir, .vop_readlink = ufs_readlink, .vop_reclaim = ufs_reclaim, .vop_remove = ufs_remove, .vop_rename = ufs_rename, .vop_rmdir = ufs_rmdir, .vop_setattr = ufs_setattr, #ifdef MAC .vop_setlabel = vop_stdsetlabel_ea, #endif .vop_strategy = ufs_strategy, .vop_symlink = ufs_symlink, .vop_whiteout = ufs_whiteout, #ifdef UFS_EXTATTR .vop_getextattr = ufs_getextattr, .vop_deleteextattr = ufs_deleteextattr, .vop_setextattr = ufs_setextattr, #endif #ifdef UFS_ACL .vop_getacl = ufs_getacl, .vop_setacl = ufs_setacl, .vop_aclcheck = ufs_aclcheck, #endif }; struct vop_vector ufs_fifoops = { .vop_default = &fifo_specops, .vop_fsync = VOP_PANIC, .vop_accessx = ufs_accessx, .vop_close = ufsfifo_close, .vop_getattr = ufs_getattr, .vop_inactive = ufs_inactive, .vop_kqfilter = ufsfifo_kqfilter, .vop_markatime = ufs_markatime, .vop_pathconf = ufs_pathconf, .vop_print = ufs_print, .vop_read = VOP_PANIC, .vop_reclaim = ufs_reclaim, .vop_setattr = ufs_setattr, #ifdef MAC .vop_setlabel = vop_stdsetlabel_ea, #endif .vop_write = VOP_PANIC, #ifdef UFS_EXTATTR .vop_getextattr = ufs_getextattr, .vop_deleteextattr = ufs_deleteextattr, .vop_setextattr = ufs_setextattr, #endif #ifdef UFS_ACL .vop_getacl = ufs_getacl, .vop_setacl = ufs_setacl, .vop_aclcheck = ufs_aclcheck, #endif }; Index: projects/import-googletest-1.8.1/sys/x86/isa/atrtc.c =================================================================== --- projects/import-googletest-1.8.1/sys/x86/isa/atrtc.c (revision 345025) +++ projects/import-googletest-1.8.1/sys/x86/isa/atrtc.c (revision 345026) @@ -1,654 +1,659 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Poul-Henning Kamp * Copyright (c) 2010 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include __FBSDID("$FreeBSD$"); #include "opt_acpi.h" #include "opt_isa.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_ISA #include #include #endif #include #include "clock_if.h" #ifdef DEV_ACPI #include #include #include #include #endif /* * atrtc_lock protects low-level access to individual hardware registers. * atrtc_time_lock protects the entire sequence of accessing multiple registers * to read or write the date and time. */ static struct mtx atrtc_lock; MTX_SYSINIT(atrtc_lock_init, &atrtc_lock, "atrtc", MTX_SPIN); /* Force RTC enabled/disabled. */ static int atrtc_enabled = -1; TUNABLE_INT("hw.atrtc.enabled", &atrtc_enabled); struct mtx atrtc_time_lock; MTX_SYSINIT(atrtc_time_lock_init, &atrtc_time_lock, "atrtc_time", MTX_DEF); int atrtcclock_disable = 0; static int rtc_reg = -1; static u_char rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF; static u_char rtc_statusb = RTCSB_24HR; +#ifdef DEV_ACPI +#define _COMPONENT ACPI_TIMER +ACPI_MODULE_NAME("ATRTC") +#endif + /* * RTC support routines */ static inline u_char rtcin_locked(int reg) { if (rtc_reg != reg) { inb(0x84); outb(IO_RTC, reg); rtc_reg = reg; inb(0x84); } return (inb(IO_RTC + 1)); } static inline void rtcout_locked(int reg, u_char val) { if (rtc_reg != reg) { inb(0x84); outb(IO_RTC, reg); rtc_reg = reg; inb(0x84); } outb(IO_RTC + 1, val); inb(0x84); } int rtcin(int reg) { u_char val; mtx_lock_spin(&atrtc_lock); val = rtcin_locked(reg); mtx_unlock_spin(&atrtc_lock); return (val); } void writertc(int reg, u_char val) { mtx_lock_spin(&atrtc_lock); rtcout_locked(reg, val); mtx_unlock_spin(&atrtc_lock); } static void atrtc_start(void) { mtx_lock_spin(&atrtc_lock); rtcout_locked(RTC_STATUSA, rtc_statusa); rtcout_locked(RTC_STATUSB, RTCSB_24HR); mtx_unlock_spin(&atrtc_lock); } static void atrtc_rate(unsigned rate) { rtc_statusa = RTCSA_DIVIDER | rate; writertc(RTC_STATUSA, rtc_statusa); } static void atrtc_enable_intr(void) { rtc_statusb |= RTCSB_PINTR; mtx_lock_spin(&atrtc_lock); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); mtx_unlock_spin(&atrtc_lock); } static void atrtc_disable_intr(void) { rtc_statusb &= ~RTCSB_PINTR; mtx_lock_spin(&atrtc_lock); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); mtx_unlock_spin(&atrtc_lock); } void atrtc_restore(void) { /* Restore all of the RTC's "status" (actually, control) registers. */ mtx_lock_spin(&atrtc_lock); rtcin_locked(RTC_STATUSA); /* dummy to get rtc_reg set */ rtcout_locked(RTC_STATUSB, RTCSB_24HR); rtcout_locked(RTC_STATUSA, rtc_statusa); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); mtx_unlock_spin(&atrtc_lock); } /********************************************************************** * RTC driver for subr_rtc */ struct atrtc_softc { int port_rid, intr_rid; struct resource *port_res; struct resource *intr_res; void *intr_handler; struct eventtimer et; #ifdef DEV_ACPI ACPI_HANDLE acpi_handle; #endif }; static int rtc_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { atrtc_rate(max(fls(period + (period >> 1)) - 17, 1)); atrtc_enable_intr(); return (0); } static int rtc_stop(struct eventtimer *et) { atrtc_disable_intr(); return (0); } /* * This routine receives statistical clock interrupts from the RTC. * As explained above, these occur at 128 interrupts per second. * When profiling, we receive interrupts at a rate of 1024 Hz. * * This does not actually add as much overhead as it sounds, because * when the statistical clock is active, the hardclock driver no longer * needs to keep (inaccurate) statistics on its own. This decouples * statistics gathering from scheduling interrupts. * * The RTC chip requires that we read status register C (RTC_INTR) * to acknowledge an interrupt, before it will generate the next one. * Under high interrupt load, rtcintr() can be indefinitely delayed and * the clock can tick immediately after the read from RTC_INTR. In this * case, the mc146818A interrupt signal will not drop for long enough * to register with the 8259 PIC. If an interrupt is missed, the stat * clock will halt, considerably degrading system performance. This is * why we use 'while' rather than a more straightforward 'if' below. * Stat clock ticks can still be lost, causing minor loss of accuracy * in the statistics, but the stat clock will no longer stop. */ static int rtc_intr(void *arg) { struct atrtc_softc *sc = (struct atrtc_softc *)arg; int flag = 0; while (rtcin(RTC_INTR) & RTCIR_PERIOD) { flag = 1; if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); } return(flag ? FILTER_HANDLED : FILTER_STRAY); } #ifdef DEV_ACPI /* * ACPI RTC CMOS address space handler */ #define ATRTC_LAST_REG 0x40 static void rtcin_region(int reg, void *buf, int len) { u_char *ptr = buf; /* Drop lock after each IO as intr and settime have greater priority */ while (len-- > 0) *ptr++ = rtcin(reg++) & 0xff; } static void rtcout_region(int reg, const void *buf, int len) { const u_char *ptr = buf; while (len-- > 0) writertc(reg++, *ptr++); } static bool atrtc_check_cmos_access(bool is_read, ACPI_PHYSICAL_ADDRESS addr, UINT32 len) { /* Block address space wrapping on out-of-bound access */ if (addr >= ATRTC_LAST_REG || addr + len > ATRTC_LAST_REG) return (false); if (is_read) { /* Reading 0x0C will muck with interrupts */ if (addr <= RTC_INTR && addr + len > RTC_INTR) return (false); } else { /* * Allow single-byte writes to alarm registers and * multi-byte writes to addr >= 0x30, else deny. */ if (!((len == 1 && (addr == RTC_SECALRM || addr == RTC_MINALRM || addr == RTC_HRSALRM)) || addr >= 0x30)) return (false); } return (true); } static ACPI_STATUS atrtc_acpi_cmos_handler(UINT32 func, ACPI_PHYSICAL_ADDRESS addr, UINT32 bitwidth, UINT64 *value, void *context, void *region_context) { device_t dev = context; UINT32 bytewidth = howmany(bitwidth, 8); bool is_read = func == ACPI_READ; /* ACPICA is very verbose on CMOS handler failures, so we, too */ #define CMOS_HANDLER_ERR(fmt, ...) \ device_printf(dev, "ACPI [SystemCMOS] handler: " fmt, ##__VA_ARGS__) ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); if (value == NULL) { CMOS_HANDLER_ERR("NULL parameter\n"); return (AE_BAD_PARAMETER); } if (bitwidth == 0 || (bitwidth & 0x07) != 0) { CMOS_HANDLER_ERR("Invalid bitwidth: %u\n", bitwidth); return (AE_BAD_PARAMETER); } if (!atrtc_check_cmos_access(is_read, addr, bytewidth)) { CMOS_HANDLER_ERR("%s access rejected: addr=%#04jx, len=%u\n", is_read ? "Read" : "Write", (uintmax_t)addr, bytewidth); return (AE_BAD_PARAMETER); } switch (func) { case ACPI_READ: rtcin_region(addr, value, bytewidth); break; case ACPI_WRITE: rtcout_region(addr, value, bytewidth); break; default: CMOS_HANDLER_ERR("Invalid function: %u\n", func); return (AE_BAD_PARAMETER); } ACPI_VPRINT(dev, acpi_device_get_parent_softc(dev), "ACPI RTC CMOS %s access: addr=%#04x, len=%u, val=%*D\n", is_read ? "read" : "write", (unsigned)addr, bytewidth, bytewidth, value, " "); return (AE_OK); } static int atrtc_reg_acpi_cmos_handler(device_t dev) { struct atrtc_softc *sc = device_get_softc(dev); ACPI_FUNCTION_TRACE((char *)(uintptr_t) __func__); /* Don't handle address space events if driver is disabled. */ if (acpi_disabled("atrtc")) return (ENXIO); sc->acpi_handle = acpi_get_handle(dev); if (sc->acpi_handle == NULL || ACPI_FAILURE(AcpiInstallAddressSpaceHandler(sc->acpi_handle, ACPI_ADR_SPACE_CMOS, atrtc_acpi_cmos_handler, NULL, dev))) { sc->acpi_handle = NULL; device_printf(dev, "Can't register ACPI CMOS address space handler\n"); return (ENXIO); } return (0); } static int atrtc_unreg_acpi_cmos_handler(device_t dev) { struct atrtc_softc *sc = device_get_softc(dev); ACPI_FUNCTION_TRACE((char *)(uintptr_t) __func__); if (sc->acpi_handle != NULL) AcpiRemoveAddressSpaceHandler(sc->acpi_handle, ACPI_ADR_SPACE_CMOS, atrtc_acpi_cmos_handler); return (0); } #endif /* DEV_ACPI */ /* * Attach to the ISA PnP descriptors for the timer and realtime clock. */ static struct isa_pnp_id atrtc_ids[] = { { 0x000bd041 /* PNP0B00 */, "AT realtime clock" }, { 0 } }; static bool atrtc_acpi_disabled(void) { #ifdef DEV_ACPI uint16_t flags; if (!acpi_get_fadt_bootflags(&flags)) return (false); return ((flags & ACPI_FADT_NO_CMOS_RTC) != 0); #else return (false); #endif } static int atrtc_probe(device_t dev) { int result; if ((atrtc_enabled == -1 && atrtc_acpi_disabled()) || (atrtc_enabled == 0)) return (ENXIO); result = ISA_PNP_PROBE(device_get_parent(dev), dev, atrtc_ids); /* ENOENT means no PnP-ID, device is hinted. */ if (result == ENOENT) { device_set_desc(dev, "AT realtime clock"); return (BUS_PROBE_LOW_PRIORITY); } return (result); } static int atrtc_attach(device_t dev) { struct atrtc_softc *sc; rman_res_t s; int i; sc = device_get_softc(dev); sc->port_res = bus_alloc_resource(dev, SYS_RES_IOPORT, &sc->port_rid, IO_RTC, IO_RTC + 1, 2, RF_ACTIVE); if (sc->port_res == NULL) device_printf(dev, "Warning: Couldn't map I/O.\n"); atrtc_start(); clock_register(dev, 1000000); bzero(&sc->et, sizeof(struct eventtimer)); if (!atrtcclock_disable && (resource_int_value(device_get_name(dev), device_get_unit(dev), "clock", &i) != 0 || i != 0)) { sc->intr_rid = 0; while (bus_get_resource(dev, SYS_RES_IRQ, sc->intr_rid, &s, NULL) == 0 && s != 8) sc->intr_rid++; sc->intr_res = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->intr_rid, 8, 8, 1, RF_ACTIVE); if (sc->intr_res == NULL) { device_printf(dev, "Can't map interrupt.\n"); return (0); } else if ((bus_setup_intr(dev, sc->intr_res, INTR_TYPE_CLK, rtc_intr, NULL, sc, &sc->intr_handler))) { device_printf(dev, "Can't setup interrupt.\n"); return (0); } else { /* Bind IRQ to BSP to avoid live migration. */ bus_bind_intr(dev, sc->intr_res, 0); } sc->et.et_name = "RTC"; sc->et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_POW2DIV; sc->et.et_quality = 0; sc->et.et_frequency = 32768; sc->et.et_min_period = 0x00080000; sc->et.et_max_period = 0x80000000; sc->et.et_start = rtc_start; sc->et.et_stop = rtc_stop; sc->et.et_priv = dev; et_register(&sc->et); } return(0); } static int atrtc_isa_attach(device_t dev) { return (atrtc_attach(dev)); } #ifdef DEV_ACPI static int atrtc_acpi_attach(device_t dev) { int ret; ret = atrtc_attach(dev); if (ret) return (ret); (void)atrtc_reg_acpi_cmos_handler(dev); return (0); } static int atrtc_acpi_detach(device_t dev) { (void)atrtc_unreg_acpi_cmos_handler(dev); return (0); } #endif /* DEV_ACPI */ static int atrtc_resume(device_t dev) { atrtc_restore(); return(0); } static int atrtc_settime(device_t dev __unused, struct timespec *ts) { struct bcd_clocktime bct; clock_ts_to_bcd(ts, &bct, false); clock_dbgprint_bcd(dev, CLOCK_DBG_WRITE, &bct); mtx_lock(&atrtc_time_lock); mtx_lock_spin(&atrtc_lock); /* Disable RTC updates and interrupts. */ rtcout_locked(RTC_STATUSB, RTCSB_HALT | RTCSB_24HR); /* Write all the time registers. */ rtcout_locked(RTC_SEC, bct.sec); rtcout_locked(RTC_MIN, bct.min); rtcout_locked(RTC_HRS, bct.hour); rtcout_locked(RTC_WDAY, bct.dow + 1); rtcout_locked(RTC_DAY, bct.day); rtcout_locked(RTC_MONTH, bct.mon); rtcout_locked(RTC_YEAR, bct.year & 0xff); #ifdef USE_RTC_CENTURY rtcout_locked(RTC_CENTURY, bct.year >> 8); #endif /* * Re-enable RTC updates and interrupts. */ rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); mtx_unlock_spin(&atrtc_lock); mtx_unlock(&atrtc_time_lock); return (0); } static int atrtc_gettime(device_t dev, struct timespec *ts) { struct bcd_clocktime bct; /* Look if we have a RTC present and the time is valid */ if (!(rtcin(RTC_STATUSD) & RTCSD_PWR)) { device_printf(dev, "WARNING: Battery failure indication\n"); return (EINVAL); } /* * wait for time update to complete * If RTCSA_TUP is zero, we have at least 244us before next update. * This is fast enough on most hardware, but a refinement would be * to make sure that no more than 240us pass after we start reading, * and try again if so. */ mtx_lock(&atrtc_time_lock); while (rtcin(RTC_STATUSA) & RTCSA_TUP) continue; mtx_lock_spin(&atrtc_lock); bct.sec = rtcin_locked(RTC_SEC); bct.min = rtcin_locked(RTC_MIN); bct.hour = rtcin_locked(RTC_HRS); bct.day = rtcin_locked(RTC_DAY); bct.mon = rtcin_locked(RTC_MONTH); bct.year = rtcin_locked(RTC_YEAR); #ifdef USE_RTC_CENTURY bct.year |= rtcin_locked(RTC_CENTURY) << 8; #endif mtx_unlock_spin(&atrtc_lock); mtx_unlock(&atrtc_time_lock); /* dow is unused in timespec conversion and we have no nsec info. */ bct.dow = 0; bct.nsec = 0; clock_dbgprint_bcd(dev, CLOCK_DBG_READ, &bct); return (clock_bcd_to_ts(&bct, ts, false)); } static device_method_t atrtc_isa_methods[] = { /* Device interface */ DEVMETHOD(device_probe, atrtc_probe), DEVMETHOD(device_attach, atrtc_isa_attach), DEVMETHOD(device_detach, bus_generic_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), /* XXX stop statclock? */ DEVMETHOD(device_resume, atrtc_resume), /* clock interface */ DEVMETHOD(clock_gettime, atrtc_gettime), DEVMETHOD(clock_settime, atrtc_settime), { 0, 0 } }; static driver_t atrtc_isa_driver = { "atrtc", atrtc_isa_methods, sizeof(struct atrtc_softc), }; #ifdef DEV_ACPI static device_method_t atrtc_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_probe, atrtc_probe), DEVMETHOD(device_attach, atrtc_acpi_attach), DEVMETHOD(device_detach, atrtc_acpi_detach), /* XXX stop statclock? */ DEVMETHOD(device_resume, atrtc_resume), /* clock interface */ DEVMETHOD(clock_gettime, atrtc_gettime), DEVMETHOD(clock_settime, atrtc_settime), { 0, 0 } }; static driver_t atrtc_acpi_driver = { "atrtc", atrtc_acpi_methods, sizeof(struct atrtc_softc), }; #endif /* DEV_ACPI */ static devclass_t atrtc_devclass; DRIVER_MODULE(atrtc, isa, atrtc_isa_driver, atrtc_devclass, 0, 0); #ifdef DEV_ACPI DRIVER_MODULE(atrtc, acpi, atrtc_acpi_driver, atrtc_devclass, 0, 0); #endif ISA_PNP_INFO(atrtc_ids); Index: projects/import-googletest-1.8.1 =================================================================== --- projects/import-googletest-1.8.1 (revision 345025) +++ projects/import-googletest-1.8.1 (revision 345026) Property changes on: projects/import-googletest-1.8.1 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r344997-345025